New NBayesClassifier for Chatbot

BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs

@@ -0,0 +1,57 @@
+using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using BotSharp.NLP;
+using BotSharp.NLP.Classify;
+using BotSharp.NLP.Txt2Vec;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Core.Engines.BotSharp
+{
+    public class BotSharpNBayesClassifier : INlpTrain, INlpPredict
+    {
+        public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
+
+        public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
+        {
+            meta.Model = "classification-nb.model";
+
+            string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
+
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence
+            {
+                Label = x.Intent.Label,
+                Text = x.Text,
+                Words = x.Tokens
+            }).ToList();
+            encoder.EncodeAll();
+
+            var options = new ClassifyOptions
+            {
+                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
+            };
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+
+            classifier.Train(encoder.Sentences);
+
+            Console.WriteLine($"Saved model to {modelFileName}");
+            meta.Meta = new JObject();
+            meta.Meta["compiled at"] = "Sep 12, 2018";
+
+            return true;
+        }
+
+        public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
+        {
+            return true;
+        }
+    }
+}

BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs

@@ -1,5 +1,6 @@
 using BotSharp.Core.Abstractions;
 using BotSharp.Core.Agents;
+using BotSharp.Core.Intents;
 using BotSharp.NLP;
 using BotSharp.NLP.Tokenize;
 using Microsoft.Extensions.Configuration;
@@ -14,14 +15,12 @@ namespace BotSharp.Core.Engines.BotSharp
     {
         public IConfiguration Configuration { get; set; }
         public PipeSettings Settings { get; set; }
-        private TokenizerFactory<RegexTokenizer> _tokenizer;
+        private TokenizerFactory<TreebankTokenizer> _tokenizer;
 
         public BotSharpTokenizer()
         {
-            _tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
+            _tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
             {
-                Pattern = RegexTokenizer.WORD_PUNC,
-                SpecialWords = new List<string> { "'s" }
             }, SupportedLanguage.English);
         }
 
@@ -48,7 +47,8 @@ namespace BotSharp.Core.Engines.BotSharp
                 doc.Sentences.Add(new NlpDocSentence
                 {
                     Tokens = _tokenizer.Tokenize(say.Text),
-                    Text = say.Text
+                    Text = say.Text,
+                    Intent = new TextClassificationResult { Label = say.Intent }
                 });
             });
 

Settings/bot.json

@@ -7,10 +7,13 @@
     },
 
     "Pipe": {
-      "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier",
-      "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier"
+      "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier",
+      "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier"
     },
-
+	
+	"BotSharpNBayesClassifier": {
+	},
+	
     "BotSharpSVMClassifier": {
       "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data"
     },