Abstractor BotSharpIntentClassifier

BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs

@@ -14,21 +14,15 @@ using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.BotSharp
 {
-    public class BotSharpNBayesClassifier : INlpTrain, INlpPredict
+    public class BotSharpIntentClassifier : INlpTrain, INlpPredict
     {
         public IConfiguration Configuration { get; set; }
         public PipeSettings Settings { get; set; }
+        private ClassifierFactory<SentenceFeatureExtractor> _classifier;
 
         public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
         {
-            meta.Model = "classification-nb.model";
+            Init(meta);
-            string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
-
-            var options = new ClassifyOptions
-            {
-                ModelFilePath = modelFileName
-            };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
 
             var sentences = doc.Sentences.Select(x => new Sentence
             {
@@ -37,20 +31,16 @@ namespace BotSharp.Core.Engines.BotSharp
                 Words = x.Tokens
             }).ToList();
 
-            classifier.Train(sentences);
+            _classifier.Train(sentences);
 
-            Console.WriteLine($"Saved model to {modelFileName}");
+            Console.WriteLine($"Saved model to {Settings.ModelDir}");
 
             return true;
         }
 
         public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
         {
-            var options = new ClassifyOptions
+            Init(meta);
-            {
-                ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model)
-            };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
 
             var sentence = doc.Sentences.Select(s => new Sentence
             {
@@ -59,16 +49,37 @@ namespace BotSharp.Core.Engines.BotSharp
             }).First();
 
 
-            var result = classifier.Classify(sentence);
+            var result = _classifier.Classify(sentence);
 
             doc.Sentences[0].Intent = new TextClassificationResult
             {
-                Classifier = "BotSharpNBayesClassifier",
+                Classifier = "BotSharpIntentClassifier",
                 Label = result.First().Item1,
                 Confidence = (decimal)result.First().Item2
             };
 
             return true;
         }
+
+        private void Init(PipeModel meta)
+        {
+            if (_classifier == null)
+            {
+                meta.Model = "intent.model";
+
+                string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
+
+                var options = new ClassifyOptions
+                {
+                    ModelFilePath = modelFileName
+                };
+
+                _classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
+
+                string classifierName = Configuration.GetValue<String>($"classifer");
+
+                _classifier.GetClassifer(classifierName);
+            }
+        }
     }
 }

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -44,7 +44,7 @@ namespace BotSharp.NLP.UnitTest
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
                 Dimension = 100
             };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+            var classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
             
             var dataset = sentences.Split(0.7M);
             classifier.Train(dataset.Item1);
@@ -73,7 +73,7 @@ namespace BotSharp.NLP.UnitTest
             {
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Gender")
             };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier, WordFeatureExtractor>(options, SupportedLanguage.English);
+            var classifier = new ClassifierFactory<WordFeatureExtractor>(options, SupportedLanguage.English);
 
             var corpus = GetLabeledCorpus(options);
 
@@ -159,7 +159,7 @@ namespace BotSharp.NLP.UnitTest
                 ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"),
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify")
             };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+            var classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
 
             var dataset = sentences.Split(0.7M);
             classifier.Train(dataset.Item1);

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -3,17 +3,17 @@ using BotSharp.NLP.Tokenize;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reflection;
 using System.Text;
 
 namespace BotSharp.NLP.Classify
 {
-    public class ClassifierFactory<IClassify, IFeatureExtractor> 
+    public class ClassifierFactory<IFeatureExtractor> 
-        where IClassify : IClassifier, new() 
         where IFeatureExtractor : ITextFeatureExtractor, new()
     {
         private SupportedLanguage _lang;
 
-        private IClassify _classifier;
+        private IClassifier _classifier;
 
         private ClassifyOptions _options;
 
@@ -23,10 +23,25 @@ namespace BotSharp.NLP.Classify
         {
             _lang = lang;
             _options = options;
-            _classifier = new IClassify();
             featureExtractor = new IFeatureExtractor();
         }
 
+        public IClassifier GetClassifer(string name)
+        {
+            List<Type> types = new List<Type>();
+
+            types.AddRange(Assembly.Load(new AssemblyName("BotSharp.Core"))
+                .GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
+
+            types.AddRange(Assembly.Load(new AssemblyName("BotSharp.NLP"))
+                .GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
+
+            Type type = types.FirstOrDefault(x => x.Name == name);
+            var instance = (IClassifier)Activator.CreateInstance(type);
+
+            return _classifier = instance;
+        }
+
         public void Train(List<Sentence> sentences)
         {
             _classifier.Train(sentences, _options);

BotSharp.NLP/Classify/SVMClassifier.cs

@@ -22,6 +22,8 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using BotSharp.Algorithm.Features;
+using BotSharp.NLP.Featuring;
+using BotSharp.NLP.Txt2Vec;
 using SVM.BotSharp.MachineLearning;
 using Txt2Vec;
 
@@ -32,6 +34,8 @@ namespace BotSharp.NLP.Classify
     /// </summary>
     public class SVMClassifier : IClassifier
     {
+        private List<string> words;
+
         public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options)
         {
             Problem predict = new Problem();
@@ -50,7 +54,26 @@ namespace BotSharp.NLP.Classify
 
         public void Train(List<Sentence> sentences, ClassifyOptions options)
         {
-            // SVMClassifierTrain(featureSets, options);
+            var tfidf = new TfIdfFeatureExtractor();
+            tfidf.Dimension = options.Dimension;
+            tfidf.Sentences = sentences;
+            tfidf.CalBasedOnCategory();
+
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = sentences;
+            encoder.Words = tfidf.Keywords();
+            words = encoder.EncodeAll();
+
+            var featureSets = new List<FeaturesWithLabel>();
+            sentences.ForEach(sent =>
+            {
+                var fl = new FeaturesWithLabel();
+                fl.Label = sent.Label;
+                fl.Features = sent.Words.Select(x => new Feature(words.IndexOf(x.Lemma).ToString(), words.Contains(x.Lemma) ? "1" : "0")).ToList();
+                featureSets.Add(fl);
+            });
+
+            SVMClassifierTrain(featureSets, options);
         }
 
         public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
@@ -93,10 +116,11 @@ namespace BotSharp.NLP.Classify
 
         public List<double> GetLabels(List<FeaturesWithLabel> featureSets)
         {
+            var categories = featureSets.Select(x => x.Label).Distinct().OrderBy(x => x).ToList();
             List<double> labels = new List<double>();
             foreach (var labelFeatureSet in featureSets)
             {
-                labels.Add(double.Parse(labelFeatureSet.Label));
+                labels.Add(double.Parse(categories.IndexOf(labelFeatureSet.Label).ToString()));
             }
 
             return labels;

BotSharp.WebHost/Settings/bot.json

@@ -7,28 +7,29 @@
     },
 
     "Pipe": {
-      "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier",
+      "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier",
-      "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier"
+      "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier"
     },
-	
+
-	"BotSharpTokenizer": {
+    "BotSharpTokenizer": {
-		"tokenizer": "TreebankTokenizer"
+      "tokenizer": "TreebankTokenizer"
-	},
+    },
-	
+
-	"BotSharpNBayesClassifier": {
+    "BotSharpIntentClassifier": {
-	},
+      "classifer": "NaiveBayesClassifier"
-	
+    },
+
     "BotSharpSVMClassifier": {
       "wordvec": ""
     },
 
     "BotSharpTagger": {
-		"tagger": "NGramTagger"
+      "tagger": "NGramTagger"
+    },
+
+    "BotSharpCRFNer": {
+      "template": "|App_Data|CRFLite/template.en"
     },
-	
-	"BotSharpCRFNer": {
-		"template": "|App_Data|CRFLite/template.en"
-	},
 
     "CRFsuiteEntityRecognizer": {
       "fields": "y w pos chk",