Multinomial Naive Bayes

BotSharp.Algorithm/Bayes/NaiveBayes.cs

@@ -1,5 +1,6 @@
-using BotSharp.Algorithm.Extensions;
-using BotSharp.Algorithm.Formulas;
+using BotSharp.Algorithm.Estimators;
+using BotSharp.Algorithm.Features;
+using BotSharp.Algorithm.Statistics;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -10,12 +11,12 @@ namespace BotSharp.Algorithm.Bayes
     /// <summary>
     /// https://en.wikipedia.org/wiki/Bayes%27_theorem
     /// </summary>
-    public class NaiveBayes<Smoother> where Smoother : ISmoother, new()
+    public class NaiveBayes<Estimator> where Estimator : IEstimator, new()
     {
         /// <summary>
         /// smoothing function
         /// </summary>
-        private Smoother smoother;
+        private Estimator estomator;
 
         public List<FeaturesDistribution> FeaturesDist { get; set; }
 
@@ -23,7 +24,7 @@ namespace BotSharp.Algorithm.Bayes
 
         public NaiveBayes()
         {
-            smoother = new Smoother();
+            estomator = new Estimator();
         }
 
         /// <summary>
@@ -40,7 +41,7 @@ namespace BotSharp.Algorithm.Bayes
             double prob = 0;
 
             // prior probability
-            prob = Math.Log(smoother.Prob(LabelDist, Y), 2);
+            prob = Math.Log(estomator.Prob(LabelDist, Y), 2);
 
             // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
             var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList();
@@ -52,46 +53,10 @@ namespace BotSharp.Algorithm.Bayes
                 var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues;
 
                 // features are independent, so calculate every feature prob and sum them
-                prob += Math.Log(smoother.Prob(fv, Xn.Value), 2);
+                prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
             }
 
             return prob;
         }
     }
-
-    public class Feature
-    {
-        public string Name { get; set; }
-        public string Value { get; set; }
-
-        public Feature(string name, string value)
-        {
-            Name = name;
-            Value = value;
-        }
-    }
-
-    public class FeaturesWithLabel
-    {
-        public List<Feature> Features { get; set; }
-        public string Label { get; set; }
-        public FeaturesWithLabel()
-        {
-            this.Features = new List<Feature>();
-        }
-    }
-
-    public class FeaturesDistribution
-    {
-        public string Label { get; set; }
-
-        public string FeatureName { get; set; }
-
-        public List<Probability> FeatureValues { get; set; }
-
-        public override string ToString()
-        {
-            return $"{Label} {FeatureName} {FeatureValues.Count}";
-        }
-    }
 }

BotSharp.Algorithm/Estimators/IEstimator.cs

@@ -1,10 +1,11 @@
-using System;
+using BotSharp.Algorithm.Statistics;
+using System;
 using System.Collections.Generic;
 using System.Text;
 
-namespace BotSharp.Algorithm
+namespace BotSharp.Algorithm.Estimators
 {
-    public interface ISmoother
+    public interface IEstimator
     {
         double Prob(List<Probability> dist, string sample);
     }

BotSharp.Algorithm/Estimators/Lidstone.cs

@@ -16,20 +16,22 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+using BotSharp.Algorithm.Statistics;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 
-namespace BotSharp.Algorithm.Formulas
+namespace BotSharp.Algorithm.Estimators
 {
     /// <summary>
     /// Lidstone smoothing is a technique used to smooth categorical data. 
     /// In statistics, it's called additive smoothing or Laplace smoothing.
     /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator.
     /// https://en.wikipedia.org/wiki/Additive_smoothing
+    /// Used as Multinomial Naive Bayes
     /// </summary>
-    public class Lidstone : ISmoother
+    public class Lidstone : IEstimator
     {
         /// <summary>
         /// α > 0 is the smoothing parameter

BotSharp.Algorithm/Features/Feature.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Algorithm.Features
+{
+    public class Feature
+    {
+        public string Name { get; set; }
+        public string Value { get; set; }
+
+        public Feature(string name, string value)
+        {
+            Name = name;
+            Value = value;
+        }
+    }
+}

BotSharp.Algorithm/Features/FeaturesDistribution.cs

@@ -0,0 +1,21 @@
+using BotSharp.Algorithm.Statistics;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Algorithm.Features
+{
+    public class FeaturesDistribution
+    {
+        public string Label { get; set; }
+
+        public string FeatureName { get; set; }
+
+        public List<Probability> FeatureValues { get; set; }
+
+        public override string ToString()
+        {
+            return $"{Label} {FeatureName} {FeatureValues.Count}";
+        }
+    }
+}

BotSharp.Algorithm/Statistics/Probability.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace BotSharp.Algorithm
+namespace BotSharp.Algorithm.Statistics
 {
     /// <summary>
     /// In probability theory and statistics, a probability distribution is a mathematical function 

BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj

@@ -10,6 +10,10 @@
     <Configurations>Debug;Release;RASA NLU;DIALOGFLOW;RASA</Configurations>
   </PropertyGroup>
 
+  <ItemGroup>
+    <None Remove="wordvec_enu.bin" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.1.1" />

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -25,8 +25,13 @@ namespace BotSharp.NLP.UnitTest
                 FileName = "cooking.stackexchange.txt"
             });
 
-            var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
-            sentences.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
+            var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);     
+            var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
+            for(int i = 0; i < newSentences.Count; i++)
+            {
+                newSentences[i].Label = sentences[i].Label;
+            }
+            sentences = newSentences;
 
             sentences.Shuffle();
 
@@ -34,7 +39,7 @@ namespace BotSharp.NLP.UnitTest
             {
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
             };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier>(options, SupportedLanguage.English);
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
             var dataset = sentences.Split(0.7M);
             classifier.Train(dataset.Item1);
 
@@ -58,7 +63,7 @@ namespace BotSharp.NLP.UnitTest
             {
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Gender")
             };
-            var classifier = new ClassifierFactory<NaiveBayesClassifier>(options, SupportedLanguage.English);
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, WordFeatureExtractor>(options, SupportedLanguage.English);
 
             var corpus = GetLabeledCorpus(options);
 

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -1,5 +1,4 @@
-using BotSharp.Algorithm.Bayes;
-using BotSharp.NLP.Corpus;
+using BotSharp.Algorithm.Features;
 using BotSharp.NLP.Tokenize;
 using System;
 using System.Collections.Generic;
@@ -8,7 +7,9 @@ using System.Text;
 
 namespace BotSharp.NLP.Classify
 {
-    public class ClassifierFactory<IClassify> where IClassify : IClassifier, new()
+    public class ClassifierFactory<IClassify, IFeatureExtractor> 
+        where IClassify : IClassifier, new() 
+        where IFeatureExtractor : ITextFeatureExtractor, new()
     {
         private SupportedLanguage _lang;
 
@@ -16,16 +17,19 @@ namespace BotSharp.NLP.Classify
 
         private ClassifyOptions _options;
 
+        private IFeatureExtractor featureExtractor;
+
         public ClassifierFactory(ClassifyOptions options, SupportedLanguage lang)
         {
             _lang = lang;
             _options = options;
             _classifier = new IClassify();
+            featureExtractor = new IFeatureExtractor();
         }
 
         public List<Tuple<string, double>> Classify(Sentence sentence)
         {
-            var classes = _classifier.Classify(GetFeatures(sentence.Words), new ClassifyOptions
+            var classes = _classifier.Classify(featureExtractor.GetFeatures(sentence.Words), new ClassifyOptions
             {
             });
 
@@ -37,20 +41,8 @@ namespace BotSharp.NLP.Classify
             _classifier.Train(sentences.Select(x => new FeaturesWithLabel
             {
                 Label = x.Label,
-                Features = GetFeatures(x.Words)
+                Features = featureExtractor.GetFeatures(x.Words)
             }).ToList(), _options);
         }
-
-        private List<Feature> GetFeatures(List<Token> words)
-        {
-            string text = words[0].Text;
-            var features = new List<Feature>();
-
-            features.Add(new Feature("alwayson", "True"));
-            features.Add(new Feature("startswith", text[0].ToString().ToLower()));
-            features.Add(new Feature("endswith", text[text.Length - 1].ToString().ToLower()));
-
-            return features;
-        }
     }
 }

BotSharp.NLP/Classify/IClassifier.cs

@@ -1,4 +1,4 @@
-using BotSharp.Algorithm.Bayes;
+using BotSharp.Algorithm.Features;
 using System;
 using System.Collections.Generic;
 using System.Text;

BotSharp.NLP/Classify/ITextFeatureExtractor.cs

@@ -0,0 +1,16 @@
+using BotSharp.Algorithm.Features;
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Classify
+{
+    /// <summary>
+    /// Featuring text
+    /// </summary>
+    public interface ITextFeatureExtractor
+    {
+        List<Feature> GetFeatures(List<Token> words);
+    }
+}

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -18,8 +18,10 @@
 
 using BotSharp.Algorithm;
 using BotSharp.Algorithm.Bayes;
+using BotSharp.Algorithm.Estimators;
 using BotSharp.Algorithm.Extensions;
-using BotSharp.Algorithm.Formulas;
+using BotSharp.Algorithm.Features;
+using BotSharp.Algorithm.Statistics;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -52,6 +54,7 @@ namespace BotSharp.NLP.Classify
                 .ToList();
 
             var fNames = featureSets[0].Features.Select(x => x.Name)
+                .Distinct()
                 .OrderBy(x => x)
                 .ToList();
 
@@ -120,4 +123,14 @@ namespace BotSharp.NLP.Classify
             return labelDist.Select(x => new Tuple<string, double>(x.Value, x.Prob)).ToList();
         }
     }
+
+    public class FeaturesWithLabel
+    {
+        public List<Feature> Features { get; set; }
+        public string Label { get; set; }
+        public FeaturesWithLabel()
+        {
+            this.Features = new List<Feature>();
+        }
+    }
 }

BotSharp.NLP/Classify/SVMClassifier.cs

@@ -21,7 +21,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
-using BotSharp.Algorithm.Bayes;
+using BotSharp.Algorithm.Features;
 using SVM.BotSharp.MachineLearning;
 using Txt2Vec;
 

BotSharp.NLP/Classify/SentenceFeatureExtractor.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BotSharp.Algorithm.Features;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Classify
+{
+    public class SentenceFeatureExtractor : ITextFeatureExtractor
+    {
+        public List<Feature> GetFeatures(List<Token> words)
+        {
+            var features = new List<Feature>();
+
+            words.Where(x => x.Text.Length > 1)
+                .ToList()
+                .ForEach(w => features.Add(new Feature("contains", w.Text.ToLower())));
+
+            return features;
+        }
+    }
+}

BotSharp.NLP/Classify/WordFeatureExtractor.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BotSharp.Algorithm.Features;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Classify
+{
+    public class WordFeatureExtractor : ITextFeatureExtractor
+    {
+        public List<Feature> GetFeatures(List<Token> words)
+        {
+            string text = words[0].Text;
+            var features = new List<Feature>();
+
+            features.Add(new Feature("alwayson", "True"));
+            features.Add(new Feature("startswith", text[0].ToString().ToLower()));
+            features.Add(new Feature("endswith", text[text.Length - 1].ToString().ToLower()));
+
+            return features;
+        }
+    }
+}

BotSharp.NLP/Tokenize/TokenizerFactory.cs

@@ -32,19 +32,16 @@ namespace BotSharp.NLP.Tokenize
             return _tokenizer.Tokenize(sentence, _options);
         }
 
-        public List<List<Token>> Tokenize(List<String> sentences)
+        public List<Sentence> Tokenize(List<String> sentences)
         {
-            var sents = sentences.Select(s => new ParallelToken { Text = s }).ToList();
+            var sents = sentences.Select(s => new Sentence { Text = s }).ToList();
 
             Parallel.ForEach(sents, (sentence) =>
             {
-                sentence.Tokens = Tokenize(sentence.Text);
+                sentence.Words = Tokenize(sentence.Text);
             });
 
-            List<List<Token>> result = new List<List<Token>>();
-            sents.ForEach(x => result.Add(x.Tokens));
-
-            return result;
+            return sents;
         }
 
         private class ParallelToken