Text classification performance upate.

BotSharp.Algorithm/Bayes/NaiveBayes.cs

@@ -50,10 +50,13 @@ namespace BotSharp.Algorithm.Bayes
             for (int x = 0; x < features.Count; x++)
             {
                 var Xn = features[x];
-                var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues;
+                var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues;
 
-                // features are independent, so calculate every feature prob and sum them
+                if(fv != null)
-                prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
+                {
+                    // features are independent, so calculate every feature prob and sum them
+                    prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
+                }
             }
 
             return prob;

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -31,8 +31,8 @@ namespace BotSharp.NLP.UnitTest
             {
                 newSentences[i].Label = sentences[i].Label;
             }
-            sentences = newSentences;
+            sentences = newSentences.Take(10).ToList();
-
+            
             sentences.Shuffle();
 
             var options = new ClassifyOptions
@@ -40,11 +40,12 @@ namespace BotSharp.NLP.UnitTest
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
             };
             var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
-            var dataset = sentences.Split(0.7M);
+            
+            var dataset = sentences.Split(0.9M);
             classifier.Train(dataset.Item1);
 
             int correct = 0;
-            dataset.Item2.ForEach(td =>
+            dataset.Item1.ToList().ForEach(td =>
             {
                 var classes = classifier.Classify(td);
                 if (td.Label == classes[0].Item1)

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -29,20 +29,26 @@ namespace BotSharp.NLP.Classify
 
         public List<Tuple<string, double>> Classify(Sentence sentence)
         {
-            var classes = _classifier.Classify(featureExtractor.GetFeatures(sentence.Words), new ClassifyOptions
+            var options = new ClassifyOptions
             {
-            });
+            };
+
+            var features = featureExtractor.GetFeatures(sentence.Words);
+
+            var classes = _classifier.Classify(features, options);
 
             return classes.OrderByDescending(x => x.Item2).ToList();
         }
 
         public void Train(List<Sentence> sentences)
         {
-            _classifier.Train(sentences.Select(x => new FeaturesWithLabel
+            var sents = sentences.Select(x => new FeaturesWithLabel
             {
                 Label = x.Label,
                 Features = featureExtractor.GetFeatures(x.Words)
-            }).ToList(), _options);
+            }).ToList();
+
+            _classifier.Train(sents, _options);
         }
     }
 }

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -27,6 +27,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.NLP.Classify
 {
@@ -53,36 +54,51 @@ namespace BotSharp.NLP.Classify
                 })
                 .ToList();
 
-            var fNames = featureSets[0].Features.Select(x => x.Name)
+            var fNames = new List<string>();
-                .Distinct()
-                .OrderBy(x => x)
-                .ToList();
 
-            // combine all features.
+            featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name)));
-            var allFeatureValues = new List<Feature>();
+            fNames = fNames.OrderBy(x => x).Distinct().ToList();
-            featureSets.ForEach(fs => fNames.ForEach(fName => allFeatureValues.Add(new Feature(fName, fs.Features.First(x => x.Name == fName).Value))));
 
-            var featureValues = fNames.Select(fn => new
+            var featureValues = new Dictionary<string, List<Feature>>();
+
+            for (int i = 0; i < featureSets.Count; i++)
             {
-                Name = fn,
+                var fs = featureSets[i];
-                Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList()
+                featureValues[fs.Label] = new List<Feature>();
-            }).ToList();
+
+                fNames.ForEach(fn =>
+                {
+                    Feature feature = null;
+                    for (int j = 0; j < fs.Features.Count; j++)
+                    {
+                        if (fs.Features[j].Name == fn)
+                        {
+                            feature = fs.Features[j];
+                            break;
+                        }
+                    }
+
+                    var fv = new Feature(fn, feature == null ? "False" : feature.Value);
+                    featureValues[fs.Label].Add(fv);
+                });
+            }
 
             featuresDist = new List<FeaturesDistribution>();
 
             labelDist.Select(x => x.Value).ToList().ForEach(label =>
             {
-                var fSets = featureSets.Where(x => x.Label == label);
+                var fSets = featureValues[label];
+
                 fNames.ForEach(fName =>
                 {
-                    var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName))
+                    var fsv = fSets.Where(fs => fs.Name == fName)
-                        .GroupBy(f => f.Value)
+                        .GroupBy(fs => fs.Value)
-                        .Select(f => new Probability
+                        .Select(fs => new Probability
                         {
-                            Value = f.Key,
+                            Value = fs.Key,
-                            Freq = f.Count()
+                            Freq = fs.Count()
                         })
-                        .OrderBy(f => f.Value)
+                        .OrderBy(fs => fs.Value)
                         .ToList();
 
                     featuresDist.Add(new FeaturesDistribution
@@ -101,8 +117,8 @@ namespace BotSharp.NLP.Classify
             var nb = new NaiveBayes<Lidstone>();
             nb.LabelDist = labelDist;
             nb.FeaturesDist = featuresDist;
-            
+
-            labelDist.ForEach(lf => lf.Prob = nb.PosteriorProb(lf.Value, features));
+            Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features));
 
             // add log
             double[] logs = labelDist.Select(x => x.Prob).ToArray();

BotSharp.NLP/Classify/SentenceFeatureExtractor.cs

@@ -13,9 +13,10 @@ namespace BotSharp.NLP.Classify
         {
             var features = new List<Feature>();
 
-            words.Where(x => x.Text.Length > 1)
+            words.Where(x => x.IsAlpha)
+                .Distinct()
                 .ToList()
-                .ForEach(w => features.Add(new Feature("contains", w.Text.ToLower())));
+                .ForEach(w => features.Add(new Feature($"contains {w.Text.ToLower()}", "True")));
 
             return features;
         }

BotSharp.NLP/Tokenize/Token.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Text.RegularExpressions;
 
 namespace BotSharp.NLP.Tokenize
 {
@@ -41,7 +42,13 @@ namespace BotSharp.NLP.Tokenize
         /// <summary>
         /// Is the token an alpha character?
         /// </summary>
-        public bool IsAlpha { get; set; }
+        public bool IsAlpha
+        {
+            get
+            {
+                return Regex.IsMatch(Text, @"^[a-zA-Z]+$");
+            }
+        }
 
         /// <summary>
         /// Is the token part of a stop list, i.e. the most common words of the language?