From d8afcb0692b63cea2b019b7941c84e7667aedff7 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 10 Sep 2018 17:25:41 -0500 Subject: [PATCH] Text classification performance upate. --- BotSharp.Algorithm/Bayes/NaiveBayes.cs | 9 ++- .../NaiveBayesClassifierTest.cs | 9 +-- BotSharp.NLP/Classify/ClassifierFactory.cs | 14 +++-- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 56 ++++++++++++------- .../Classify/SentenceFeatureExtractor.cs | 5 +- BotSharp.NLP/Tokenize/Token.cs | 9 ++- 6 files changed, 68 insertions(+), 34 deletions(-) diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs index 2087e222..099ca6eb 100644 --- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs +++ b/BotSharp.Algorithm/Bayes/NaiveBayes.cs @@ -50,10 +50,13 @@ namespace BotSharp.Algorithm.Bayes for (int x = 0; x < features.Count; x++) { var Xn = features[x]; - var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues; + var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues; - // features are independent, so calculate every feature prob and sum them - prob += Math.Log(estomator.Prob(fv, Xn.Value), 2); + if(fv != null) + { + // features are independent, so calculate every feature prob and sum them + prob += Math.Log(estomator.Prob(fv, Xn.Value), 2); + } } return prob; diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 1ba79291..5a243fad 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -31,8 +31,8 @@ namespace BotSharp.NLP.UnitTest { newSentences[i].Label = sentences[i].Label; } - sentences = newSentences; - + sentences = newSentences.Take(10).ToList(); + sentences.Shuffle(); var options = new ClassifyOptions @@ -40,11 +40,12 @@ namespace BotSharp.NLP.UnitTest TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(0.7M); + + var dataset = sentences.Split(0.9M); classifier.Train(dataset.Item1); int correct = 0; - dataset.Item2.ForEach(td => + dataset.Item1.ToList().ForEach(td => { var classes = classifier.Classify(td); if (td.Label == classes[0].Item1) diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index 05b05167..09d62df4 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -29,20 +29,26 @@ namespace BotSharp.NLP.Classify public List> Classify(Sentence sentence) { - var classes = _classifier.Classify(featureExtractor.GetFeatures(sentence.Words), new ClassifyOptions + var options = new ClassifyOptions { - }); + }; + + var features = featureExtractor.GetFeatures(sentence.Words); + + var classes = _classifier.Classify(features, options); return classes.OrderByDescending(x => x.Item2).ToList(); } public void Train(List sentences) { - _classifier.Train(sentences.Select(x => new FeaturesWithLabel + var sents = sentences.Select(x => new FeaturesWithLabel { Label = x.Label, Features = featureExtractor.GetFeatures(x.Words) - }).ToList(), _options); + }).ToList(); + + _classifier.Train(sents, _options); } } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 85796c44..b6586611 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -27,6 +27,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace BotSharp.NLP.Classify { @@ -53,36 +54,51 @@ namespace BotSharp.NLP.Classify }) .ToList(); - var fNames = featureSets[0].Features.Select(x => x.Name) - .Distinct() - .OrderBy(x => x) - .ToList(); + var fNames = new List(); - // combine all features. - var allFeatureValues = new List(); - featureSets.ForEach(fs => fNames.ForEach(fName => allFeatureValues.Add(new Feature(fName, fs.Features.First(x => x.Name == fName).Value)))); + featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name))); + fNames = fNames.OrderBy(x => x).Distinct().ToList(); - var featureValues = fNames.Select(fn => new + var featureValues = new Dictionary>(); + + for (int i = 0; i < featureSets.Count; i++) { - Name = fn, - Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList() - }).ToList(); + var fs = featureSets[i]; + featureValues[fs.Label] = new List(); + + fNames.ForEach(fn => + { + Feature feature = null; + for (int j = 0; j < fs.Features.Count; j++) + { + if (fs.Features[j].Name == fn) + { + feature = fs.Features[j]; + break; + } + } + + var fv = new Feature(fn, feature == null ? "False" : feature.Value); + featureValues[fs.Label].Add(fv); + }); + } featuresDist = new List(); labelDist.Select(x => x.Value).ToList().ForEach(label => { - var fSets = featureSets.Where(x => x.Label == label); + var fSets = featureValues[label]; + fNames.ForEach(fName => { - var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName)) - .GroupBy(f => f.Value) - .Select(f => new Probability + var fsv = fSets.Where(fs => fs.Name == fName) + .GroupBy(fs => fs.Value) + .Select(fs => new Probability { - Value = f.Key, - Freq = f.Count() + Value = fs.Key, + Freq = fs.Count() }) - .OrderBy(f => f.Value) + .OrderBy(fs => fs.Value) .ToList(); featuresDist.Add(new FeaturesDistribution @@ -101,8 +117,8 @@ namespace BotSharp.NLP.Classify var nb = new NaiveBayes(); nb.LabelDist = labelDist; nb.FeaturesDist = featuresDist; - - labelDist.ForEach(lf => lf.Prob = nb.PosteriorProb(lf.Value, features)); + + Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features)); // add log double[] logs = labelDist.Select(x => x.Prob).ToArray(); diff --git a/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs b/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs index f0f06e58..dde56451 100644 --- a/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs +++ b/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs @@ -13,9 +13,10 @@ namespace BotSharp.NLP.Classify { var features = new List(); - words.Where(x => x.Text.Length > 1) + words.Where(x => x.IsAlpha) + .Distinct() .ToList() - .ForEach(w => features.Add(new Feature("contains", w.Text.ToLower()))); + .ForEach(w => features.Add(new Feature($"contains {w.Text.ToLower()}", "True"))); return features; } diff --git a/BotSharp.NLP/Tokenize/Token.cs b/BotSharp.NLP/Tokenize/Token.cs index c11cd3ed..e6637642 100644 --- a/BotSharp.NLP/Tokenize/Token.cs +++ b/BotSharp.NLP/Tokenize/Token.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using System.Text.RegularExpressions; namespace BotSharp.NLP.Tokenize { @@ -41,7 +42,13 @@ namespace BotSharp.NLP.Tokenize /// /// Is the token an alpha character? /// - public bool IsAlpha { get; set; } + public bool IsAlpha + { + get + { + return Regex.IsMatch(Text, @"^[a-zA-Z]+$"); + } + } /// /// Is the token part of a stop list, i.e. the most common words of the language?