From 1f5bdbbc8207c8adc9f23fa8abb4174ab08de736 Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Sun, 9 Sep 2018 22:56:32 -0500 Subject: [PATCH] Multinomial Naive Bayes --- BotSharp.Algorithm/Bayes/NaiveBayes.cs | 51 +++---------------- .../IEstimator.cs} | 7 +-- .../{Formulas => Estimators}/Lidstone.cs | 6 ++- BotSharp.Algorithm/Features/Feature.cs | 18 +++++++ .../Features/FeaturesDistribution.cs | 21 ++++++++ .../{ => Statistics}/Probability.cs | 2 +- .../BotSharp.NLP.UnitTest.csproj | 4 ++ .../NaiveBayesClassifierTest.cs | 13 +++-- BotSharp.NLP/Classify/ClassifierFactory.cs | 26 ++++------ BotSharp.NLP/Classify/IClassifier.cs | 2 +- .../Classify/ITextFeatureExtractor.cs | 16 ++++++ BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 15 +++++- BotSharp.NLP/Classify/SVMClassifier.cs | 2 +- .../Classify/SentenceFeatureExtractor.cs | 23 +++++++++ BotSharp.NLP/Classify/WordFeatureExtractor.cs | 23 +++++++++ BotSharp.NLP/Tokenize/TokenizerFactory.cs | 11 ++-- 16 files changed, 160 insertions(+), 80 deletions(-) rename BotSharp.Algorithm/{ISmoother.cs => Estimators/IEstimator.cs} (50%) rename BotSharp.Algorithm/{Formulas => Estimators}/Lidstone.cs (92%) create mode 100644 BotSharp.Algorithm/Features/Feature.cs create mode 100644 BotSharp.Algorithm/Features/FeaturesDistribution.cs rename BotSharp.Algorithm/{ => Statistics}/Probability.cs (96%) create mode 100644 BotSharp.NLP/Classify/ITextFeatureExtractor.cs create mode 100644 BotSharp.NLP/Classify/SentenceFeatureExtractor.cs create mode 100644 BotSharp.NLP/Classify/WordFeatureExtractor.cs diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs index e2217130..2087e222 100644 --- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs +++ b/BotSharp.Algorithm/Bayes/NaiveBayes.cs @@ -1,5 +1,6 @@ -using BotSharp.Algorithm.Extensions; -using BotSharp.Algorithm.Formulas; +using BotSharp.Algorithm.Estimators; +using BotSharp.Algorithm.Features; +using BotSharp.Algorithm.Statistics; using System; using System.Collections.Generic; using System.Linq; @@ -10,12 +11,12 @@ namespace BotSharp.Algorithm.Bayes /// /// https://en.wikipedia.org/wiki/Bayes%27_theorem /// - public class NaiveBayes where Smoother : ISmoother, new() + public class NaiveBayes where Estimator : IEstimator, new() { /// /// smoothing function /// - private Smoother smoother; + private Estimator estomator; public List FeaturesDist { get; set; } @@ -23,7 +24,7 @@ namespace BotSharp.Algorithm.Bayes public NaiveBayes() { - smoother = new Smoother(); + estomator = new Estimator(); } /// @@ -40,7 +41,7 @@ namespace BotSharp.Algorithm.Bayes double prob = 0; // prior probability - prob = Math.Log(smoother.Prob(LabelDist, Y), 2); + prob = Math.Log(estomator.Prob(LabelDist, Y), 2); // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList(); @@ -52,46 +53,10 @@ namespace BotSharp.Algorithm.Bayes var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues; // features are independent, so calculate every feature prob and sum them - prob += Math.Log(smoother.Prob(fv, Xn.Value), 2); + prob += Math.Log(estomator.Prob(fv, Xn.Value), 2); } return prob; } } - - public class Feature - { - public string Name { get; set; } - public string Value { get; set; } - - public Feature(string name, string value) - { - Name = name; - Value = value; - } - } - - public class FeaturesWithLabel - { - public List Features { get; set; } - public string Label { get; set; } - public FeaturesWithLabel() - { - this.Features = new List(); - } - } - - public class FeaturesDistribution - { - public string Label { get; set; } - - public string FeatureName { get; set; } - - public List FeatureValues { get; set; } - - public override string ToString() - { - return $"{Label} {FeatureName} {FeatureValues.Count}"; - } - } } diff --git a/BotSharp.Algorithm/ISmoother.cs b/BotSharp.Algorithm/Estimators/IEstimator.cs similarity index 50% rename from BotSharp.Algorithm/ISmoother.cs rename to BotSharp.Algorithm/Estimators/IEstimator.cs index 322dd6ef..9ca8cd8b 100644 --- a/BotSharp.Algorithm/ISmoother.cs +++ b/BotSharp.Algorithm/Estimators/IEstimator.cs @@ -1,10 +1,11 @@ -using System; +using BotSharp.Algorithm.Statistics; +using System; using System.Collections.Generic; using System.Text; -namespace BotSharp.Algorithm +namespace BotSharp.Algorithm.Estimators { - public interface ISmoother + public interface IEstimator { double Prob(List dist, string sample); } diff --git a/BotSharp.Algorithm/Formulas/Lidstone.cs b/BotSharp.Algorithm/Estimators/Lidstone.cs similarity index 92% rename from BotSharp.Algorithm/Formulas/Lidstone.cs rename to BotSharp.Algorithm/Estimators/Lidstone.cs index 78bcc3a3..d1153b16 100644 --- a/BotSharp.Algorithm/Formulas/Lidstone.cs +++ b/BotSharp.Algorithm/Estimators/Lidstone.cs @@ -16,20 +16,22 @@ * along with this program. If not, see . */ +using BotSharp.Algorithm.Statistics; using System; using System.Collections.Generic; using System.Linq; using System.Text; -namespace BotSharp.Algorithm.Formulas +namespace BotSharp.Algorithm.Estimators { /// /// Lidstone smoothing is a technique used to smooth categorical data. /// In statistics, it's called additive smoothing or Laplace smoothing. /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator. /// https://en.wikipedia.org/wiki/Additive_smoothing + /// Used as Multinomial Naive Bayes /// - public class Lidstone : ISmoother + public class Lidstone : IEstimator { /// /// α > 0 is the smoothing parameter diff --git a/BotSharp.Algorithm/Features/Feature.cs b/BotSharp.Algorithm/Features/Feature.cs new file mode 100644 index 00000000..5212206e --- /dev/null +++ b/BotSharp.Algorithm/Features/Feature.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Features +{ + public class Feature + { + public string Name { get; set; } + public string Value { get; set; } + + public Feature(string name, string value) + { + Name = name; + Value = value; + } + } +} diff --git a/BotSharp.Algorithm/Features/FeaturesDistribution.cs b/BotSharp.Algorithm/Features/FeaturesDistribution.cs new file mode 100644 index 00000000..00cf8594 --- /dev/null +++ b/BotSharp.Algorithm/Features/FeaturesDistribution.cs @@ -0,0 +1,21 @@ +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Features +{ + public class FeaturesDistribution + { + public string Label { get; set; } + + public string FeatureName { get; set; } + + public List FeatureValues { get; set; } + + public override string ToString() + { + return $"{Label} {FeatureName} {FeatureValues.Count}"; + } + } +} diff --git a/BotSharp.Algorithm/Probability.cs b/BotSharp.Algorithm/Statistics/Probability.cs similarity index 96% rename from BotSharp.Algorithm/Probability.cs rename to BotSharp.Algorithm/Statistics/Probability.cs index e2c58afa..2629c7e2 100644 --- a/BotSharp.Algorithm/Probability.cs +++ b/BotSharp.Algorithm/Statistics/Probability.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace BotSharp.Algorithm +namespace BotSharp.Algorithm.Statistics { /// /// In probability theory and statistics, a probability distribution is a mathematical function diff --git a/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj b/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj index b851ea1c..b8c19b33 100644 --- a/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj +++ b/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj @@ -10,6 +10,10 @@ Debug;Release;RASA NLU;DIALOGFLOW;RASA + + + + diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 8c16a8e1..1ba79291 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -25,8 +25,13 @@ namespace BotSharp.NLP.UnitTest FileName = "cooking.stackexchange.txt" }); - var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English); - sentences.ForEach(x => x.Words = tokenizer.Tokenize(x.Text)); + var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English); + var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList()); + for(int i = 0; i < newSentences.Count; i++) + { + newSentences[i].Label = sentences[i].Label; + } + sentences = newSentences; sentences.Shuffle(); @@ -34,7 +39,7 @@ namespace BotSharp.NLP.UnitTest { TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + var classifier = new ClassifierFactory(options, SupportedLanguage.English); var dataset = sentences.Split(0.7M); classifier.Train(dataset.Item1); @@ -58,7 +63,7 @@ namespace BotSharp.NLP.UnitTest { TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Gender") }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + var classifier = new ClassifierFactory(options, SupportedLanguage.English); var corpus = GetLabeledCorpus(options); diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index a90d2109..05b05167 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -1,5 +1,4 @@ -using BotSharp.Algorithm.Bayes; -using BotSharp.NLP.Corpus; +using BotSharp.Algorithm.Features; using BotSharp.NLP.Tokenize; using System; using System.Collections.Generic; @@ -8,7 +7,9 @@ using System.Text; namespace BotSharp.NLP.Classify { - public class ClassifierFactory where IClassify : IClassifier, new() + public class ClassifierFactory + where IClassify : IClassifier, new() + where IFeatureExtractor : ITextFeatureExtractor, new() { private SupportedLanguage _lang; @@ -16,16 +17,19 @@ namespace BotSharp.NLP.Classify private ClassifyOptions _options; + private IFeatureExtractor featureExtractor; + public ClassifierFactory(ClassifyOptions options, SupportedLanguage lang) { _lang = lang; _options = options; _classifier = new IClassify(); + featureExtractor = new IFeatureExtractor(); } public List> Classify(Sentence sentence) { - var classes = _classifier.Classify(GetFeatures(sentence.Words), new ClassifyOptions + var classes = _classifier.Classify(featureExtractor.GetFeatures(sentence.Words), new ClassifyOptions { }); @@ -37,20 +41,8 @@ namespace BotSharp.NLP.Classify _classifier.Train(sentences.Select(x => new FeaturesWithLabel { Label = x.Label, - Features = GetFeatures(x.Words) + Features = featureExtractor.GetFeatures(x.Words) }).ToList(), _options); } - - private List GetFeatures(List words) - { - string text = words[0].Text; - var features = new List(); - - features.Add(new Feature("alwayson", "True")); - features.Add(new Feature("startswith", text[0].ToString().ToLower())); - features.Add(new Feature("endswith", text[text.Length - 1].ToString().ToLower())); - - return features; - } } } diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index e52b995c..8197f081 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -1,4 +1,4 @@ -using BotSharp.Algorithm.Bayes; +using BotSharp.Algorithm.Features; using System; using System.Collections.Generic; using System.Text; diff --git a/BotSharp.NLP/Classify/ITextFeatureExtractor.cs b/BotSharp.NLP/Classify/ITextFeatureExtractor.cs new file mode 100644 index 00000000..f43006b0 --- /dev/null +++ b/BotSharp.NLP/Classify/ITextFeatureExtractor.cs @@ -0,0 +1,16 @@ +using BotSharp.Algorithm.Features; +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Classify +{ + /// + /// Featuring text + /// + public interface ITextFeatureExtractor + { + List GetFeatures(List words); + } +} diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 8ef10aaa..85796c44 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -18,8 +18,10 @@ using BotSharp.Algorithm; using BotSharp.Algorithm.Bayes; +using BotSharp.Algorithm.Estimators; using BotSharp.Algorithm.Extensions; -using BotSharp.Algorithm.Formulas; +using BotSharp.Algorithm.Features; +using BotSharp.Algorithm.Statistics; using System; using System.Collections.Generic; using System.IO; @@ -52,6 +54,7 @@ namespace BotSharp.NLP.Classify .ToList(); var fNames = featureSets[0].Features.Select(x => x.Name) + .Distinct() .OrderBy(x => x) .ToList(); @@ -120,4 +123,14 @@ namespace BotSharp.NLP.Classify return labelDist.Select(x => new Tuple(x.Value, x.Prob)).ToList(); } } + + public class FeaturesWithLabel + { + public List Features { get; set; } + public string Label { get; set; } + public FeaturesWithLabel() + { + this.Features = new List(); + } + } } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 4bbc4711..99748ded 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -21,7 +21,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; -using BotSharp.Algorithm.Bayes; +using BotSharp.Algorithm.Features; using SVM.BotSharp.MachineLearning; using Txt2Vec; diff --git a/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs b/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs new file mode 100644 index 00000000..f0f06e58 --- /dev/null +++ b/BotSharp.NLP/Classify/SentenceFeatureExtractor.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using BotSharp.Algorithm.Features; +using BotSharp.NLP.Tokenize; + +namespace BotSharp.NLP.Classify +{ + public class SentenceFeatureExtractor : ITextFeatureExtractor + { + public List GetFeatures(List words) + { + var features = new List(); + + words.Where(x => x.Text.Length > 1) + .ToList() + .ForEach(w => features.Add(new Feature("contains", w.Text.ToLower()))); + + return features; + } + } +} diff --git a/BotSharp.NLP/Classify/WordFeatureExtractor.cs b/BotSharp.NLP/Classify/WordFeatureExtractor.cs new file mode 100644 index 00000000..ccfdfa4c --- /dev/null +++ b/BotSharp.NLP/Classify/WordFeatureExtractor.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BotSharp.Algorithm.Features; +using BotSharp.NLP.Tokenize; + +namespace BotSharp.NLP.Classify +{ + public class WordFeatureExtractor : ITextFeatureExtractor + { + public List GetFeatures(List words) + { + string text = words[0].Text; + var features = new List(); + + features.Add(new Feature("alwayson", "True")); + features.Add(new Feature("startswith", text[0].ToString().ToLower())); + features.Add(new Feature("endswith", text[text.Length - 1].ToString().ToLower())); + + return features; + } + } +} diff --git a/BotSharp.NLP/Tokenize/TokenizerFactory.cs b/BotSharp.NLP/Tokenize/TokenizerFactory.cs index 32efc4e2..5518326e 100644 --- a/BotSharp.NLP/Tokenize/TokenizerFactory.cs +++ b/BotSharp.NLP/Tokenize/TokenizerFactory.cs @@ -32,19 +32,16 @@ namespace BotSharp.NLP.Tokenize return _tokenizer.Tokenize(sentence, _options); } - public List> Tokenize(List sentences) + public List Tokenize(List sentences) { - var sents = sentences.Select(s => new ParallelToken { Text = s }).ToList(); + var sents = sentences.Select(s => new Sentence { Text = s }).ToList(); Parallel.ForEach(sents, (sentence) => { - sentence.Tokens = Tokenize(sentence.Text); + sentence.Words = Tokenize(sentence.Text); }); - List> result = new List>(); - sents.ForEach(x => result.Add(x.Tokens)); - - return result; + return sents; } private class ParallelToken