From 553d5c231e039ef33a75ba22c493619016e03259 Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Sun, 9 Sep 2018 10:32:10 -0500 Subject: [PATCH] Encapsulate NaiveBayes' theorem and add optional smoother function. --- BotSharp.Algorithm/Bayes/NaiveBayes.cs | 48 +++++++++++-------- BotSharp.Algorithm/Formulas/Lidstone.cs | 40 ++++------------ BotSharp.Algorithm/ISmoother.cs | 11 +++++ .../Engines/BotSharp/BotSharpSVMClassifier.cs | 2 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 7 +-- BotSharp.NLP/Classify/IClassifier.cs | 4 +- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 23 ++++----- BotSharp.NLP/Classify/SVMClassifier.cs | 28 +++++------ 8 files changed, 75 insertions(+), 88 deletions(-) create mode 100644 BotSharp.Algorithm/ISmoother.cs diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs index 764b14f9..e2217130 100644 --- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs +++ b/BotSharp.Algorithm/Bayes/NaiveBayes.cs @@ -10,20 +10,20 @@ namespace BotSharp.Algorithm.Bayes /// /// https://en.wikipedia.org/wiki/Bayes%27_theorem /// - public class NaiveBayes + public class NaiveBayes where Smoother : ISmoother, new() { /// /// smoothing function /// - private Lidstone smoother; + private Smoother smoother; - public List FeatureDist { get; set; } + public List FeaturesDist { get; set; } public List LabelDist { get; set; } public NaiveBayes() { - smoother = new Lidstone(); + smoother = new Smoother(); } /// @@ -35,19 +35,25 @@ namespace BotSharp.Algorithm.Bayes /// label /// /// - public double PosteriorProb(string Y, LabeledFeatureSet featureSet) + public double PosteriorProb(string Y, List features) { double prob = 0; // prior probability - prob = smoother.Log2Prob(LabelDist, Y); + prob = Math.Log(smoother.Prob(LabelDist, Y), 2); // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - featureSet.Features.ForEach(f => + var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList(); + + // loop features + for (int x = 0; x < features.Count; x++) { - var fv = FeatureDist.Find(x => x.Label == Y && x.FeatureName == f.Name).FeatureValues; - prob += smoother.Log2Prob(fv, f.Value); - }); + var Xn = features[x]; + var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues; + + // features are independent, so calculate every feature prob and sum them + prob += Math.Log(smoother.Prob(fv, Xn.Value), 2); + } return prob; } @@ -65,7 +71,17 @@ namespace BotSharp.Algorithm.Bayes } } - public class FeatureFrequencyDistribution + public class FeaturesWithLabel + { + public List Features { get; set; } + public string Label { get; set; } + public FeaturesWithLabel() + { + this.Features = new List(); + } + } + + public class FeaturesDistribution { public string Label { get; set; } @@ -78,14 +94,4 @@ namespace BotSharp.Algorithm.Bayes return $"{Label} {FeatureName} {FeatureValues.Count}"; } } - - public class LabeledFeatureSet - { - public List Features { get; set; } - public string Label { get; set; } - public LabeledFeatureSet() - { - this.Features = new List(); - } - } } diff --git a/BotSharp.Algorithm/Formulas/Lidstone.cs b/BotSharp.Algorithm/Formulas/Lidstone.cs index 8596c894..78bcc3a3 100644 --- a/BotSharp.Algorithm/Formulas/Lidstone.cs +++ b/BotSharp.Algorithm/Formulas/Lidstone.cs @@ -29,17 +29,12 @@ namespace BotSharp.Algorithm.Formulas /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator. /// https://en.wikipedia.org/wiki/Additive_smoothing /// - public class Lidstone + public class Lidstone : ISmoother { /// /// α > 0 is the smoothing parameter /// - private double _a; - - public Lidstone(double alpha = 0.5D) - { - _a = alpha; - } + public double Alpha { get; set; } /// /// Probability @@ -49,6 +44,11 @@ namespace BotSharp.Algorithm.Formulas /// public double Prob(List dist, string sample) { + if(Alpha == 0) + { + Alpha = 0.5D; + } + // observation x = (x1, ..., xd) var p = dist.Find(f => f.Value == sample); int x = p == null ? 0 : p.Freq; @@ -58,31 +58,7 @@ namespace BotSharp.Algorithm.Formulas int _d = dist.Count; - return (x + _a) / (_N + _a * _d); - } - - /// - /// 2 based Log probability - /// - /// distribution - /// sample value - /// - public double Log2Prob(List dist, string sample) - { - var d = Prob(dist, sample); - return Math.Log(d, 2); - } - - /// - /// 10 based Log probability - /// - /// distribution - /// sample value - /// - public double Log10Prob(List dist, string sample) - { - var d = Prob(dist, sample); - return Math.Log(d, 10); + return (x + Alpha) / (_N + Alpha * _d); } } } diff --git a/BotSharp.Algorithm/ISmoother.cs b/BotSharp.Algorithm/ISmoother.cs new file mode 100644 index 00000000..322dd6ef --- /dev/null +++ b/BotSharp.Algorithm/ISmoother.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm +{ + public interface ISmoother + { + double Prob(List dist, string sample); + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs index b7dcc0ac..9def3cc4 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -100,7 +100,7 @@ namespace BotSharp.Core.Engines.BotSharp NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier(); Args args = new Args(); args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); - List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels); + var featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels); /* // try using spacy doc2vec diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index 4cf2247a..a90d2109 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -25,10 +25,7 @@ namespace BotSharp.NLP.Classify public List> Classify(Sentence sentence) { - var classes = _classifier.Classify(new LabeledFeatureSet - { - Features = GetFeatures(sentence.Words) - }, new ClassifyOptions + var classes = _classifier.Classify(GetFeatures(sentence.Words), new ClassifyOptions { }); @@ -37,7 +34,7 @@ namespace BotSharp.NLP.Classify public void Train(List sentences) { - _classifier.Train(sentences.Select(x => new LabeledFeatureSet + _classifier.Train(sentences.Select(x => new FeaturesWithLabel { Label = x.Label, Features = GetFeatures(x.Words) diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index 833a2732..e52b995c 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -7,8 +7,8 @@ namespace BotSharp.NLP.Classify { public interface IClassifier { - void Train(List featureSets, ClassifyOptions options); + void Train(List featureSets, ClassifyOptions options); - List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options); + List> Classify(List features, ClassifyOptions options); } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index afbb51d1..8ef10aaa 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -37,11 +37,11 @@ namespace BotSharp.NLP.Classify /// public class NaiveBayesClassifier : IClassifier { - private List featureDist; + private List featuresDist; private List labelDist; - public void Train(List featureSets, ClassifyOptions options) + public void Train(List featureSets, ClassifyOptions options) { labelDist = featureSets.GroupBy(x => x.Label) .Select(x => new Probability @@ -65,7 +65,7 @@ namespace BotSharp.NLP.Classify Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList() }).ToList(); - featureDist = new List(); + featuresDist = new List(); labelDist.Select(x => x.Value).ToList().ForEach(label => { @@ -82,7 +82,7 @@ namespace BotSharp.NLP.Classify .OrderBy(f => f.Value) .ToList(); - featureDist.Add(new FeatureFrequencyDistribution + featuresDist.Add(new FeaturesDistribution { Label = label, FeatureName = fName, @@ -92,17 +92,14 @@ namespace BotSharp.NLP.Classify }); } - public List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options) + public List> Classify(List features, ClassifyOptions options) { - var nb = new NaiveBayes(); + // calculate prop + var nb = new NaiveBayes(); nb.LabelDist = labelDist; - nb.FeatureDist = featureDist; - - labelDist.ForEach(lf => - { - // prior probability - lf.Prob = nb.PosteriorProb(lf.Value, featureSet); - }); + nb.FeaturesDist = featuresDist; + + labelDist.ForEach(lf => lf.Prob = nb.PosteriorProb(lf.Value, features)); // add log double[] logs = labelDist.Select(x => x.Prob).ToArray(); diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 26bd1dd4..4bbc4711 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -32,15 +32,15 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { - public List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options) + public List> Classify(List features, ClassifyOptions options) { return null; } - public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options) + public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options) { Problem predict = new Problem(); - List featureSets = new List(); + List featureSets = new List(); featureSets.Add(featureSet); predict.X = GetData(featureSets).ToArray(); predict.Y = new double[1]; @@ -53,12 +53,12 @@ namespace BotSharp.NLP.Classify return Prediction.PredictLabelsProbability(options.Model, scaled); } - public void Train(List featureSets, ClassifyOptions options) + public void Train(List featureSets, ClassifyOptions options) { SVMClassifierTrain(featureSets, options); } - public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) + public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) { // copy test multiclass Model Problem train = new Problem(); @@ -91,10 +91,10 @@ namespace BotSharp.NLP.Classify Console.Write("Training finished!"); } - public List GetLabels(List featureSets) + public List GetLabels(List featureSets) { List labels = new List(); - foreach (LabeledFeatureSet labelFeatureSet in featureSets) + foreach (var labelFeatureSet in featureSets) { labels.Add(double.Parse(labelFeatureSet.Label)); } @@ -102,11 +102,11 @@ namespace BotSharp.NLP.Classify return labels; } - public List GetData(List featureSets) + public List GetData(List featureSets) { List datas = new List(); - foreach (LabeledFeatureSet labelFeatureSet in featureSets) + foreach (var labelFeatureSet in featureSets) { List curNodes = new List(); labelFeatureSet.Features.ForEach(features => { @@ -119,15 +119,15 @@ namespace BotSharp.NLP.Classify return datas; } - public List FeatureSetsGenerator(List sentenceVectors, List labels) + public List FeatureSetsGenerator(List sentenceVectors, List labels) { - List res = new List(); + var res = new List(); int j; for (int i = 0; i < labels.Count; i++) { string curLabel = labels[i]; Vec curVec = sentenceVectors[i]; - LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet(); + var labeledFeatureSet = new FeaturesWithLabel(); j = 1; foreach (double node in curVec.VecNodes) { @@ -141,9 +141,9 @@ namespace BotSharp.NLP.Classify return res; } - public LabeledFeatureSet FeatureSetsGenerator(Vec sentenceVectors, String label) + public FeaturesWithLabel FeatureSetsGenerator(Vec sentenceVectors, String label) { - LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet(); + var labeledFeatureSet = new FeaturesWithLabel(); int j = 1; foreach (double node in sentenceVectors.VecNodes) {