From aec1984ab838f58cf22a1a352a9d099999b82157 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Tue, 11 Sep 2018 17:29:36 -0500 Subject: [PATCH] define bin model --- .../Bayes/MultinomiaNaiveBayes.cs | 55 +++++++++++-------- .../Bayes/MultinomiaNaiveBayesModel.cs | 14 +++++ .../Engines/BotSharp/BotSharpSVMClassifier.cs | 2 +- .../NaiveBayesClassifierTest.cs | 4 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 2 +- BotSharp.NLP/Classify/IClassifier.cs | 2 +- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 29 ++++++++-- BotSharp.NLP/Classify/SVMClassifier.cs | 26 +++------ 8 files changed, 81 insertions(+), 53 deletions(-) create mode 100644 BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs index f10f268b..977f207f 100644 --- a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs @@ -35,7 +35,12 @@ namespace BotSharp.Algorithm.Bayes public List> FeatureSet { get; set; } - public double Alpha { get; set; } + private double alpha { get; set; } + + public MultinomiaNaiveBayes(double alpha = 0.5) + { + this.alpha = alpha; + } /// /// prior probability @@ -48,7 +53,29 @@ namespace BotSharp.Algorithm.Bayes int k = LabelDist.Count; int Nyk = LabelDist.First(x => x.Value == Y).Freq; - return (Nyk + Alpha) / (N + k * Alpha); + return (Nyk + alpha) / (N + k * alpha); + } + + public double CalCondProb(int x, string Y, double feature) + { + // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); + var matrix = ConstructMatrix(featuresIfY); + + int freq = 0; + for (int y = 0; y < featuresIfY.Count; y++) + { + if (matrix[y, x] == feature) + { + freq++; + } + } + + int Nyk = featuresIfY.Count; + int n = featuresIfY.Count; + int Nykx = freq; + + return Math.Log((Nykx + alpha) / (Nyk + n * alpha)); } /// @@ -57,35 +84,17 @@ namespace BotSharp.Algorithm.Bayes /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) /// - public double PosteriorProb(string Y, double[] features, double priorProb) + public double CalPosteriorProb(string Y, double[] features, double priorProb, Dictionary condProbDictionary) { - Alpha = 0.5; - int featureCount = features.Length; double postProb = priorProb; - // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); - var matrix = ConstructMatrix(featuresIfY); - // loop features for (int x = 0; x < featureCount; x++) { - int freq = 0; - for (int y = 0; y < featuresIfY.Count; y++) - { - if(matrix[y, x] == features[x]) - { - freq++; - } - } - - int Nyk = featuresIfY.Count; - int n = featureCount; - int Nykx = freq; - - postProb += Math.Log((Nykx + Alpha) / (Nyk + n * Alpha)); + string key = $"{Y} f{x} {features[x]}"; + postProb += condProbDictionary[key]; } return Math.Pow(2, postProb); diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs new file mode 100644 index 00000000..ecccfb43 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs @@ -0,0 +1,14 @@ +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class MultinomiaNaiveBayesModel + { + public List LabelDist { get; set; } + + public Dictionary CondProbDictionary { get; set; } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs index 9def3cc4..c9b1d0a6 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -129,7 +129,7 @@ namespace BotSharp.Core.Engines.BotSharp ClassifyOptions classifyOptions = new ClassifyOptions(); classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data"); - svmClassifier.Train(featureSetList, classifyOptions); + // svmClassifier.Train(featureSetList, classifyOptions); meta.Meta = new JObject(); meta.Meta["compiled at"] = "Aug 31, 2018"; diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index e8f295e7..cffd7c00 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -32,9 +32,9 @@ namespace BotSharp.NLP.UnitTest { newSentences[i].Label = sentences[i].Label; } - sentences = newSentences.Take(10).ToList(); + sentences = newSentences.ToList(); - //sentences.Shuffle(); + // sentences.Shuffle(); var encoder = new OneHotEncoder(); encoder.Sentences = sentences; diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index eebccb07..824c9eed 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -33,7 +33,7 @@ namespace BotSharp.NLP.Classify var sents = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - _classifier.Train(sents, _options); + _classifier.Train(sents, new double[] { 0, 1 }, _options); } public List> Classify(Sentence sentence) diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index c52a71c2..d84c0ab0 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -12,7 +12,7 @@ namespace BotSharp.NLP.Classify /// /// /// - void Train(List> featureSets, ClassifyOptions options); + void Train(List> featureSets, double[] values, ClassifyOptions options); /// /// Predict by feature vector diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index ae141862..1d81aac8 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -44,12 +44,9 @@ namespace BotSharp.NLP.Classify private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes(); - /// - /// Cache all categories' prior probability - /// - private Dictionary PriorPropDictionary = new Dictionary(); + private Dictionary condProbDictionary = new Dictionary(); - public void Train(List> featureSets, ClassifyOptions options) + public void Train(List> featureSets, double[] values, ClassifyOptions options) { labelDist = featureSets.GroupBy(x => x.Item1) .Select(x => new Probability @@ -66,7 +63,27 @@ namespace BotSharp.NLP.Classify labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value)); // calculate posterior prob + // loop features + var featureCount = nb.FeatureSet[0].Item2.Length; + labelDist.ForEach(label => + { + for (int x = 0; x < featureCount; x++) + { + for (int v = 0; v < values.Length; v++) + { + string key = $"{label.Value} f{x} {values[v]}"; + condProbDictionary[key] = nb.CalCondProb(x, label.Value, values[v]); + } + } + }); + + // save the model + var model = new MultinomiaNaiveBayesModel + { + LabelDist = labelDist, + CondProbDictionary = condProbDictionary + }; } public List> Classify(double[] features, ClassifyOptions options) @@ -76,7 +93,7 @@ namespace BotSharp.NLP.Classify // calculate prop labelDist.ForEach(lf => { - var prob = nb.PosteriorProb(lf.Value, features, lf.Prob); + var prob = nb.CalPosteriorProb(lf.Value, features, lf.Prob, condProbDictionary); results.Add(new Tuple(lf.Value, prob)); }); diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index be97f77e..cf56e333 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -32,11 +32,6 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { - public List> Classify(List features, ClassifyOptions options) - { - return null; - } - public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options) { Problem predict = new Problem(); @@ -53,9 +48,14 @@ namespace BotSharp.NLP.Classify return Prediction.PredictLabelsProbability(options.Model, scaled); } - public void Train(List featureSets, ClassifyOptions options) + public void Train(List> featureSets, double[] values, ClassifyOptions options) { - SVMClassifierTrain(featureSets, options); + // SVMClassifierTrain(featureSets, options); + } + + public List> Classify(double[] features, ClassifyOptions options) + { + throw new NotImplementedException(); } public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) @@ -154,17 +154,5 @@ namespace BotSharp.NLP.Classify return labeledFeatureSet; } - - public void Train(List> featureSets, ClassifyOptions options) - { - throw new NotImplementedException(); - } - - public List> Classify(double[] features, ClassifyOptions options) - { - throw new NotImplementedException(); - } } - - }