diff --git a/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs b/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs new file mode 100644 index 00000000..901c3737 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class BernoulliNaiveBayes + { + } +} diff --git a/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs b/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs new file mode 100644 index 00000000..f02d44b4 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class GaussianNaiveBayes + { + } +} diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs new file mode 100644 index 00000000..f10f268b --- /dev/null +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs @@ -0,0 +1,110 @@ +/* + * BotSharp.Algorithm + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.Algorithm.Estimators; +using BotSharp.Algorithm.Features; +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + /// + /// https://en.wikipedia.org/wiki/Bayes%27_theorem + /// + public class MultinomiaNaiveBayes + { + public List LabelDist { get; set; } + + public List> FeatureSet { get; set; } + + public double Alpha { get; set; } + + /// + /// prior probability + /// + /// + /// + public double CalPriorProb(string Y) + { + int N = FeatureSet.Count; + int k = LabelDist.Count; + int Nyk = LabelDist.First(x => x.Value == Y).Freq; + + return (Nyk + Alpha) / (N + k * Alpha); + } + + /// + /// calculate posterior probability P(Y|X) + /// X is feature set, Y is label + /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) + /// + public double PosteriorProb(string Y, double[] features, double priorProb) + { + Alpha = 0.5; + + int featureCount = features.Length; + + double postProb = priorProb; + + // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); + var matrix = ConstructMatrix(featuresIfY); + + // loop features + for (int x = 0; x < featureCount; x++) + { + int freq = 0; + for (int y = 0; y < featuresIfY.Count; y++) + { + if(matrix[y, x] == features[x]) + { + freq++; + } + } + + int Nyk = featuresIfY.Count; + int n = featureCount; + int Nykx = freq; + + postProb += Math.Log((Nykx + Alpha) / (Nyk + n * Alpha)); + } + + return Math.Pow(2, postProb); + } + + private double[,] ConstructMatrix(List> featuresIfY) + { + var featureCount = featuresIfY[0].Item2.Length; + + double[,] matrix = new double[featuresIfY.Count, featureCount]; + for (int y = 0; y < featuresIfY.Count; y++) + { + for (int x = 0; x < featureCount; x++) + { + matrix[y, x] = featuresIfY[y].Item2[x]; + } + } + + return matrix; + } + } +} diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs deleted file mode 100644 index 099ca6eb..00000000 --- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs +++ /dev/null @@ -1,65 +0,0 @@ -using BotSharp.Algorithm.Estimators; -using BotSharp.Algorithm.Features; -using BotSharp.Algorithm.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace BotSharp.Algorithm.Bayes -{ - /// - /// https://en.wikipedia.org/wiki/Bayes%27_theorem - /// - public class NaiveBayes where Estimator : IEstimator, new() - { - /// - /// smoothing function - /// - private Estimator estomator; - - public List FeaturesDist { get; set; } - - public List LabelDist { get; set; } - - public NaiveBayes() - { - estomator = new Estimator(); - } - - /// - /// calculate posterior probability P(Y|X) - /// X is feature set, Y is label - /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) - /// - /// label - /// - /// - public double PosteriorProb(string Y, List features) - { - double prob = 0; - - // prior probability - prob = Math.Log(estomator.Prob(LabelDist, Y), 2); - - // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList(); - - // loop features - for (int x = 0; x < features.Count; x++) - { - var Xn = features[x]; - var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues; - - if(fv != null) - { - // features are independent, so calculate every feature prob and sum them - prob += Math.Log(estomator.Prob(fv, Xn.Value), 2); - } - } - - return prob; - } - } -} diff --git a/BotSharp.Algorithm/Estimators/Lidstone.cs b/BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs similarity index 75% rename from BotSharp.Algorithm/Estimators/Lidstone.cs rename to BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs index d1153b16..0e934031 100644 --- a/BotSharp.Algorithm/Estimators/Lidstone.cs +++ b/BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs @@ -31,10 +31,12 @@ namespace BotSharp.Algorithm.Estimators /// https://en.wikipedia.org/wiki/Additive_smoothing /// Used as Multinomial Naive Bayes /// - public class Lidstone : IEstimator + public class AdditiveSmoothing : IEstimator { /// - /// α > 0 is the smoothing parameter + /// 1 > α > 0 is the smoothing parameter is Lidstone + /// α = 1 is Laplace + /// α = 0 no smoothing /// public double Alpha { get; set; } @@ -62,5 +64,24 @@ namespace BotSharp.Algorithm.Estimators return (x + Alpha) / (_N + Alpha * _d); } + + public double Prob(List> dist, string sample) + { + if (Alpha == 0) + { + Alpha = 0.5D; + } + + // observation x = (x1, ..., xd) + var p = dist.Find(f => f.Item1 == sample); + double x = p == null ? 0D : p.Item2; + + // N trials + double _N = dist.Sum(f => f.Item2); + + int _d = dist.Count; + + return (x + Alpha) / (_N + Alpha * _d); + } } } diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 5cfefb69..e8f295e7 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -32,9 +32,9 @@ namespace BotSharp.NLP.UnitTest { newSentences[i].Label = sentences[i].Label; } - sentences = newSentences.ToList(); + sentences = newSentences.Take(10).ToList(); - sentences.Shuffle(); + //sentences.Shuffle(); var encoder = new OneHotEncoder(); encoder.Sentences = sentences; @@ -46,9 +46,7 @@ namespace BotSharp.NLP.UnitTest }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(0.9M); - classifier.TrainInVector(dataset.Item1); - + var dataset = sentences.Split(1M); classifier.Train(dataset.Item1); int correct = 0; @@ -58,10 +56,13 @@ namespace BotSharp.NLP.UnitTest if (td.Label == classes[0].Item1) { correct++; + } }); - var accuracy = (float)correct / dataset.Item2.Count; + var accuracy = (float)correct / dataset.Item1.Count; + + Assert.IsTrue(accuracy > 0.8); } [TestMethod] diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index bd32aac8..eebccb07 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -27,31 +27,7 @@ namespace BotSharp.NLP.Classify featureExtractor = new IFeatureExtractor(); } - public List> Classify(Sentence sentence) - { - var options = new ClassifyOptions - { - }; - - var features = featureExtractor.GetFeatures(sentence.Words); - - var classes = _classifier.Classify(features, options); - - return classes.OrderByDescending(x => x.Item2).ToList(); - } - public void Train(List sentences) - { - var sents = sentences.Select(x => new FeaturesWithLabel - { - Label = x.Label, - Features = featureExtractor.GetFeatures(x.Words) - }).ToList(); - - _classifier.Train(sents, _options); - } - - public void TrainInVector(List sentences) { var vectors = new List>(); @@ -59,5 +35,16 @@ namespace BotSharp.NLP.Classify _classifier.Train(sents, _options); } + + public List> Classify(Sentence sentence) + { + var options = new ClassifyOptions + { + }; + + var classes = _classifier.Classify(sentence.Vector, options); + + return classes.OrderByDescending(x => x.Item2).ToList(); + } } } diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index 5b011508..c52a71c2 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -7,10 +7,6 @@ namespace BotSharp.NLP.Classify { public interface IClassifier { - void Train(List featureSets, ClassifyOptions options); - - List> Classify(List features, ClassifyOptions options); - /// /// Training by feature vector /// diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 834b1864..ae141862 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -40,104 +40,14 @@ namespace BotSharp.NLP.Classify /// public class NaiveBayesClassifier : IClassifier { - private List featuresDist; - private List labelDist; - public void Train(List featureSets, ClassifyOptions options) - { - labelDist = featureSets.GroupBy(x => x.Label) - .Select(x => new Probability - { - Value = x.Key, - Freq = x.Count() - }) - .ToList(); + private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes(); - var fNames = new List(); - - featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name))); - fNames = fNames.OrderBy(x => x).Distinct().ToList(); - - var featureValues = new Dictionary>(); - - for (int i = 0; i < featureSets.Count; i++) - { - var fs = featureSets[i]; - featureValues[fs.Label] = new List(); - - fNames.ForEach(fn => - { - Feature feature = null; - for (int j = 0; j < fs.Features.Count; j++) - { - if (fs.Features[j].Name == fn) - { - feature = fs.Features[j]; - break; - } - } - - var fv = new Feature(fn, feature == null ? "False" : feature.Value); - featureValues[fs.Label].Add(fv); - }); - } - - featuresDist = new List(); - - labelDist.Select(x => x.Value).ToList().ForEach(label => - { - var fSets = featureValues[label]; - - fNames.ForEach(fName => - { - var fsv = fSets.Where(fs => fs.Name == fName) - .GroupBy(fs => fs.Value) - .Select(fs => new Probability - { - Value = fs.Key, - Freq = fs.Count() - }) - .OrderBy(fs => fs.Value) - .ToList(); - - featuresDist.Add(new FeaturesDistribution - { - Label = label, - FeatureName = fName, - FeatureValues = fsv - }); - }); - }); - } - - public List> Classify(List features, ClassifyOptions options) - { - // calculate prop - var nb = new NaiveBayes(); - nb.LabelDist = labelDist; - nb.FeaturesDist = featuresDist; - - Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features)); - - // add log - double[] logs = labelDist.Select(x => x.Prob).ToArray(); - - var sumLogs = logs.Reduce((log1, next) => - { - double min = log1; - if (next < log1) - { - min = next; - } - - return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2); - }); - - labelDist.ForEach(d => d.Prob -= sumLogs); - - return labelDist.Select(x => new Tuple(x.Value, x.Prob)).ToList(); - } + /// + /// Cache all categories' prior probability + /// + private Dictionary PriorPropDictionary = new Dictionary(); public void Train(List> featureSets, ClassifyOptions options) { @@ -148,11 +58,35 @@ namespace BotSharp.NLP.Classify Freq = x.Count() }) .ToList(); + + nb.LabelDist = labelDist; + nb.FeatureSet = featureSets; + + // calculate prior prob + labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value)); + + // calculate posterior prob + } public List> Classify(double[] features, ClassifyOptions options) { - throw new NotImplementedException(); + var results = new List>(); + + // calculate prop + labelDist.ForEach(lf => + { + var prob = nb.PosteriorProb(lf.Value, features, lf.Prob); + results.Add(new Tuple(lf.Value, prob)); + }); + + /*Parallel.ForEach(labelDist, (lf) => + { + nb.Y = lf.Value; + lf.Prob = nb.PosteriorProb(); + });*/ + + return results; } } diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 7d630bac..9e4f51d5 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -38,10 +38,8 @@ namespace BotSharp.NLP.Txt2Vec public void EncodeAll() { InitDictionary(); - Parallel.ForEach(Sentences, sent => - { - Encode(sent); - }); + Sentences.ForEach(sent => Encode(sent)); + //Parallel.ForEach(Sentences, sent => Encode(sent)); } private void InitDictionary()