diff --git a/BotSharp.Algorithm/Formulas/Lidstone.cs b/BotSharp.Algorithm/Formulas/Lidstone.cs new file mode 100644 index 00000000..a754bce5 --- /dev/null +++ b/BotSharp.Algorithm/Formulas/Lidstone.cs @@ -0,0 +1,87 @@ +/* + * BotSharp.Algorithm + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Algorithm.Formulas +{ + /// + /// Lidstone smoothing is a technique used to smooth categorical data. + /// In statistics, it's called additive smoothing or Laplace smoothing. + /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator. + /// https://en.wikipedia.org/wiki/Additive_smoothing + /// + public class Lidstone + { + /// + /// α > 0 is the smoothing parameter + /// + private double _a; + + public Lidstone(double alpha = 0.5D) + { + _a = alpha; + } + + /// + /// Probability + /// + /// distribution + /// sample value + /// + public double Prob(List dist, string sample) + { + // observation x = (x1, ..., xd) + int x = dist.Find(f => f.Value == sample).Freq; + + // N trials + int _N = dist.Sum(f => f.Freq); + + int _d = dist.Count; + + return (x + _a) / (_N + _a * _d); + } + + /// + /// 2 based Log probability + /// + /// distribution + /// sample value + /// + public double Log2Prob(List dist, string sample) + { + var d = Prob(dist, sample); + return Math.Log(d, 2); + } + + /// + /// 10 based Log probability + /// + /// distribution + /// sample value + /// + public double Log10Prob(List dist, string sample) + { + var d = Prob(dist, sample); + return Math.Log(d, 10); + } + } +} diff --git a/BotSharp.Algorithm/Probability.cs b/BotSharp.Algorithm/Probability.cs new file mode 100644 index 00000000..e2c58afa --- /dev/null +++ b/BotSharp.Algorithm/Probability.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm +{ + /// + /// In probability theory and statistics, a probability distribution is a mathematical function + /// that provides the probabilities of occurrence of different possible outcomes in an experiment. + /// https://en.wikipedia.org/wiki/Probability_distribution + /// + public class Probability + { + /// + /// one value of all samples + /// + public string Value { get; set; } + + /// + /// the number of times that something happens within a particular period of time + /// + public int Freq { get; set; } + + /// + /// how likely something is, sometimes calculated in a mathematical way + /// + public double Prob { get; set; } + + public override string ToString() + { + return $"{Value} {Freq} {Prob}"; + } + } +} diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index d8070ebc..2ad15bdb 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -32,6 +32,9 @@ namespace BotSharp.NLP.UnitTest corpus.ForEach(x => x.Words = tokenizer.Tokenize(x.Text)); classifier.Train(corpus); + + string text = "Aamir"; + classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) }); } private List GetLabeledCorpus(ClassifyOptions options) diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index c2823524..db8a02f4 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -24,7 +24,12 @@ namespace BotSharp.NLP.Classify public void Classify(Sentence sentence) { - + _classifier.Classify(new LabeledFeatureSet + { + Features = GetFeatures(sentence.Words) + }, new ClassifyOptions + { + }); } public void Train(List sentences) diff --git a/BotSharp.NLP/Classify/Lidstone.cs b/BotSharp.NLP/Classify/Lidstone.cs deleted file mode 100644 index 3602c61d..00000000 --- a/BotSharp.NLP/Classify/Lidstone.cs +++ /dev/null @@ -1,53 +0,0 @@ -/* - * BotSharp.NLP Library - * Copyright (C) 2018 Haiping Chen - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.NLP.Classify -{ - /// - /// Lidstone smoothing, is a technique used to smooth categorical data. - /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator: - /// Refer https://en.wikipedia.org/wiki/Additive_smoothing - /// - public class Lidstone : IEstimator - { - /// - /// x = (x1, …, xd) - /// - private int _d; - - /// - /// α > 0 is the smoothing parameter - /// - private float _a; - - /// - /// N trials - /// - private int _N; - - public Lidstone(float alpha, int bins) - { - _a = alpha; - _d = bins; - } - } -} diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 6f99658d..b037ea88 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -16,6 +16,8 @@ * along with this program. If not, see . */ +using BotSharp.Algorithm; +using BotSharp.Algorithm.Formulas; using System; using System.Collections.Generic; using System.IO; @@ -33,15 +35,18 @@ namespace BotSharp.NLP.Classify /// public class NaiveBayesClassifier : IClassifier { - public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options) - { - throw new NotImplementedException(); - } + private List featureDist; + + private List labelDist; public void Train(List featureSets, ClassifyOptions options) { - var labelFreqDist = featureSets.GroupBy(x => x.Label) - .Select(x => new { Label = x.Key, Count = x.Count() }) + labelDist = featureSets.GroupBy(x => x.Label) + .Select(x => new Probability + { + Value = x.Key, + Freq = x.Count() + }) .ToList(); var fNames = featureSets[0].Features.Select(x => x.Name).ToList(); @@ -56,20 +61,24 @@ namespace BotSharp.NLP.Classify Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList() }).ToList(); - var featureFreqDist = new List(); + featureDist = new List(); - labelFreqDist.Select(x => x.Label).ToList().ForEach(label => + labelDist.Select(x => x.Value).ToList().ForEach(label => { var fSets = featureSets.Where(x => x.Label == label); fNames.ForEach(fName => { var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName)) .GroupBy(f => f.Value) - .Select(f => new Tuple(f.Key, f.Count())) - .OrderBy(f => f.Item1) + .Select(f => new Probability + { + Value = f.Key, + Freq = f.Count() + }) + .OrderBy(f => f.Value) .ToList(); - featureFreqDist.Add(new FeatureFrequencyDistribution + featureDist.Add(new FeatureFrequencyDistribution { Label = label, FeatureName = fName, @@ -77,11 +86,26 @@ namespace BotSharp.NLP.Classify }); }); }); + } - featureFreqDist.ForEach(ffd => + public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options) + { + var estimator = new Lidstone(); + + labelDist.ForEach(lf => { + lf.Prob = estimator.Log2Prob(labelDist, lf.Value); + }); + featureDist.ForEach(fd => + { + fd.FeatureValues.ForEach(fv => + { + fv.Prob = estimator.Log2Prob(fd.FeatureValues, fv.Value); + var p = labelDist.Find(l => l.Value == fd.Label); + p.Prob += fv.Prob; + }); }); } } @@ -128,7 +152,7 @@ namespace BotSharp.NLP.Classify public string FeatureName { get; set; } - public List> FeatureValues { get; set; } + public List FeatureValues { get; set; } public override string ToString() {