diff --git a/BotSharp.Algorithm/Extensions/Reduce.cs b/BotSharp.Algorithm/Extensions/Reduce.cs new file mode 100644 index 00000000..e5bc04d0 --- /dev/null +++ b/BotSharp.Algorithm/Extensions/Reduce.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Algorithm.Extensions +{ + public static partial class IListExtensions + { + /// + /// equivalent reduce function in Python + /// https://docs.python.org/3/library/functools.html?highlight=reduce#functools.reduce + /// + /// + /// + /// + /// + public static TAccumulate Reduce(this IList source, Func func) + { + return source.Skip(1).Aggregate(source[0], func); + } + } +} diff --git a/BotSharp.Algorithm/Extensions/Shuffle.cs b/BotSharp.Algorithm/Extensions/Shuffle.cs new file mode 100644 index 00000000..d90628ed --- /dev/null +++ b/BotSharp.Algorithm/Extensions/Shuffle.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; + +namespace BotSharp.Algorithm.Extensions +{ + public static partial class IListExtensions + { + public static void Shuffle2(this IList list) + { + var provider = new RNGCryptoServiceProvider(); + int count = list.Count; + while (count > 1) + { + var box = new byte[1]; + + do provider.GetBytes(box); + while (!(box[0] < count * (Byte.MaxValue / count))); + + var k = (box[0] % count); + count--; + + var value = list[k]; + list[k] = list[count]; + list[count] = value; + } + } + + public static void Shuffle(this IList list) + { + var rng = new Random(); + var count = list.Count; + while (count > 1) + { + count--; + var k = rng.Next(count + 1); + var value = list[k]; + list[k] = list[count]; + list[count] = value; + } + } + } +} diff --git a/BotSharp.Algorithm/Formulas/Lidstone.cs b/BotSharp.Algorithm/Formulas/Lidstone.cs index a754bce5..8596c894 100644 --- a/BotSharp.Algorithm/Formulas/Lidstone.cs +++ b/BotSharp.Algorithm/Formulas/Lidstone.cs @@ -50,7 +50,8 @@ namespace BotSharp.Algorithm.Formulas public double Prob(List dist, string sample) { // observation x = (x1, ..., xd) - int x = dist.Find(f => f.Value == sample).Freq; + var p = dist.Find(f => f.Value == sample); + int x = p == null ? 0 : p.Freq; // N trials int _N = dist.Sum(f => f.Freq); diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index 1c0a4c15..a7a25a63 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -71,6 +71,7 @@ If you feel that this project is helpful to you, please Star on the project, we + diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 2ad15bdb..b4020594 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -6,7 +6,9 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; +using BotSharp.Algorithm.Extensions; namespace BotSharp.NLP.UnitTest { @@ -31,10 +33,25 @@ namespace BotSharp.NLP.UnitTest corpus.ForEach(x => x.Words = tokenizer.Tokenize(x.Text)); - classifier.Train(corpus); + // classifier.Train(corpus); + // string text = "Bridget"; + // classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) }); + corpus.Shuffle(); + var trainingData = corpus.Skip(2000).ToList(); + classifier.Train(trainingData); - string text = "Aamir"; - classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) }); + var testData = corpus.Take(2000).ToList(); + int correct = 0; + testData.ForEach(td => + { + var classes = classifier.Classify(td); + if(td.Label == classes[0].Item1) + { + correct++; + } + }); + + var accuracy = (float)correct / testData.Count; } private List GetLabeledCorpus(ClassifyOptions options) diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj index 46b9f277..11c2572b 100644 --- a/BotSharp.NLP/BotSharp.NLP.csproj +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -39,6 +39,7 @@ Naive Bayes Classifier + diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index db8a02f4..24df05e2 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -22,14 +22,16 @@ namespace BotSharp.NLP.Classify _classifier = new IClassify(); } - public void Classify(Sentence sentence) + public List> Classify(Sentence sentence) { - _classifier.Classify(new LabeledFeatureSet + var classes = _classifier.Classify(new LabeledFeatureSet { Features = GetFeatures(sentence.Words) }, new ClassifyOptions { }); + + return classes.OrderByDescending(x => x.Item2).ToList(); } public void Train(List sentences) diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index 94db47e7..fc72882e 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -8,6 +8,6 @@ namespace BotSharp.NLP.Classify { void Train(List featureSets, ClassifyOptions options); - void Classify(LabeledFeatureSet featureSet, ClassifyOptions options); + List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options); } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index b037ea88..a565cb99 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -17,6 +17,7 @@ */ using BotSharp.Algorithm; +using BotSharp.Algorithm.Extensions; using BotSharp.Algorithm.Formulas; using System; using System.Collections.Generic; @@ -32,6 +33,8 @@ namespace BotSharp.NLP.Classify /// This technique works well for topic classification; /// say we have a set of academic papers, and we want to classify them into different topics (computer science, biology, mathematics). /// Naive Bayes is best for Less training data + /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) + /// Y is label, X is features. /// public class NaiveBayesClassifier : IClassifier { @@ -49,7 +52,9 @@ namespace BotSharp.NLP.Classify }) .ToList(); - var fNames = featureSets[0].Features.Select(x => x.Name).ToList(); + var fNames = featureSets[0].Features.Select(x => x.Name) + .OrderBy(x => x) + .ToList(); // combine all features. var allFeatureValues = new List(); @@ -88,25 +93,40 @@ namespace BotSharp.NLP.Classify }); } - public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options) + public List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options) { var estimator = new Lidstone(); labelDist.ForEach(lf => { + // prior probability lf.Prob = estimator.Log2Prob(labelDist, lf.Value); - }); - featureDist.ForEach(fd => - { - fd.FeatureValues.ForEach(fv => + // post probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + featureSet.Features.ForEach(f => { - fv.Prob = estimator.Log2Prob(fd.FeatureValues, fv.Value); - - var p = labelDist.Find(l => l.Value == fd.Label); - p.Prob += fv.Prob; + var fv = featureDist.Find(x => x.Label == lf.Value && x.FeatureName == f.Name).FeatureValues; + lf.Prob += estimator.Log2Prob(fv, f.Value); }); }); + + // add log + double[] logs = labelDist.Select(x => x.Prob).ToArray(); + + var sumLogs = logs.Reduce((log1, next) => + { + double min = log1; + if (next < log1) + { + min = next; + } + + return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2); + }); + + labelDist.ForEach(d => d.Prob -= sumLogs); + + return labelDist.Select(x => new Tuple(x.Value, x.Prob)).ToList(); } } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 433b1c83..6ab2c8ff 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -31,9 +31,9 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { - public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options) + public List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options) { - + return null; } public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options) diff --git a/BotSharp.NLP/Corpus/FasttextDataReader.cs b/BotSharp.NLP/Corpus/FasttextDataReader.cs new file mode 100644 index 00000000..87a6e6f1 --- /dev/null +++ b/BotSharp.NLP/Corpus/FasttextDataReader.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.NLP.Corpus +{ + /// + /// Fasttext labeled data reader + /// + public class FasttextDataReader + { + public List Read(ReaderOptions options) + { + var sentences = new List(); + using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName))) + { + while (!reader.EndOfStream) + { + string line = reader.ReadLine(); + if (!String.IsNullOrEmpty(line)) + { + var ms = Regex.Matches(line, @"__label__\w+\s").Cast().ToList(); + + sentences.Add(new Sentence + { + // Label = lable, + Text = line + }); + } + } + } + + return sentences; + } + } +}