diff --git a/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs b/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs new file mode 100644 index 00000000..901c3737 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class BernoulliNaiveBayes + { + } +} diff --git a/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs b/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs new file mode 100644 index 00000000..f02d44b4 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class GaussianNaiveBayes + { + } +} diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs new file mode 100644 index 00000000..977f207f --- /dev/null +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs @@ -0,0 +1,119 @@ +/* + * BotSharp.Algorithm + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.Algorithm.Estimators; +using BotSharp.Algorithm.Features; +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + /// + /// https://en.wikipedia.org/wiki/Bayes%27_theorem + /// + public class MultinomiaNaiveBayes + { + public List LabelDist { get; set; } + + public List> FeatureSet { get; set; } + + private double alpha { get; set; } + + public MultinomiaNaiveBayes(double alpha = 0.5) + { + this.alpha = alpha; + } + + /// + /// prior probability + /// + /// + /// + public double CalPriorProb(string Y) + { + int N = FeatureSet.Count; + int k = LabelDist.Count; + int Nyk = LabelDist.First(x => x.Value == Y).Freq; + + return (Nyk + alpha) / (N + k * alpha); + } + + public double CalCondProb(int x, string Y, double feature) + { + // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); + var matrix = ConstructMatrix(featuresIfY); + + int freq = 0; + for (int y = 0; y < featuresIfY.Count; y++) + { + if (matrix[y, x] == feature) + { + freq++; + } + } + + int Nyk = featuresIfY.Count; + int n = featuresIfY.Count; + int Nykx = freq; + + return Math.Log((Nykx + alpha) / (Nyk + n * alpha)); + } + + /// + /// calculate posterior probability P(Y|X) + /// X is feature set, Y is label + /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) + /// + public double CalPosteriorProb(string Y, double[] features, double priorProb, Dictionary condProbDictionary) + { + int featureCount = features.Length; + + double postProb = priorProb; + + // loop features + for (int x = 0; x < featureCount; x++) + { + string key = $"{Y} f{x} {features[x]}"; + postProb += condProbDictionary[key]; + } + + return Math.Pow(2, postProb); + } + + private double[,] ConstructMatrix(List> featuresIfY) + { + var featureCount = featuresIfY[0].Item2.Length; + + double[,] matrix = new double[featuresIfY.Count, featureCount]; + for (int y = 0; y < featuresIfY.Count; y++) + { + for (int x = 0; x < featureCount; x++) + { + matrix[y, x] = featuresIfY[y].Item2[x]; + } + } + + return matrix; + } + } +} diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs new file mode 100644 index 00000000..17e47538 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs @@ -0,0 +1,16 @@ +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class MultinomiaNaiveBayesModel + { + public List LabelDist { get; set; } + + public Dictionary CondProbDictionary { get; set; } + + public List Values { get; set; } + } +} diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs deleted file mode 100644 index 099ca6eb..00000000 --- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs +++ /dev/null @@ -1,65 +0,0 @@ -using BotSharp.Algorithm.Estimators; -using BotSharp.Algorithm.Features; -using BotSharp.Algorithm.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace BotSharp.Algorithm.Bayes -{ - /// - /// https://en.wikipedia.org/wiki/Bayes%27_theorem - /// - public class NaiveBayes where Estimator : IEstimator, new() - { - /// - /// smoothing function - /// - private Estimator estomator; - - public List FeaturesDist { get; set; } - - public List LabelDist { get; set; } - - public NaiveBayes() - { - estomator = new Estimator(); - } - - /// - /// calculate posterior probability P(Y|X) - /// X is feature set, Y is label - /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) - /// - /// label - /// - /// - public double PosteriorProb(string Y, List features) - { - double prob = 0; - - // prior probability - prob = Math.Log(estomator.Prob(LabelDist, Y), 2); - - // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList(); - - // loop features - for (int x = 0; x < features.Count; x++) - { - var Xn = features[x]; - var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues; - - if(fv != null) - { - // features are independent, so calculate every feature prob and sum them - prob += Math.Log(estomator.Prob(fv, Xn.Value), 2); - } - } - - return prob; - } - } -} diff --git a/BotSharp.Algorithm/Estimators/Lidstone.cs b/BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs similarity index 75% rename from BotSharp.Algorithm/Estimators/Lidstone.cs rename to BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs index d1153b16..0e934031 100644 --- a/BotSharp.Algorithm/Estimators/Lidstone.cs +++ b/BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs @@ -31,10 +31,12 @@ namespace BotSharp.Algorithm.Estimators /// https://en.wikipedia.org/wiki/Additive_smoothing /// Used as Multinomial Naive Bayes /// - public class Lidstone : IEstimator + public class AdditiveSmoothing : IEstimator { /// - /// α > 0 is the smoothing parameter + /// 1 > α > 0 is the smoothing parameter is Lidstone + /// α = 1 is Laplace + /// α = 0 no smoothing /// public double Alpha { get; set; } @@ -62,5 +64,24 @@ namespace BotSharp.Algorithm.Estimators return (x + Alpha) / (_N + Alpha * _d); } + + public double Prob(List> dist, string sample) + { + if (Alpha == 0) + { + Alpha = 0.5D; + } + + // observation x = (x1, ..., xd) + var p = dist.Find(f => f.Item1 == sample); + double x = p == null ? 0D : p.Item2; + + // N trials + double _N = dist.Sum(f => f.Item2); + + int _d = dist.Count; + + return (x + Alpha) / (_N + Alpha * _d); + } } } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs new file mode 100644 index 00000000..bc1380e5 --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -0,0 +1,76 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP; +using BotSharp.NLP.Classify; +using BotSharp.NLP.Txt2Vec; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpNBayesClassifier : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + meta.Model = "classification-nb.model"; + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + + var options = new ClassifyOptions + { + ModelFilePath = modelFileName + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var sentences = doc.Sentences.Select(x => new Sentence + { + Label = x.Intent.Label, + Text = x.Text, + Words = x.Tokens + }).ToList(); + + classifier.Train(sentences); + + Console.WriteLine($"Saved model to {modelFileName}"); + meta.Meta = new JObject(); + meta.Meta["compiled at"] = "Sep 12, 2018"; + + return true; + } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + var options = new ClassifyOptions + { + ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model) + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var sentence = doc.Sentences.Select(s => new Sentence + { + Text = s.Text, + Words = s.Tokens + }).First(); + + + var result = classifier.Classify(sentence); + + doc.Sentences[0].Intent = new TextClassificationResult + { + Classifier = "BotSharpNBayesClassifier", + Label = result.First().Item1, + Confidence = (decimal)result.First().Item2 + }; + + return true; + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs index 9def3cc4..c9b1d0a6 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -129,7 +129,7 @@ namespace BotSharp.Core.Engines.BotSharp ClassifyOptions classifyOptions = new ClassifyOptions(); classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data"); - svmClassifier.Train(featureSetList, classifyOptions); + // svmClassifier.Train(featureSetList, classifyOptions); meta.Meta = new JObject(); meta.Meta["compiled at"] = "Aug 31, 2018"; diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs index 0bba34a9..40d2476a 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs @@ -1,5 +1,6 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; +using BotSharp.Core.Intents; using BotSharp.NLP; using BotSharp.NLP.Tokenize; using Microsoft.Extensions.Configuration; @@ -14,14 +15,12 @@ namespace BotSharp.Core.Engines.BotSharp { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } - private TokenizerFactory _tokenizer; + private TokenizerFactory _tokenizer; public BotSharpTokenizer() { - _tokenizer = new TokenizerFactory(new TokenizationOptions + _tokenizer = new TokenizerFactory(new TokenizationOptions { - Pattern = RegexTokenizer.WORD_PUNC, - SpecialWords = new List { "'s" } }, SupportedLanguage.English); } @@ -48,7 +47,8 @@ namespace BotSharp.Core.Engines.BotSharp doc.Sentences.Add(new NlpDocSentence { Tokens = _tokenizer.Tokenize(say.Text), - Text = say.Text + Text = say.Text, + Intent = new TextClassificationResult { Label = say.Intent } }); }); diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 5cfefb69..2b8c2e0b 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -36,32 +36,31 @@ namespace BotSharp.NLP.UnitTest sentences.Shuffle(); - var encoder = new OneHotEncoder(); - encoder.Sentences = sentences; - encoder.EncodeAll(); - var options = new ClassifyOptions { + ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"), TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(0.9M); - classifier.TrainInVector(dataset.Item1); - + var dataset = sentences.Split(1M); classifier.Train(dataset.Item1); int correct = 0; - dataset.Item1.ToList().ForEach(td => + int total = 0; + dataset.Item1.ForEach(td => { var classes = classifier.Classify(td); if (td.Label == classes[0].Item1) { correct++; } + total++; }); - var accuracy = (float)correct / dataset.Item2.Count; + var accuracy = (float)correct / total; + + Assert.IsTrue(accuracy > 0.5); } [TestMethod] diff --git a/BotSharp.NLP.UnitTest/SVMClassifierTest.cs b/BotSharp.NLP.UnitTest/SVMClassifierTest.cs index f9b1d48a..732af1f5 100644 --- a/BotSharp.NLP.UnitTest/SVMClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/SVMClassifierTest.cs @@ -1,6 +1,5 @@ using BotSharp.NLP.Classify; using BotSharp.NLP.Corpus; -using BotSharp.NLP.Models.TF_IDF; using BotSharp.NLP.Tokenize; using Microsoft.Extensions.Configuration; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -29,8 +28,8 @@ namespace BotSharp.NLP.UnitTest "see you Bolo", "byebye Haiping" }; - TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); - List> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents); + /*TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); + List> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents);*/ } [TestMethod] diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index bd32aac8..5f91b735 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -27,37 +27,24 @@ namespace BotSharp.NLP.Classify featureExtractor = new IFeatureExtractor(); } + public void Train(List sentences) + { + _classifier.Train(sentences, _options); + _classifier.SaveModel(_options); + } + public List> Classify(Sentence sentence) { var options = new ClassifyOptions { + ModelFilePath = _options.ModelFilePath }; - var features = featureExtractor.GetFeatures(sentence.Words); + _classifier.LoadModel(options); - var classes = _classifier.Classify(features, options); + var classes = _classifier.Classify(sentence, options); return classes.OrderByDescending(x => x.Item2).ToList(); } - - public void Train(List sentences) - { - var sents = sentences.Select(x => new FeaturesWithLabel - { - Label = x.Label, - Features = featureExtractor.GetFeatures(x.Words) - }).ToList(); - - _classifier.Train(sents, _options); - } - - public void TrainInVector(List sentences) - { - var vectors = new List>(); - - var sents = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - - _classifier.Train(sents, _options); - } } } diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index 5b011508..f0aa48b7 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -7,23 +7,23 @@ namespace BotSharp.NLP.Classify { public interface IClassifier { - void Train(List featureSets, ClassifyOptions options); - - List> Classify(List features, ClassifyOptions options); - /// /// Training by feature vector /// - /// + /// /// - void Train(List> featureSets, ClassifyOptions options); + void Train(List sentences, ClassifyOptions options); /// /// Predict by feature vector /// - /// + /// /// /// - List> Classify(double[] features, ClassifyOptions options); + List> Classify(Sentence sentence, ClassifyOptions options); + + String SaveModel(ClassifyOptions options); + + Object LoadModel(ClassifyOptions options); } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 834b1864..38f28d7b 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -22,6 +22,9 @@ using BotSharp.Algorithm.Estimators; using BotSharp.Algorithm.Extensions; using BotSharp.Algorithm.Features; using BotSharp.Algorithm.Statistics; +using BotSharp.NLP.Featuring; +using BotSharp.NLP.Txt2Vec; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; @@ -40,119 +43,123 @@ namespace BotSharp.NLP.Classify /// public class NaiveBayesClassifier : IClassifier { - private List featuresDist; - private List labelDist; - public void Train(List featureSets, ClassifyOptions options) + private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes(); + + private Dictionary condProbDictionary = new Dictionary(); + + private List words; + private double[] features = new double[] { 0, 1 }; + + public void Train(List sentences, ClassifyOptions options) { - labelDist = featureSets.GroupBy(x => x.Label) - .Select(x => new Probability - { - Value = x.Key, - Freq = x.Count() - }) - .ToList(); + var tfidf = new TfIdfFeatureExtractor(); + tfidf.Sentences = sentences; + tfidf.CalBasedOnCategory(); + var keyWords = tfidf.Features(); + string keywords2 = String.Join(",", keyWords.ToArray()); + var encoder = new OneHotEncoder(); + encoder.Sentences = sentences; + words = encoder.EncodeAll(); - var fNames = new List(); + var featureSets = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name))); - fNames = fNames.OrderBy(x => x).Distinct().ToList(); - - var featureValues = new Dictionary>(); - - for (int i = 0; i < featureSets.Count; i++) - { - var fs = featureSets[i]; - featureValues[fs.Label] = new List(); - - fNames.ForEach(fn => - { - Feature feature = null; - for (int j = 0; j < fs.Features.Count; j++) - { - if (fs.Features[j].Name == fn) - { - feature = fs.Features[j]; - break; - } - } - - var fv = new Feature(fn, feature == null ? "False" : feature.Value); - featureValues[fs.Label].Add(fv); - }); - } - - featuresDist = new List(); - - labelDist.Select(x => x.Value).ToList().ForEach(label => - { - var fSets = featureValues[label]; - - fNames.ForEach(fName => - { - var fsv = fSets.Where(fs => fs.Name == fName) - .GroupBy(fs => fs.Value) - .Select(fs => new Probability - { - Value = fs.Key, - Freq = fs.Count() - }) - .OrderBy(fs => fs.Value) - .ToList(); - - featuresDist.Add(new FeaturesDistribution - { - Label = label, - FeatureName = fName, - FeatureValues = fsv - }); - }); - }); - } - - public List> Classify(List features, ClassifyOptions options) - { - // calculate prop - var nb = new NaiveBayes(); - nb.LabelDist = labelDist; - nb.FeaturesDist = featuresDist; - - Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features)); - - // add log - double[] logs = labelDist.Select(x => x.Prob).ToArray(); - - var sumLogs = logs.Reduce((log1, next) => - { - double min = log1; - if (next < log1) - { - min = next; - } - - return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2); - }); - - labelDist.ForEach(d => d.Prob -= sumLogs); - - return labelDist.Select(x => new Tuple(x.Value, x.Prob)).ToList(); - } - - public void Train(List> featureSets, ClassifyOptions options) - { labelDist = featureSets.GroupBy(x => x.Item1) .Select(x => new Probability { Value = x.Key, Freq = x.Count() }) + .OrderBy(x => x.Value) .ToList(); + + nb.LabelDist = labelDist; + nb.FeatureSet = featureSets; + + // calculate prior prob + labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value)); + + // calculate posterior prob + // loop features + var featureCount = nb.FeatureSet[0].Item2.Length; + + labelDist.ForEach(label => + { + for (int x = 0; x < featureCount; x++) + { + for (int v = 0; v < features.Length; v++) + { + string key = $"{label.Value} f{x} {features[v]}"; + condProbDictionary[key] = nb.CalCondProb(x, label.Value, features[v]); + } + } + }); } - public List> Classify(double[] features, ClassifyOptions options) + public List> Classify(Sentence sentence, ClassifyOptions options) { - throw new NotImplementedException(); + var encoder = new OneHotEncoder(); + encoder.Words = words; + encoder.Encode(sentence); + + var results = new List>(); + + // calculate prop + labelDist.ForEach(lf => + { + var prob = nb.CalPosteriorProb(lf.Value, sentence.Vector, lf.Prob, condProbDictionary); + results.Add(new Tuple(lf.Value, prob)); + }); + + /*Parallel.ForEach(labelDist, (lf) => + { + nb.Y = lf.Value; + lf.Prob = nb.PosteriorProb(); + });*/ + + return results; + } + + public string SaveModel(ClassifyOptions options) + { + // save the model + var model = new MultinomiaNaiveBayesModel + { + LabelDist = labelDist, + CondProbDictionary = condProbDictionary, + Values = words + }; + + //save the file + using (var bw = new BinaryWriter(new FileStream(options.ModelFilePath, FileMode.Create))) + { + var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model)); + bw.Write(bytes); + } + + return options.ModelFilePath; + } + + public Object LoadModel(ClassifyOptions options) + { + string json = String.Empty; + + //read the file + using (var br = new BinaryReader(new FileStream(options.ModelFilePath, FileMode.Open))) + { + byte[] bytes = br.ReadBytes((int)br.BaseStream.Length); + + json = Encoding.UTF8.GetString(bytes); + } + + var model = JsonConvert.DeserializeObject(json); + + labelDist = model.LabelDist; + condProbDictionary = model.CondProbDictionary; + words = model.Values; + + return model; } } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index be97f77e..01ae8cec 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -32,11 +32,6 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { - public List> Classify(List features, ClassifyOptions options) - { - return null; - } - public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options) { Problem predict = new Problem(); @@ -53,9 +48,14 @@ namespace BotSharp.NLP.Classify return Prediction.PredictLabelsProbability(options.Model, scaled); } - public void Train(List featureSets, ClassifyOptions options) + public void Train(List sentences, ClassifyOptions options) { - SVMClassifierTrain(featureSets, options); + // SVMClassifierTrain(featureSets, options); + } + + public List> Classify(Sentence sentence, ClassifyOptions options) + { + throw new NotImplementedException(); } public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) @@ -155,16 +155,14 @@ namespace BotSharp.NLP.Classify return labeledFeatureSet; } - public void Train(List> featureSets, ClassifyOptions options) + public string SaveModel(ClassifyOptions options) { throw new NotImplementedException(); } - public List> Classify(double[] features, ClassifyOptions options) + object IClassifier.LoadModel(ClassifyOptions options) { throw new NotImplementedException(); } } - - } diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs new file mode 100644 index 00000000..785f1d30 --- /dev/null +++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Featuring +{ + public interface IFeatureExtractor + { + } +} diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs new file mode 100644 index 00000000..8f3ccf77 --- /dev/null +++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs @@ -0,0 +1,155 @@ +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Serialization.Formatters.Binary; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.NLP.Featuring +{ + public class TfIdfFeatureExtractor : IFeatureExtractor + { + public List Sentences { get; set; } + + private List> tfs; + + private List Categories { get; set; } + + public void Extract(Sentence sentence) + { + + } + + public List Features() + { + var tfs2 = tfs.OrderByDescending(x => x.Item2) + .Select(x => x.Item1) + .Distinct() + .Take(Sentences.Count / Categories.Count) + .ToList(); + + return tfs2; + } + + public void CalBasedOnSentence() + { + Categories = Sentences.Select(x => x.Label).Distinct().ToList(); + + tfs = new List>(); + + Sentences.ForEach(sent => + { + sent.Words.ForEach(word => + { + // TF + int c1 = sent.Words.Count(x => x.Lemma == word.Lemma); + double tf = (c1 + 1.0) / sent.Words.Count(); + + // IDF + var c2 = Sentences.Count(s => s.Words.Select(x => x.Lemma).Contains(word.Lemma)); + double idf = Math.Log(Sentences.Count / (c2 + 1.0)); + + word.Vector = tf * idf; + + tfs.Add(new Tuple(word.Lemma, word.Vector)); + }); + }); + } + + public void CalBasedOnCategory() + { + tfs = new List>(); + + Categories = Sentences.Select(x => x.Label).Distinct().ToList(); + + Categories.ForEach(label => + { + var allTokens = new List(); + Sentences.Where(x => x.Label == label) + .ToList() + .ForEach(s => allTokens.AddRange(s.Words)); + + allTokens.Select(x => x.Lemma).Distinct() + .ToList() + .ForEach(word => + { + // TF + int c1 = allTokens.Count(x => x.Lemma == word); + double tf = (c1 + 1.0) / allTokens.Count(); + + // IDF + var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word)) + .GroupBy(x => x.Label).Count(); + double idf = Math.Log(Categories.Count / (c2 + 1.0)); + + tfs.Add(new Tuple(word, tf * idf)); + }); + }); + } + + /// + /// Normalizes a TF*IDF array of vectors using L2-Norm. + /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + /// + /// List> + /// List> + public static List> Normalize(List> vectors) + { + // Normalize the vectors using L2-Norm. + List> normalizedVectors = new List>(); + foreach (var vector in vectors) + { + var normalized = Normalize(vector); + normalizedVectors.Add(normalized); + } + + return normalizedVectors; + } + + /// + /// Normalizes a TF*IDF vector using L2-Norm. + /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + /// + /// List + /// List + public static List Normalize(List vector) + { + List result = new List(); + + double sumSquared = 0; + foreach (var value in vector) + { + sumSquared += value * value; + } + + double SqrtSumSquared = Math.Sqrt(sumSquared); + + foreach (var value in vector) + { + // L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + result.Add(value / SqrtSumSquared); + } + return result; + } + } +} diff --git a/BotSharp.NLP/Models/TF-IDF/TFIDF.cs b/BotSharp.NLP/Models/TF-IDF/TFIDF.cs deleted file mode 100644 index 1fcf67f6..00000000 --- a/BotSharp.NLP/Models/TF-IDF/TFIDF.cs +++ /dev/null @@ -1,232 +0,0 @@ -using BotSharp.NLP.Tokenize; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.Serialization.Formatters.Binary; -using System.Text; -using System.Text.RegularExpressions; - -namespace BotSharp.NLP.Models.TF_IDF -{ - /// - /// Copyright (c) 2018 Bo Peng - /// - /// Permission is hereby granted, free of charge, to any person obtaining - /// a copy of this software and associated documentation files (the - /// "Software"), to deal in the Software without restriction, including - /// without limitation the rights to use, copy, modify, merge, publish, - /// distribute, sublicense, and/or sell copies of the Software, and to - /// permit persons to whom the Software is furnished to do so, subject to - /// the following conditions: - /// - /// The above copyright notice and this permission notice shall be - /// included in all copies or substantial portions of the Software. - /// - public class TFIDF - { - List vocabulary { get; set; } - - public TFIDF() - { - } - - /// - /// Document vocabulary, containing each word's IDF value. - /// - private static Dictionary _vocabularyIDF = new Dictionary(); - - public static List> GetTFIDFWeightsVectors(string[] documents, int vocabularyThreshold = 1) - { - List> stemmedDocs; - List vocabulary; - // Get the vocabulary and stem the documents at the same time. - vocabulary = GetVocabulary(documents, out stemmedDocs, vocabularyThreshold); - if (_vocabularyIDF.Count == 0) - { - // Calculate the IDF for each vocabulary term. - foreach (var term in vocabulary) - { - double numberOfDocsContainingTerm = stemmedDocs.Where(d => d.Contains(term)).Count(); - _vocabularyIDF[term] = Math.Log((double)stemmedDocs.Count / ((double)1 + numberOfDocsContainingTerm)); - } - } - // Transform each document into a vector of tfidf values. - List> vectors = new List>(); - foreach (var doc in stemmedDocs) - { - List vector = new List(); - foreach (string word in doc) - { - double tf = doc.Where(d => d == word).Count(); - double tfidf = tf * _vocabularyIDF[word]; - vector.Add(tfidf); - } - vectors.Add(vector); - } - return vectors; - } - - - /// - /// Normalizes a TF*IDF array of vectors using L2-Norm. - /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - /// - /// List> - /// List> - public static List> Normalize(List> vectors) - { - // Normalize the vectors using L2-Norm. - List> normalizedVectors = new List>(); - foreach (var vector in vectors) - { - var normalized = Normalize(vector); - normalizedVectors.Add(normalized); - } - - return normalizedVectors; - } - - /// - /// Normalizes a TF*IDF vector using L2-Norm. - /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - /// - /// List - /// List - public static List Normalize(List vector) - { - List result = new List(); - - double sumSquared = 0; - foreach (var value in vector) - { - sumSquared += value * value; - } - - double SqrtSumSquared = Math.Sqrt(sumSquared); - - foreach (var value in vector) - { - // L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - result.Add(value / SqrtSumSquared); - } - return result; - } - - /// - /// Saves the TFIDF vocabulary to disk. - /// - /// File path - public static void Save(string filePath = "vocabulary.dat") - { - // Save result to disk. - using (FileStream fs = new FileStream(filePath, FileMode.Create)) - { - BinaryFormatter formatter = new BinaryFormatter(); - formatter.Serialize(fs, _vocabularyIDF); - } - } - - /// - /// Loads the TFIDF vocabulary from disk. - /// - /// File path - public static void Load(string filePath = "vocabulary.dat") - { - // Load from disk. - using (FileStream fs = new FileStream(filePath, FileMode.Open)) - { - BinaryFormatter formatter = new BinaryFormatter(); - _vocabularyIDF = (Dictionary)formatter.Deserialize(fs); - } - } - - /// - /// Parses and tokenizes a list of documents, returning a vocabulary of words. - /// - /// string[] - /// List of List of string - /// Vocabulary (list of strings) - private static List GetVocabulary(string[] docs, out List> stemmedDocs, int vocabularyThreshold) - { - List vocabulary = new List(); - Dictionary wordCountList = new Dictionary(); - stemmedDocs = new List>(); - int docIndex = 0; - var tokenizer = new TokenizerFactory(new TokenizationOptions - { - Pattern = RegexTokenizer.WHITE_SPACE - }, SupportedLanguage.English); - - foreach (var doc in docs) - { - List stemmedDoc = new List(); - docIndex++; - if (docIndex % 100 == 0) - { - Console.WriteLine("Processing " + docIndex + "/" + docs.Length); - } - - List tokens = tokenizer.Tokenize(doc); - List list = new List(); - tokenizer.Tokenize(doc).ForEach( token => { - list.Add(token.Text.ToLower()); - }); - string[] parts2 = list.ToArray(); - //string[] parts2 = Tokenize(doc); - List words = new List(); - foreach (string part in parts2) - { - // Strip non-alphanumeric characters. - string stripped = Regex.Replace(part, "[^a-zA-Z0-9]", ""); - try - { - var english = new EnglishWord(stripped); - string stem = english.Stem; - words.Add(stem); - - if (stem.Length > 0) - { - // Build the word count list. - if (wordCountList.ContainsKey(stem)) - { - wordCountList[stem]++; - } - else - { - wordCountList.Add(stem, 0); - } - stemmedDoc.Add(stem); - } - } - catch - { - } - } - stemmedDocs.Add(stemmedDoc); - } - // Get the top words. - var vocabList = wordCountList.Where(w => w.Value >= vocabularyThreshold); - foreach (var item in vocabList) - { - vocabulary.Add(item.Key); - } - return vocabulary; - } - - - } - public class EnglishWord - { - public EnglishWord(string input) - { - this.Original = input; - this.Stem = input; - this.Length = input.Length; - } - - public string Stem { get; set; } - public string Original { get; } - public int Length { get; } - } -} diff --git a/BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs b/BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs deleted file mode 100644 index 670f84b3..00000000 --- a/BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.NLP.Models.TF_IDF -{ - /// - /// Copyright (c) 2018 Bo Peng - /// - /// Permission is hereby granted, free of charge, to any person obtaining - /// a copy of this software and associated documentation files (the - /// "Software"), to deal in the Software without restriction, including - /// without limitation the rights to use, copy, modify, merge, publish, - /// distribute, sublicense, and/or sell copies of the Software, and to - /// permit persons to whom the Software is furnished to do so, subject to - /// the following conditions: - /// - /// The above copyright notice and this permission notice shall be - /// included in all copies or substantial portions of the Software. - /// - public class TFIDFGenerator - { - public List> TFIDFWeightVectorsForSentences(string[]documents) - { - List> res = TFIDF.GetTFIDFWeightsVectors(documents, 0); - res = TFIDF.Normalize(res); - return res; - } - } -} diff --git a/BotSharp.NLP/Tokenize/Token.cs b/BotSharp.NLP/Tokenize/Token.cs index e6637642..4013fe52 100644 --- a/BotSharp.NLP/Tokenize/Token.cs +++ b/BotSharp.NLP/Tokenize/Token.cs @@ -67,5 +67,7 @@ namespace BotSharp.NLP.Tokenize { return $"{Text} {Start} {Pos}"; } + + public double Vector { get; set; } } } diff --git a/BotSharp.NLP/Tokenize/TokenizerFactory.cs b/BotSharp.NLP/Tokenize/TokenizerFactory.cs index 5518326e..fd8b22ec 100644 --- a/BotSharp.NLP/Tokenize/TokenizerFactory.cs +++ b/BotSharp.NLP/Tokenize/TokenizerFactory.cs @@ -29,7 +29,9 @@ namespace BotSharp.NLP.Tokenize public List Tokenize(string sentence) { - return _tokenizer.Tokenize(sentence, _options); + var tokens = _tokenizer.Tokenize(sentence, _options); + tokens.ForEach(x => x.Lemma = x.Text.ToLower()); + return tokens; } public List Tokenize(List sentences) @@ -39,6 +41,7 @@ namespace BotSharp.NLP.Tokenize Parallel.ForEach(sents, (sentence) => { sentence.Words = Tokenize(sentence.Text); + sentence.Words.ForEach(x => x.Lemma = x.Text.ToLower()); }); return sents; diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 7d630bac..510a37d9 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -15,17 +15,17 @@ namespace BotSharp.NLP.Txt2Vec { public List Sentences { get; set; } - private List words; + public List Words { get; set; } public void Encode(Sentence sentence) { InitDictionary(); - var vector = words.Select(x => 0D).ToArray(); + var vector = Words.Select(x => 0D).ToArray(); sentence.Words.ForEach(w => { - int index = words.IndexOf(w.Text.ToLower()); + int index = Words.IndexOf(w.Lemma.ToLower()); if(index > 0) { vector[index] = 1; @@ -35,26 +35,24 @@ namespace BotSharp.NLP.Txt2Vec sentence.Vector = vector; } - public void EncodeAll() + public List EncodeAll() { InitDictionary(); - Parallel.ForEach(Sentences, sent => - { - Encode(sent); - }); + + Sentences.ForEach(sent => Encode(sent)); + //Parallel.ForEach(Sentences, sent => Encode(sent)); + + return Words; } - private void InitDictionary() + private List InitDictionary() { - if (words == null) + if (Words == null) { - words = new List(); - Sentences.ForEach(x => - { - words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); - }); - words = words.Distinct().OrderBy(x => x).ToList(); + // Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move".Split(',').ToList(); } + + return Words; } } } diff --git a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs index dba24342..7c848031 100644 --- a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs +++ b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs @@ -5,7 +5,6 @@ using System.Text; using System.Threading.Tasks; using System.IO; using System.Threading; -using BotSharp.NLP.Models.TF_IDF; //using AdvUtils; namespace Txt2Vec @@ -37,8 +36,8 @@ namespace Txt2Vec public List Sentence2Vec(List sentences, WeightingScheme weightingScheme = WeightingScheme.AVG) { // Inplementing TF-IDF - TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); - List> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray()); + // TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); + List> weights = null;// tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray()); List> matixList = new List>(); diff --git a/Settings/bot.json b/Settings/bot.json index 013beac1..34e48d70 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -7,10 +7,13 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier" }, - + + "BotSharpNBayesClassifier": { + }, + "BotSharpSVMClassifier": { "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" },