From f87e50641747460519016b7af268b2578251c333 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Tue, 11 Sep 2018 16:17:12 -0500 Subject: [PATCH 1/7] Fix MultinomiaNaiveBayes bug. --- .../Bayes/BernoulliNaiveBayes.cs | 10 ++ .../Bayes/GaussianNaiveBayes.cs | 10 ++ .../Bayes/MultinomiaNaiveBayes.cs | 110 +++++++++++++++ BotSharp.Algorithm/Bayes/NaiveBayes.cs | 65 --------- .../{Lidstone.cs => AdditiveSmoothing.cs} | 25 +++- .../NaiveBayesClassifierTest.cs | 13 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 35 ++--- BotSharp.NLP/Classify/IClassifier.cs | 4 - BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 126 +++++------------- BotSharp.NLP/Txt2Vec/OneHotEncoder.cs | 6 +- 10 files changed, 203 insertions(+), 201 deletions(-) create mode 100644 BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs create mode 100644 BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs create mode 100644 BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs delete mode 100644 BotSharp.Algorithm/Bayes/NaiveBayes.cs rename BotSharp.Algorithm/Estimators/{Lidstone.cs => AdditiveSmoothing.cs} (75%) diff --git a/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs b/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs new file mode 100644 index 00000000..901c3737 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class BernoulliNaiveBayes + { + } +} diff --git a/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs b/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs new file mode 100644 index 00000000..f02d44b4 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class GaussianNaiveBayes + { + } +} diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs new file mode 100644 index 00000000..f10f268b --- /dev/null +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs @@ -0,0 +1,110 @@ +/* + * BotSharp.Algorithm + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.Algorithm.Estimators; +using BotSharp.Algorithm.Features; +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + /// + /// https://en.wikipedia.org/wiki/Bayes%27_theorem + /// + public class MultinomiaNaiveBayes + { + public List LabelDist { get; set; } + + public List> FeatureSet { get; set; } + + public double Alpha { get; set; } + + /// + /// prior probability + /// + /// + /// + public double CalPriorProb(string Y) + { + int N = FeatureSet.Count; + int k = LabelDist.Count; + int Nyk = LabelDist.First(x => x.Value == Y).Freq; + + return (Nyk + Alpha) / (N + k * Alpha); + } + + /// + /// calculate posterior probability P(Y|X) + /// X is feature set, Y is label + /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) + /// + public double PosteriorProb(string Y, double[] features, double priorProb) + { + Alpha = 0.5; + + int featureCount = features.Length; + + double postProb = priorProb; + + // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); + var matrix = ConstructMatrix(featuresIfY); + + // loop features + for (int x = 0; x < featureCount; x++) + { + int freq = 0; + for (int y = 0; y < featuresIfY.Count; y++) + { + if(matrix[y, x] == features[x]) + { + freq++; + } + } + + int Nyk = featuresIfY.Count; + int n = featureCount; + int Nykx = freq; + + postProb += Math.Log((Nykx + Alpha) / (Nyk + n * Alpha)); + } + + return Math.Pow(2, postProb); + } + + private double[,] ConstructMatrix(List> featuresIfY) + { + var featureCount = featuresIfY[0].Item2.Length; + + double[,] matrix = new double[featuresIfY.Count, featureCount]; + for (int y = 0; y < featuresIfY.Count; y++) + { + for (int x = 0; x < featureCount; x++) + { + matrix[y, x] = featuresIfY[y].Item2[x]; + } + } + + return matrix; + } + } +} diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs deleted file mode 100644 index 099ca6eb..00000000 --- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs +++ /dev/null @@ -1,65 +0,0 @@ -using BotSharp.Algorithm.Estimators; -using BotSharp.Algorithm.Features; -using BotSharp.Algorithm.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace BotSharp.Algorithm.Bayes -{ - /// - /// https://en.wikipedia.org/wiki/Bayes%27_theorem - /// - public class NaiveBayes where Estimator : IEstimator, new() - { - /// - /// smoothing function - /// - private Estimator estomator; - - public List FeaturesDist { get; set; } - - public List LabelDist { get; set; } - - public NaiveBayes() - { - estomator = new Estimator(); - } - - /// - /// calculate posterior probability P(Y|X) - /// X is feature set, Y is label - /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) - /// - /// label - /// - /// - public double PosteriorProb(string Y, List features) - { - double prob = 0; - - // prior probability - prob = Math.Log(estomator.Prob(LabelDist, Y), 2); - - // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList(); - - // loop features - for (int x = 0; x < features.Count; x++) - { - var Xn = features[x]; - var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues; - - if(fv != null) - { - // features are independent, so calculate every feature prob and sum them - prob += Math.Log(estomator.Prob(fv, Xn.Value), 2); - } - } - - return prob; - } - } -} diff --git a/BotSharp.Algorithm/Estimators/Lidstone.cs b/BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs similarity index 75% rename from BotSharp.Algorithm/Estimators/Lidstone.cs rename to BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs index d1153b16..0e934031 100644 --- a/BotSharp.Algorithm/Estimators/Lidstone.cs +++ b/BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs @@ -31,10 +31,12 @@ namespace BotSharp.Algorithm.Estimators /// https://en.wikipedia.org/wiki/Additive_smoothing /// Used as Multinomial Naive Bayes /// - public class Lidstone : IEstimator + public class AdditiveSmoothing : IEstimator { /// - /// α > 0 is the smoothing parameter + /// 1 > α > 0 is the smoothing parameter is Lidstone + /// α = 1 is Laplace + /// α = 0 no smoothing /// public double Alpha { get; set; } @@ -62,5 +64,24 @@ namespace BotSharp.Algorithm.Estimators return (x + Alpha) / (_N + Alpha * _d); } + + public double Prob(List> dist, string sample) + { + if (Alpha == 0) + { + Alpha = 0.5D; + } + + // observation x = (x1, ..., xd) + var p = dist.Find(f => f.Item1 == sample); + double x = p == null ? 0D : p.Item2; + + // N trials + double _N = dist.Sum(f => f.Item2); + + int _d = dist.Count; + + return (x + Alpha) / (_N + Alpha * _d); + } } } diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 5cfefb69..e8f295e7 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -32,9 +32,9 @@ namespace BotSharp.NLP.UnitTest { newSentences[i].Label = sentences[i].Label; } - sentences = newSentences.ToList(); + sentences = newSentences.Take(10).ToList(); - sentences.Shuffle(); + //sentences.Shuffle(); var encoder = new OneHotEncoder(); encoder.Sentences = sentences; @@ -46,9 +46,7 @@ namespace BotSharp.NLP.UnitTest }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(0.9M); - classifier.TrainInVector(dataset.Item1); - + var dataset = sentences.Split(1M); classifier.Train(dataset.Item1); int correct = 0; @@ -58,10 +56,13 @@ namespace BotSharp.NLP.UnitTest if (td.Label == classes[0].Item1) { correct++; + } }); - var accuracy = (float)correct / dataset.Item2.Count; + var accuracy = (float)correct / dataset.Item1.Count; + + Assert.IsTrue(accuracy > 0.8); } [TestMethod] diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index bd32aac8..eebccb07 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -27,31 +27,7 @@ namespace BotSharp.NLP.Classify featureExtractor = new IFeatureExtractor(); } - public List> Classify(Sentence sentence) - { - var options = new ClassifyOptions - { - }; - - var features = featureExtractor.GetFeatures(sentence.Words); - - var classes = _classifier.Classify(features, options); - - return classes.OrderByDescending(x => x.Item2).ToList(); - } - public void Train(List sentences) - { - var sents = sentences.Select(x => new FeaturesWithLabel - { - Label = x.Label, - Features = featureExtractor.GetFeatures(x.Words) - }).ToList(); - - _classifier.Train(sents, _options); - } - - public void TrainInVector(List sentences) { var vectors = new List>(); @@ -59,5 +35,16 @@ namespace BotSharp.NLP.Classify _classifier.Train(sents, _options); } + + public List> Classify(Sentence sentence) + { + var options = new ClassifyOptions + { + }; + + var classes = _classifier.Classify(sentence.Vector, options); + + return classes.OrderByDescending(x => x.Item2).ToList(); + } } } diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index 5b011508..c52a71c2 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -7,10 +7,6 @@ namespace BotSharp.NLP.Classify { public interface IClassifier { - void Train(List featureSets, ClassifyOptions options); - - List> Classify(List features, ClassifyOptions options); - /// /// Training by feature vector /// diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 834b1864..ae141862 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -40,104 +40,14 @@ namespace BotSharp.NLP.Classify /// public class NaiveBayesClassifier : IClassifier { - private List featuresDist; - private List labelDist; - public void Train(List featureSets, ClassifyOptions options) - { - labelDist = featureSets.GroupBy(x => x.Label) - .Select(x => new Probability - { - Value = x.Key, - Freq = x.Count() - }) - .ToList(); + private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes(); - var fNames = new List(); - - featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name))); - fNames = fNames.OrderBy(x => x).Distinct().ToList(); - - var featureValues = new Dictionary>(); - - for (int i = 0; i < featureSets.Count; i++) - { - var fs = featureSets[i]; - featureValues[fs.Label] = new List(); - - fNames.ForEach(fn => - { - Feature feature = null; - for (int j = 0; j < fs.Features.Count; j++) - { - if (fs.Features[j].Name == fn) - { - feature = fs.Features[j]; - break; - } - } - - var fv = new Feature(fn, feature == null ? "False" : feature.Value); - featureValues[fs.Label].Add(fv); - }); - } - - featuresDist = new List(); - - labelDist.Select(x => x.Value).ToList().ForEach(label => - { - var fSets = featureValues[label]; - - fNames.ForEach(fName => - { - var fsv = fSets.Where(fs => fs.Name == fName) - .GroupBy(fs => fs.Value) - .Select(fs => new Probability - { - Value = fs.Key, - Freq = fs.Count() - }) - .OrderBy(fs => fs.Value) - .ToList(); - - featuresDist.Add(new FeaturesDistribution - { - Label = label, - FeatureName = fName, - FeatureValues = fsv - }); - }); - }); - } - - public List> Classify(List features, ClassifyOptions options) - { - // calculate prop - var nb = new NaiveBayes(); - nb.LabelDist = labelDist; - nb.FeaturesDist = featuresDist; - - Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features)); - - // add log - double[] logs = labelDist.Select(x => x.Prob).ToArray(); - - var sumLogs = logs.Reduce((log1, next) => - { - double min = log1; - if (next < log1) - { - min = next; - } - - return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2); - }); - - labelDist.ForEach(d => d.Prob -= sumLogs); - - return labelDist.Select(x => new Tuple(x.Value, x.Prob)).ToList(); - } + /// + /// Cache all categories' prior probability + /// + private Dictionary PriorPropDictionary = new Dictionary(); public void Train(List> featureSets, ClassifyOptions options) { @@ -148,11 +58,35 @@ namespace BotSharp.NLP.Classify Freq = x.Count() }) .ToList(); + + nb.LabelDist = labelDist; + nb.FeatureSet = featureSets; + + // calculate prior prob + labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value)); + + // calculate posterior prob + } public List> Classify(double[] features, ClassifyOptions options) { - throw new NotImplementedException(); + var results = new List>(); + + // calculate prop + labelDist.ForEach(lf => + { + var prob = nb.PosteriorProb(lf.Value, features, lf.Prob); + results.Add(new Tuple(lf.Value, prob)); + }); + + /*Parallel.ForEach(labelDist, (lf) => + { + nb.Y = lf.Value; + lf.Prob = nb.PosteriorProb(); + });*/ + + return results; } } diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 7d630bac..9e4f51d5 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -38,10 +38,8 @@ namespace BotSharp.NLP.Txt2Vec public void EncodeAll() { InitDictionary(); - Parallel.ForEach(Sentences, sent => - { - Encode(sent); - }); + Sentences.ForEach(sent => Encode(sent)); + //Parallel.ForEach(Sentences, sent => Encode(sent)); } private void InitDictionary() From aec1984ab838f58cf22a1a352a9d099999b82157 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Tue, 11 Sep 2018 17:29:36 -0500 Subject: [PATCH 2/7] define bin model --- .../Bayes/MultinomiaNaiveBayes.cs | 55 +++++++++++-------- .../Bayes/MultinomiaNaiveBayesModel.cs | 14 +++++ .../Engines/BotSharp/BotSharpSVMClassifier.cs | 2 +- .../NaiveBayesClassifierTest.cs | 4 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 2 +- BotSharp.NLP/Classify/IClassifier.cs | 2 +- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 29 ++++++++-- BotSharp.NLP/Classify/SVMClassifier.cs | 26 +++------ 8 files changed, 81 insertions(+), 53 deletions(-) create mode 100644 BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs index f10f268b..977f207f 100644 --- a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs @@ -35,7 +35,12 @@ namespace BotSharp.Algorithm.Bayes public List> FeatureSet { get; set; } - public double Alpha { get; set; } + private double alpha { get; set; } + + public MultinomiaNaiveBayes(double alpha = 0.5) + { + this.alpha = alpha; + } /// /// prior probability @@ -48,7 +53,29 @@ namespace BotSharp.Algorithm.Bayes int k = LabelDist.Count; int Nyk = LabelDist.First(x => x.Value == Y).Freq; - return (Nyk + Alpha) / (N + k * Alpha); + return (Nyk + alpha) / (N + k * alpha); + } + + public double CalCondProb(int x, string Y, double feature) + { + // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) + var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); + var matrix = ConstructMatrix(featuresIfY); + + int freq = 0; + for (int y = 0; y < featuresIfY.Count; y++) + { + if (matrix[y, x] == feature) + { + freq++; + } + } + + int Nyk = featuresIfY.Count; + int n = featuresIfY.Count; + int Nykx = freq; + + return Math.Log((Nykx + alpha) / (Nyk + n * alpha)); } /// @@ -57,35 +84,17 @@ namespace BotSharp.Algorithm.Bayes /// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X) /// - public double PosteriorProb(string Y, double[] features, double priorProb) + public double CalPosteriorProb(string Y, double[] features, double priorProb, Dictionary condProbDictionary) { - Alpha = 0.5; - int featureCount = features.Length; double postProb = priorProb; - // posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y) - var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList(); - var matrix = ConstructMatrix(featuresIfY); - // loop features for (int x = 0; x < featureCount; x++) { - int freq = 0; - for (int y = 0; y < featuresIfY.Count; y++) - { - if(matrix[y, x] == features[x]) - { - freq++; - } - } - - int Nyk = featuresIfY.Count; - int n = featureCount; - int Nykx = freq; - - postProb += Math.Log((Nykx + Alpha) / (Nyk + n * Alpha)); + string key = $"{Y} f{x} {features[x]}"; + postProb += condProbDictionary[key]; } return Math.Pow(2, postProb); diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs new file mode 100644 index 00000000..ecccfb43 --- /dev/null +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs @@ -0,0 +1,14 @@ +using BotSharp.Algorithm.Statistics; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Bayes +{ + public class MultinomiaNaiveBayesModel + { + public List LabelDist { get; set; } + + public Dictionary CondProbDictionary { get; set; } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs index 9def3cc4..c9b1d0a6 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -129,7 +129,7 @@ namespace BotSharp.Core.Engines.BotSharp ClassifyOptions classifyOptions = new ClassifyOptions(); classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data"); - svmClassifier.Train(featureSetList, classifyOptions); + // svmClassifier.Train(featureSetList, classifyOptions); meta.Meta = new JObject(); meta.Meta["compiled at"] = "Aug 31, 2018"; diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index e8f295e7..cffd7c00 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -32,9 +32,9 @@ namespace BotSharp.NLP.UnitTest { newSentences[i].Label = sentences[i].Label; } - sentences = newSentences.Take(10).ToList(); + sentences = newSentences.ToList(); - //sentences.Shuffle(); + // sentences.Shuffle(); var encoder = new OneHotEncoder(); encoder.Sentences = sentences; diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index eebccb07..824c9eed 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -33,7 +33,7 @@ namespace BotSharp.NLP.Classify var sents = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - _classifier.Train(sents, _options); + _classifier.Train(sents, new double[] { 0, 1 }, _options); } public List> Classify(Sentence sentence) diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index c52a71c2..d84c0ab0 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -12,7 +12,7 @@ namespace BotSharp.NLP.Classify /// /// /// - void Train(List> featureSets, ClassifyOptions options); + void Train(List> featureSets, double[] values, ClassifyOptions options); /// /// Predict by feature vector diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index ae141862..1d81aac8 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -44,12 +44,9 @@ namespace BotSharp.NLP.Classify private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes(); - /// - /// Cache all categories' prior probability - /// - private Dictionary PriorPropDictionary = new Dictionary(); + private Dictionary condProbDictionary = new Dictionary(); - public void Train(List> featureSets, ClassifyOptions options) + public void Train(List> featureSets, double[] values, ClassifyOptions options) { labelDist = featureSets.GroupBy(x => x.Item1) .Select(x => new Probability @@ -66,7 +63,27 @@ namespace BotSharp.NLP.Classify labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value)); // calculate posterior prob + // loop features + var featureCount = nb.FeatureSet[0].Item2.Length; + labelDist.ForEach(label => + { + for (int x = 0; x < featureCount; x++) + { + for (int v = 0; v < values.Length; v++) + { + string key = $"{label.Value} f{x} {values[v]}"; + condProbDictionary[key] = nb.CalCondProb(x, label.Value, values[v]); + } + } + }); + + // save the model + var model = new MultinomiaNaiveBayesModel + { + LabelDist = labelDist, + CondProbDictionary = condProbDictionary + }; } public List> Classify(double[] features, ClassifyOptions options) @@ -76,7 +93,7 @@ namespace BotSharp.NLP.Classify // calculate prop labelDist.ForEach(lf => { - var prob = nb.PosteriorProb(lf.Value, features, lf.Prob); + var prob = nb.CalPosteriorProb(lf.Value, features, lf.Prob, condProbDictionary); results.Add(new Tuple(lf.Value, prob)); }); diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index be97f77e..cf56e333 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -32,11 +32,6 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { - public List> Classify(List features, ClassifyOptions options) - { - return null; - } - public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options) { Problem predict = new Problem(); @@ -53,9 +48,14 @@ namespace BotSharp.NLP.Classify return Prediction.PredictLabelsProbability(options.Model, scaled); } - public void Train(List featureSets, ClassifyOptions options) + public void Train(List> featureSets, double[] values, ClassifyOptions options) { - SVMClassifierTrain(featureSets, options); + // SVMClassifierTrain(featureSets, options); + } + + public List> Classify(double[] features, ClassifyOptions options) + { + throw new NotImplementedException(); } public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) @@ -154,17 +154,5 @@ namespace BotSharp.NLP.Classify return labeledFeatureSet; } - - public void Train(List> featureSets, ClassifyOptions options) - { - throw new NotImplementedException(); - } - - public List> Classify(double[] features, ClassifyOptions options) - { - throw new NotImplementedException(); - } } - - } From ee115ebde6629cc7b8a47bdc268985c1f1bc944e Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Tue, 11 Sep 2018 17:46:07 -0500 Subject: [PATCH 3/7] update CookingTest --- BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index cffd7c00..be599c14 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -34,7 +34,7 @@ namespace BotSharp.NLP.UnitTest } sentences = newSentences.ToList(); - // sentences.Shuffle(); + sentences.Shuffle(); var encoder = new OneHotEncoder(); encoder.Sentences = sentences; @@ -46,23 +46,24 @@ namespace BotSharp.NLP.UnitTest }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(1M); + var dataset = sentences.Split(0.7M); classifier.Train(dataset.Item1); int correct = 0; - dataset.Item1.ToList().ForEach(td => + int total = 0; + dataset.Item1.ForEach(td => { var classes = classifier.Classify(td); if (td.Label == classes[0].Item1) { correct++; - } + total++; }); - var accuracy = (float)correct / dataset.Item1.Count; + var accuracy = (float)correct / total; - Assert.IsTrue(accuracy > 0.8); + Assert.IsTrue(accuracy > 0.5); } [TestMethod] From 347cd5cd9c5eb3d06fe45d33543e701b5fe5d4d6 Mon Sep 17 00:00:00 2001 From: Esther2013 Date: Wed, 12 Sep 2018 07:34:56 -0500 Subject: [PATCH 4/7] New NBayesClassifier for Chatbot --- .../BotSharp/BotSharpNBayesClassifier.cs | 57 +++++++++++++++++++ .../Engines/BotSharp/BotSharpTokenizer.cs | 10 ++-- Settings/bot.json | 9 ++- 3 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs new file mode 100644 index 00000000..61418c4c --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -0,0 +1,57 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP; +using BotSharp.NLP.Classify; +using BotSharp.NLP.Txt2Vec; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpNBayesClassifier : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + meta.Model = "classification-nb.model"; + + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + + var encoder = new OneHotEncoder(); + encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence + { + Label = x.Intent.Label, + Text = x.Text, + Words = x.Tokens + }).ToList(); + encoder.EncodeAll(); + + var options = new ClassifyOptions + { + TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + classifier.Train(encoder.Sentences); + + Console.WriteLine($"Saved model to {modelFileName}"); + meta.Meta = new JObject(); + meta.Meta["compiled at"] = "Sep 12, 2018"; + + return true; + } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + return true; + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs index 0bba34a9..40d2476a 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs @@ -1,5 +1,6 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; +using BotSharp.Core.Intents; using BotSharp.NLP; using BotSharp.NLP.Tokenize; using Microsoft.Extensions.Configuration; @@ -14,14 +15,12 @@ namespace BotSharp.Core.Engines.BotSharp { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } - private TokenizerFactory _tokenizer; + private TokenizerFactory _tokenizer; public BotSharpTokenizer() { - _tokenizer = new TokenizerFactory(new TokenizationOptions + _tokenizer = new TokenizerFactory(new TokenizationOptions { - Pattern = RegexTokenizer.WORD_PUNC, - SpecialWords = new List { "'s" } }, SupportedLanguage.English); } @@ -48,7 +47,8 @@ namespace BotSharp.Core.Engines.BotSharp doc.Sentences.Add(new NlpDocSentence { Tokens = _tokenizer.Tokenize(say.Text), - Text = say.Text + Text = say.Text, + Intent = new TextClassificationResult { Label = say.Intent } }); }); diff --git a/Settings/bot.json b/Settings/bot.json index 013beac1..34e48d70 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -7,10 +7,13 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier" }, - + + "BotSharpNBayesClassifier": { + }, + "BotSharpSVMClassifier": { "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" }, From 30f0eea87cc18f7fafa225360bca50fbada481ba Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 12 Sep 2018 15:31:20 -0500 Subject: [PATCH 5/7] Intergate NBClassifier into NLP pipeline. --- .../Bayes/MultinomiaNaiveBayesModel.cs | 2 + .../BotSharp/BotSharpNBayesClassifier.cs | 41 +++++++--- .../NaiveBayesClassifierTest.cs | 7 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 12 +-- BotSharp.NLP/Classify/IClassifier.cs | 12 ++- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 76 +++++++++++++++---- BotSharp.NLP/Classify/SVMClassifier.cs | 14 +++- BotSharp.NLP/Txt2Vec/OneHotEncoder.cs | 23 +++--- 8 files changed, 137 insertions(+), 50 deletions(-) diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs index ecccfb43..17e47538 100644 --- a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs @@ -10,5 +10,7 @@ namespace BotSharp.Algorithm.Bayes public List LabelDist { get; set; } public Dictionary CondProbDictionary { get; set; } + + public List Values { get; set; } } } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs index 61418c4c..bc1380e5 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -22,25 +22,22 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { meta.Model = "classification-nb.model"; - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - var encoder = new OneHotEncoder(); - encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence + var options = new ClassifyOptions + { + ModelFilePath = modelFileName + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var sentences = doc.Sentences.Select(x => new Sentence { Label = x.Intent.Label, Text = x.Text, Words = x.Tokens }).ToList(); - encoder.EncodeAll(); - var options = new ClassifyOptions - { - TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") - }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); - - classifier.Train(encoder.Sentences); + classifier.Train(sentences); Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); @@ -51,6 +48,28 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { + var options = new ClassifyOptions + { + ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model) + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var sentence = doc.Sentences.Select(s => new Sentence + { + Text = s.Text, + Words = s.Tokens + }).First(); + + + var result = classifier.Classify(sentence); + + doc.Sentences[0].Intent = new TextClassificationResult + { + Classifier = "BotSharpNBayesClassifier", + Label = result.First().Item1, + Confidence = (decimal)result.First().Item2 + }; + return true; } } diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index be599c14..2b8c2e0b 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -36,17 +36,14 @@ namespace BotSharp.NLP.UnitTest sentences.Shuffle(); - var encoder = new OneHotEncoder(); - encoder.Sentences = sentences; - encoder.EncodeAll(); - var options = new ClassifyOptions { + ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"), TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(0.7M); + var dataset = sentences.Split(1M); classifier.Train(dataset.Item1); int correct = 0; diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index 824c9eed..5f91b735 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -29,20 +29,20 @@ namespace BotSharp.NLP.Classify public void Train(List sentences) { - var vectors = new List>(); - - var sents = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - - _classifier.Train(sents, new double[] { 0, 1 }, _options); + _classifier.Train(sentences, _options); + _classifier.SaveModel(_options); } public List> Classify(Sentence sentence) { var options = new ClassifyOptions { + ModelFilePath = _options.ModelFilePath }; - var classes = _classifier.Classify(sentence.Vector, options); + _classifier.LoadModel(options); + + var classes = _classifier.Classify(sentence, options); return classes.OrderByDescending(x => x.Item2).ToList(); } diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index d84c0ab0..f0aa48b7 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -10,16 +10,20 @@ namespace BotSharp.NLP.Classify /// /// Training by feature vector /// - /// + /// /// - void Train(List> featureSets, double[] values, ClassifyOptions options); + void Train(List sentences, ClassifyOptions options); /// /// Predict by feature vector /// - /// + /// /// /// - List> Classify(double[] features, ClassifyOptions options); + List> Classify(Sentence sentence, ClassifyOptions options); + + String SaveModel(ClassifyOptions options); + + Object LoadModel(ClassifyOptions options); } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 1d81aac8..e29a881d 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -22,6 +22,8 @@ using BotSharp.Algorithm.Estimators; using BotSharp.Algorithm.Extensions; using BotSharp.Algorithm.Features; using BotSharp.Algorithm.Statistics; +using BotSharp.NLP.Txt2Vec; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; @@ -46,14 +48,24 @@ namespace BotSharp.NLP.Classify private Dictionary condProbDictionary = new Dictionary(); - public void Train(List> featureSets, double[] values, ClassifyOptions options) + private List words; + private double[] features = new double[] { 0, 1 }; + + public void Train(List sentences, ClassifyOptions options) { + var encoder = new OneHotEncoder(); + encoder.Sentences = sentences; + words = encoder.EncodeAll(); + + var featureSets = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); + labelDist = featureSets.GroupBy(x => x.Item1) .Select(x => new Probability { Value = x.Key, Freq = x.Count() }) + .OrderBy(x => x.Value) .ToList(); nb.LabelDist = labelDist; @@ -70,30 +82,27 @@ namespace BotSharp.NLP.Classify { for (int x = 0; x < featureCount; x++) { - for (int v = 0; v < values.Length; v++) + for (int v = 0; v < features.Length; v++) { - string key = $"{label.Value} f{x} {values[v]}"; - condProbDictionary[key] = nb.CalCondProb(x, label.Value, values[v]); + string key = $"{label.Value} f{x} {features[v]}"; + condProbDictionary[key] = nb.CalCondProb(x, label.Value, features[v]); } } }); - - // save the model - var model = new MultinomiaNaiveBayesModel - { - LabelDist = labelDist, - CondProbDictionary = condProbDictionary - }; } - public List> Classify(double[] features, ClassifyOptions options) + public List> Classify(Sentence sentence, ClassifyOptions options) { + var encoder = new OneHotEncoder(); + encoder.Words = words; + encoder.Encode(sentence); + var results = new List>(); // calculate prop labelDist.ForEach(lf => { - var prob = nb.CalPosteriorProb(lf.Value, features, lf.Prob, condProbDictionary); + var prob = nb.CalPosteriorProb(lf.Value, sentence.Vector, lf.Prob, condProbDictionary); results.Add(new Tuple(lf.Value, prob)); }); @@ -105,6 +114,47 @@ namespace BotSharp.NLP.Classify return results; } + + public string SaveModel(ClassifyOptions options) + { + // save the model + var model = new MultinomiaNaiveBayesModel + { + LabelDist = labelDist, + CondProbDictionary = condProbDictionary, + Values = words + }; + + //save the file + using (var bw = new BinaryWriter(new FileStream(options.ModelFilePath, FileMode.Create))) + { + var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model)); + bw.Write(bytes); + } + + return options.ModelFilePath; + } + + public Object LoadModel(ClassifyOptions options) + { + string json = String.Empty; + + //read the file + using (var br = new BinaryReader(new FileStream(options.ModelFilePath, FileMode.Open))) + { + byte[] bytes = br.ReadBytes((int)br.BaseStream.Length); + + json = Encoding.UTF8.GetString(bytes); + } + + var model = JsonConvert.DeserializeObject(json); + + labelDist = model.LabelDist; + condProbDictionary = model.CondProbDictionary; + words = model.Values; + + return model; + } } public class FeaturesWithLabel diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index cf56e333..01ae8cec 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -48,12 +48,12 @@ namespace BotSharp.NLP.Classify return Prediction.PredictLabelsProbability(options.Model, scaled); } - public void Train(List> featureSets, double[] values, ClassifyOptions options) + public void Train(List sentences, ClassifyOptions options) { // SVMClassifierTrain(featureSets, options); } - public List> Classify(double[] features, ClassifyOptions options) + public List> Classify(Sentence sentence, ClassifyOptions options) { throw new NotImplementedException(); } @@ -154,5 +154,15 @@ namespace BotSharp.NLP.Classify return labeledFeatureSet; } + + public string SaveModel(ClassifyOptions options) + { + throw new NotImplementedException(); + } + + object IClassifier.LoadModel(ClassifyOptions options) + { + throw new NotImplementedException(); + } } } diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 9e4f51d5..6c3152a0 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -15,17 +15,17 @@ namespace BotSharp.NLP.Txt2Vec { public List Sentences { get; set; } - private List words; + public List Words { get; set; } public void Encode(Sentence sentence) { InitDictionary(); - var vector = words.Select(x => 0D).ToArray(); + var vector = Words.Select(x => 0D).ToArray(); sentence.Words.ForEach(w => { - int index = words.IndexOf(w.Text.ToLower()); + int index = Words.IndexOf(w.Text.ToLower()); if(index > 0) { vector[index] = 1; @@ -35,24 +35,29 @@ namespace BotSharp.NLP.Txt2Vec sentence.Vector = vector; } - public void EncodeAll() + public List EncodeAll() { InitDictionary(); + Sentences.ForEach(sent => Encode(sent)); //Parallel.ForEach(Sentences, sent => Encode(sent)); + + return Words; } - private void InitDictionary() + private List InitDictionary() { - if (words == null) + if (Words == null) { - words = new List(); + Words = new List(); Sentences.ForEach(x => { - words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); + Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); }); - words = words.Distinct().OrderBy(x => x).ToList(); + Words = Words.Distinct().OrderBy(x => x).ToList(); } + + return Words; } } } From 1528bd42ff718a19817cde47a16805ae7d875581 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 12 Sep 2018 17:52:10 -0500 Subject: [PATCH 6/7] TF-IDF --- BotSharp.NLP.UnitTest/SVMClassifierTest.cs | 5 +- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 4 + BotSharp.NLP/Models/TF-IDF/TFIDF.cs | 232 ------------------ BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs | 30 --- BotSharp.NLP/Txt2Vec/TFIDF.cs | 148 +++++++++++ BotSharp.NLP/Txt2Vec/VectorGenerator.cs | 5 +- 6 files changed, 156 insertions(+), 268 deletions(-) delete mode 100644 BotSharp.NLP/Models/TF-IDF/TFIDF.cs delete mode 100644 BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs create mode 100644 BotSharp.NLP/Txt2Vec/TFIDF.cs diff --git a/BotSharp.NLP.UnitTest/SVMClassifierTest.cs b/BotSharp.NLP.UnitTest/SVMClassifierTest.cs index f9b1d48a..732af1f5 100644 --- a/BotSharp.NLP.UnitTest/SVMClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/SVMClassifierTest.cs @@ -1,6 +1,5 @@ using BotSharp.NLP.Classify; using BotSharp.NLP.Corpus; -using BotSharp.NLP.Models.TF_IDF; using BotSharp.NLP.Tokenize; using Microsoft.Extensions.Configuration; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -29,8 +28,8 @@ namespace BotSharp.NLP.UnitTest "see you Bolo", "byebye Haiping" }; - TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); - List> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents); + /*TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); + List> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents);*/ } [TestMethod] diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index e29a881d..3baaef1b 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -53,6 +53,10 @@ namespace BotSharp.NLP.Classify public void Train(List sentences, ClassifyOptions options) { + var tfidf = new TFIDF(); + tfidf.Sentences = sentences; + words = tfidf.EncodeAll(); + var encoder = new OneHotEncoder(); encoder.Sentences = sentences; words = encoder.EncodeAll(); diff --git a/BotSharp.NLP/Models/TF-IDF/TFIDF.cs b/BotSharp.NLP/Models/TF-IDF/TFIDF.cs deleted file mode 100644 index 1fcf67f6..00000000 --- a/BotSharp.NLP/Models/TF-IDF/TFIDF.cs +++ /dev/null @@ -1,232 +0,0 @@ -using BotSharp.NLP.Tokenize; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.Serialization.Formatters.Binary; -using System.Text; -using System.Text.RegularExpressions; - -namespace BotSharp.NLP.Models.TF_IDF -{ - /// - /// Copyright (c) 2018 Bo Peng - /// - /// Permission is hereby granted, free of charge, to any person obtaining - /// a copy of this software and associated documentation files (the - /// "Software"), to deal in the Software without restriction, including - /// without limitation the rights to use, copy, modify, merge, publish, - /// distribute, sublicense, and/or sell copies of the Software, and to - /// permit persons to whom the Software is furnished to do so, subject to - /// the following conditions: - /// - /// The above copyright notice and this permission notice shall be - /// included in all copies or substantial portions of the Software. - /// - public class TFIDF - { - List vocabulary { get; set; } - - public TFIDF() - { - } - - /// - /// Document vocabulary, containing each word's IDF value. - /// - private static Dictionary _vocabularyIDF = new Dictionary(); - - public static List> GetTFIDFWeightsVectors(string[] documents, int vocabularyThreshold = 1) - { - List> stemmedDocs; - List vocabulary; - // Get the vocabulary and stem the documents at the same time. - vocabulary = GetVocabulary(documents, out stemmedDocs, vocabularyThreshold); - if (_vocabularyIDF.Count == 0) - { - // Calculate the IDF for each vocabulary term. - foreach (var term in vocabulary) - { - double numberOfDocsContainingTerm = stemmedDocs.Where(d => d.Contains(term)).Count(); - _vocabularyIDF[term] = Math.Log((double)stemmedDocs.Count / ((double)1 + numberOfDocsContainingTerm)); - } - } - // Transform each document into a vector of tfidf values. - List> vectors = new List>(); - foreach (var doc in stemmedDocs) - { - List vector = new List(); - foreach (string word in doc) - { - double tf = doc.Where(d => d == word).Count(); - double tfidf = tf * _vocabularyIDF[word]; - vector.Add(tfidf); - } - vectors.Add(vector); - } - return vectors; - } - - - /// - /// Normalizes a TF*IDF array of vectors using L2-Norm. - /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - /// - /// List> - /// List> - public static List> Normalize(List> vectors) - { - // Normalize the vectors using L2-Norm. - List> normalizedVectors = new List>(); - foreach (var vector in vectors) - { - var normalized = Normalize(vector); - normalizedVectors.Add(normalized); - } - - return normalizedVectors; - } - - /// - /// Normalizes a TF*IDF vector using L2-Norm. - /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - /// - /// List - /// List - public static List Normalize(List vector) - { - List result = new List(); - - double sumSquared = 0; - foreach (var value in vector) - { - sumSquared += value * value; - } - - double SqrtSumSquared = Math.Sqrt(sumSquared); - - foreach (var value in vector) - { - // L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - result.Add(value / SqrtSumSquared); - } - return result; - } - - /// - /// Saves the TFIDF vocabulary to disk. - /// - /// File path - public static void Save(string filePath = "vocabulary.dat") - { - // Save result to disk. - using (FileStream fs = new FileStream(filePath, FileMode.Create)) - { - BinaryFormatter formatter = new BinaryFormatter(); - formatter.Serialize(fs, _vocabularyIDF); - } - } - - /// - /// Loads the TFIDF vocabulary from disk. - /// - /// File path - public static void Load(string filePath = "vocabulary.dat") - { - // Load from disk. - using (FileStream fs = new FileStream(filePath, FileMode.Open)) - { - BinaryFormatter formatter = new BinaryFormatter(); - _vocabularyIDF = (Dictionary)formatter.Deserialize(fs); - } - } - - /// - /// Parses and tokenizes a list of documents, returning a vocabulary of words. - /// - /// string[] - /// List of List of string - /// Vocabulary (list of strings) - private static List GetVocabulary(string[] docs, out List> stemmedDocs, int vocabularyThreshold) - { - List vocabulary = new List(); - Dictionary wordCountList = new Dictionary(); - stemmedDocs = new List>(); - int docIndex = 0; - var tokenizer = new TokenizerFactory(new TokenizationOptions - { - Pattern = RegexTokenizer.WHITE_SPACE - }, SupportedLanguage.English); - - foreach (var doc in docs) - { - List stemmedDoc = new List(); - docIndex++; - if (docIndex % 100 == 0) - { - Console.WriteLine("Processing " + docIndex + "/" + docs.Length); - } - - List tokens = tokenizer.Tokenize(doc); - List list = new List(); - tokenizer.Tokenize(doc).ForEach( token => { - list.Add(token.Text.ToLower()); - }); - string[] parts2 = list.ToArray(); - //string[] parts2 = Tokenize(doc); - List words = new List(); - foreach (string part in parts2) - { - // Strip non-alphanumeric characters. - string stripped = Regex.Replace(part, "[^a-zA-Z0-9]", ""); - try - { - var english = new EnglishWord(stripped); - string stem = english.Stem; - words.Add(stem); - - if (stem.Length > 0) - { - // Build the word count list. - if (wordCountList.ContainsKey(stem)) - { - wordCountList[stem]++; - } - else - { - wordCountList.Add(stem, 0); - } - stemmedDoc.Add(stem); - } - } - catch - { - } - } - stemmedDocs.Add(stemmedDoc); - } - // Get the top words. - var vocabList = wordCountList.Where(w => w.Value >= vocabularyThreshold); - foreach (var item in vocabList) - { - vocabulary.Add(item.Key); - } - return vocabulary; - } - - - } - public class EnglishWord - { - public EnglishWord(string input) - { - this.Original = input; - this.Stem = input; - this.Length = input.Length; - } - - public string Stem { get; set; } - public string Original { get; } - public int Length { get; } - } -} diff --git a/BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs b/BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs deleted file mode 100644 index 670f84b3..00000000 --- a/BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.NLP.Models.TF_IDF -{ - /// - /// Copyright (c) 2018 Bo Peng - /// - /// Permission is hereby granted, free of charge, to any person obtaining - /// a copy of this software and associated documentation files (the - /// "Software"), to deal in the Software without restriction, including - /// without limitation the rights to use, copy, modify, merge, publish, - /// distribute, sublicense, and/or sell copies of the Software, and to - /// permit persons to whom the Software is furnished to do so, subject to - /// the following conditions: - /// - /// The above copyright notice and this permission notice shall be - /// included in all copies or substantial portions of the Software. - /// - public class TFIDFGenerator - { - public List> TFIDFWeightVectorsForSentences(string[]documents) - { - List> res = TFIDF.GetTFIDFWeightsVectors(documents, 0); - res = TFIDF.Normalize(res); - return res; - } - } -} diff --git a/BotSharp.NLP/Txt2Vec/TFIDF.cs b/BotSharp.NLP/Txt2Vec/TFIDF.cs new file mode 100644 index 00000000..b7e64a94 --- /dev/null +++ b/BotSharp.NLP/Txt2Vec/TFIDF.cs @@ -0,0 +1,148 @@ +/// +/// Copyright (c) 2018 Bo Peng +/// +/// Permission is hereby granted, free of charge, to any person obtaining +/// a copy of this software and associated documentation files (the +/// "Software"), to deal in the Software without restriction, including +/// without limitation the rights to use, copy, modify, merge, publish, +/// distribute, sublicense, and/or sell copies of the Software, and to +/// permit persons to whom the Software is furnished to do so, subject to +/// the following conditions: +/// +/// The above copyright notice and this permission notice shall be +/// included in all copies or substantial portions of the Software. +/// +/// +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Serialization.Formatters.Binary; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.NLP.Txt2Vec +{ + public class TFIDF + { + public List Sentences { get; set; } + + public List Words { get; set; } + + public void Encode(Sentence sentence) + { + InitDictionary(); + + // var featureSets = Sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); + + var labelDist = Sentences.Select(x => x.Label).Distinct().ToList(); + + labelDist.ForEach(label => + { + // https://zhuanlan.zhihu.com/p/31197209 + // calculate TF + // all words in the article + List words = new List(); + Sentences.Where(x => x.Label == label).ToList().ForEach(sent => + { + words.AddRange(sent.Words.Select(w => w.Text)); + }); + + List> tfs = new List>(); + words.Distinct().ToList().ForEach(w => + { + // TF + int c1 = words.Count(x => x == w); + double tf = (c1 + 1.0) / words.Count(); + + // IDF + var sents = Sentences.Where(s => s.Words.Select(x => x.Text).Contains(w)).ToList(); + double idf = Math.Log(Sentences.Count / (sents.Count() + 1.0)); + + tfs.Add(new Tuple(w, tf * idf)); + }); + + tfs = tfs.OrderByDescending(x => x.Item2).Take(words.Count / 10).ToList(); + }); + + + + sentence.Words.ForEach(w => + { + int index = Words.IndexOf(w.Text.ToLower()); + }); + } + + public List EncodeAll() + { + InitDictionary(); + + Sentences.ForEach(sent => Encode(sent)); + //Parallel.ForEach(Sentences, sent => Encode(sent)); + + return Words; + } + + private List InitDictionary() + { + if (Words == null) + { + Words = new List(); + Sentences.ForEach(x => + { + Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); + }); + Words = Words.Distinct().OrderBy(x => x).ToList(); + } + + return Words; + } + + + /// + /// Normalizes a TF*IDF array of vectors using L2-Norm. + /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + /// + /// List> + /// List> + public static List> Normalize(List> vectors) + { + // Normalize the vectors using L2-Norm. + List> normalizedVectors = new List>(); + foreach (var vector in vectors) + { + var normalized = Normalize(vector); + normalizedVectors.Add(normalized); + } + + return normalizedVectors; + } + + /// + /// Normalizes a TF*IDF vector using L2-Norm. + /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + /// + /// List + /// List + public static List Normalize(List vector) + { + List result = new List(); + + double sumSquared = 0; + foreach (var value in vector) + { + sumSquared += value * value; + } + + double SqrtSumSquared = Math.Sqrt(sumSquared); + + foreach (var value in vector) + { + // L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + result.Add(value / SqrtSumSquared); + } + return result; + } + } +} diff --git a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs index dba24342..7c848031 100644 --- a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs +++ b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs @@ -5,7 +5,6 @@ using System.Text; using System.Threading.Tasks; using System.IO; using System.Threading; -using BotSharp.NLP.Models.TF_IDF; //using AdvUtils; namespace Txt2Vec @@ -37,8 +36,8 @@ namespace Txt2Vec public List Sentence2Vec(List sentences, WeightingScheme weightingScheme = WeightingScheme.AVG) { // Inplementing TF-IDF - TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); - List> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray()); + // TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); + List> weights = null;// tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray()); List> matixList = new List>(); From a8c5242c7b0d7f8266f71c7907c66db2b9a0fddf Mon Sep 17 00:00:00 2001 From: Esther2013 Date: Wed, 12 Sep 2018 23:33:34 -0500 Subject: [PATCH 7/7] extractor key words from document. --- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 8 +- BotSharp.NLP/Featuring/IFeatureExtractor.cs | 10 ++ .../Featuring/TfIdfFeatureExtractor.cs | 155 ++++++++++++++++++ BotSharp.NLP/Tokenize/Token.cs | 2 + BotSharp.NLP/Tokenize/TokenizerFactory.cs | 5 +- BotSharp.NLP/Txt2Vec/OneHotEncoder.cs | 9 +- BotSharp.NLP/Txt2Vec/TFIDF.cs | 148 ----------------- 7 files changed, 178 insertions(+), 159 deletions(-) create mode 100644 BotSharp.NLP/Featuring/IFeatureExtractor.cs create mode 100644 BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs delete mode 100644 BotSharp.NLP/Txt2Vec/TFIDF.cs diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 3baaef1b..38f28d7b 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -22,6 +22,7 @@ using BotSharp.Algorithm.Estimators; using BotSharp.Algorithm.Extensions; using BotSharp.Algorithm.Features; using BotSharp.Algorithm.Statistics; +using BotSharp.NLP.Featuring; using BotSharp.NLP.Txt2Vec; using Newtonsoft.Json; using System; @@ -53,10 +54,11 @@ namespace BotSharp.NLP.Classify public void Train(List sentences, ClassifyOptions options) { - var tfidf = new TFIDF(); + var tfidf = new TfIdfFeatureExtractor(); tfidf.Sentences = sentences; - words = tfidf.EncodeAll(); - + tfidf.CalBasedOnCategory(); + var keyWords = tfidf.Features(); + string keywords2 = String.Join(",", keyWords.ToArray()); var encoder = new OneHotEncoder(); encoder.Sentences = sentences; words = encoder.EncodeAll(); diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs new file mode 100644 index 00000000..785f1d30 --- /dev/null +++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Featuring +{ + public interface IFeatureExtractor + { + } +} diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs new file mode 100644 index 00000000..8f3ccf77 --- /dev/null +++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs @@ -0,0 +1,155 @@ +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Serialization.Formatters.Binary; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.NLP.Featuring +{ + public class TfIdfFeatureExtractor : IFeatureExtractor + { + public List Sentences { get; set; } + + private List> tfs; + + private List Categories { get; set; } + + public void Extract(Sentence sentence) + { + + } + + public List Features() + { + var tfs2 = tfs.OrderByDescending(x => x.Item2) + .Select(x => x.Item1) + .Distinct() + .Take(Sentences.Count / Categories.Count) + .ToList(); + + return tfs2; + } + + public void CalBasedOnSentence() + { + Categories = Sentences.Select(x => x.Label).Distinct().ToList(); + + tfs = new List>(); + + Sentences.ForEach(sent => + { + sent.Words.ForEach(word => + { + // TF + int c1 = sent.Words.Count(x => x.Lemma == word.Lemma); + double tf = (c1 + 1.0) / sent.Words.Count(); + + // IDF + var c2 = Sentences.Count(s => s.Words.Select(x => x.Lemma).Contains(word.Lemma)); + double idf = Math.Log(Sentences.Count / (c2 + 1.0)); + + word.Vector = tf * idf; + + tfs.Add(new Tuple(word.Lemma, word.Vector)); + }); + }); + } + + public void CalBasedOnCategory() + { + tfs = new List>(); + + Categories = Sentences.Select(x => x.Label).Distinct().ToList(); + + Categories.ForEach(label => + { + var allTokens = new List(); + Sentences.Where(x => x.Label == label) + .ToList() + .ForEach(s => allTokens.AddRange(s.Words)); + + allTokens.Select(x => x.Lemma).Distinct() + .ToList() + .ForEach(word => + { + // TF + int c1 = allTokens.Count(x => x.Lemma == word); + double tf = (c1 + 1.0) / allTokens.Count(); + + // IDF + var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word)) + .GroupBy(x => x.Label).Count(); + double idf = Math.Log(Categories.Count / (c2 + 1.0)); + + tfs.Add(new Tuple(word, tf * idf)); + }); + }); + } + + /// + /// Normalizes a TF*IDF array of vectors using L2-Norm. + /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + /// + /// List> + /// List> + public static List> Normalize(List> vectors) + { + // Normalize the vectors using L2-Norm. + List> normalizedVectors = new List>(); + foreach (var vector in vectors) + { + var normalized = Normalize(vector); + normalizedVectors.Add(normalized); + } + + return normalizedVectors; + } + + /// + /// Normalizes a TF*IDF vector using L2-Norm. + /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + /// + /// List + /// List + public static List Normalize(List vector) + { + List result = new List(); + + double sumSquared = 0; + foreach (var value in vector) + { + sumSquared += value * value; + } + + double SqrtSumSquared = Math.Sqrt(sumSquared); + + foreach (var value in vector) + { + // L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) + result.Add(value / SqrtSumSquared); + } + return result; + } + } +} diff --git a/BotSharp.NLP/Tokenize/Token.cs b/BotSharp.NLP/Tokenize/Token.cs index e6637642..4013fe52 100644 --- a/BotSharp.NLP/Tokenize/Token.cs +++ b/BotSharp.NLP/Tokenize/Token.cs @@ -67,5 +67,7 @@ namespace BotSharp.NLP.Tokenize { return $"{Text} {Start} {Pos}"; } + + public double Vector { get; set; } } } diff --git a/BotSharp.NLP/Tokenize/TokenizerFactory.cs b/BotSharp.NLP/Tokenize/TokenizerFactory.cs index 5518326e..fd8b22ec 100644 --- a/BotSharp.NLP/Tokenize/TokenizerFactory.cs +++ b/BotSharp.NLP/Tokenize/TokenizerFactory.cs @@ -29,7 +29,9 @@ namespace BotSharp.NLP.Tokenize public List Tokenize(string sentence) { - return _tokenizer.Tokenize(sentence, _options); + var tokens = _tokenizer.Tokenize(sentence, _options); + tokens.ForEach(x => x.Lemma = x.Text.ToLower()); + return tokens; } public List Tokenize(List sentences) @@ -39,6 +41,7 @@ namespace BotSharp.NLP.Tokenize Parallel.ForEach(sents, (sentence) => { sentence.Words = Tokenize(sentence.Text); + sentence.Words.ForEach(x => x.Lemma = x.Text.ToLower()); }); return sents; diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 6c3152a0..510a37d9 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec sentence.Words.ForEach(w => { - int index = Words.IndexOf(w.Text.ToLower()); + int index = Words.IndexOf(w.Lemma.ToLower()); if(index > 0) { vector[index] = 1; @@ -49,12 +49,7 @@ namespace BotSharp.NLP.Txt2Vec { if (Words == null) { - Words = new List(); - Sentences.ForEach(x => - { - Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); - }); - Words = Words.Distinct().OrderBy(x => x).ToList(); + // Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move".Split(',').ToList(); } return Words; diff --git a/BotSharp.NLP/Txt2Vec/TFIDF.cs b/BotSharp.NLP/Txt2Vec/TFIDF.cs deleted file mode 100644 index b7e64a94..00000000 --- a/BotSharp.NLP/Txt2Vec/TFIDF.cs +++ /dev/null @@ -1,148 +0,0 @@ -/// -/// Copyright (c) 2018 Bo Peng -/// -/// Permission is hereby granted, free of charge, to any person obtaining -/// a copy of this software and associated documentation files (the -/// "Software"), to deal in the Software without restriction, including -/// without limitation the rights to use, copy, modify, merge, publish, -/// distribute, sublicense, and/or sell copies of the Software, and to -/// permit persons to whom the Software is furnished to do so, subject to -/// the following conditions: -/// -/// The above copyright notice and this permission notice shall be -/// included in all copies or substantial portions of the Software. -/// -/// -using BotSharp.NLP.Tokenize; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.Serialization.Formatters.Binary; -using System.Text; -using System.Text.RegularExpressions; - -namespace BotSharp.NLP.Txt2Vec -{ - public class TFIDF - { - public List Sentences { get; set; } - - public List Words { get; set; } - - public void Encode(Sentence sentence) - { - InitDictionary(); - - // var featureSets = Sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - - var labelDist = Sentences.Select(x => x.Label).Distinct().ToList(); - - labelDist.ForEach(label => - { - // https://zhuanlan.zhihu.com/p/31197209 - // calculate TF - // all words in the article - List words = new List(); - Sentences.Where(x => x.Label == label).ToList().ForEach(sent => - { - words.AddRange(sent.Words.Select(w => w.Text)); - }); - - List> tfs = new List>(); - words.Distinct().ToList().ForEach(w => - { - // TF - int c1 = words.Count(x => x == w); - double tf = (c1 + 1.0) / words.Count(); - - // IDF - var sents = Sentences.Where(s => s.Words.Select(x => x.Text).Contains(w)).ToList(); - double idf = Math.Log(Sentences.Count / (sents.Count() + 1.0)); - - tfs.Add(new Tuple(w, tf * idf)); - }); - - tfs = tfs.OrderByDescending(x => x.Item2).Take(words.Count / 10).ToList(); - }); - - - - sentence.Words.ForEach(w => - { - int index = Words.IndexOf(w.Text.ToLower()); - }); - } - - public List EncodeAll() - { - InitDictionary(); - - Sentences.ForEach(sent => Encode(sent)); - //Parallel.ForEach(Sentences, sent => Encode(sent)); - - return Words; - } - - private List InitDictionary() - { - if (Words == null) - { - Words = new List(); - Sentences.ForEach(x => - { - Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); - }); - Words = Words.Distinct().OrderBy(x => x).ToList(); - } - - return Words; - } - - - /// - /// Normalizes a TF*IDF array of vectors using L2-Norm. - /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - /// - /// List> - /// List> - public static List> Normalize(List> vectors) - { - // Normalize the vectors using L2-Norm. - List> normalizedVectors = new List>(); - foreach (var vector in vectors) - { - var normalized = Normalize(vector); - normalizedVectors.Add(normalized); - } - - return normalizedVectors; - } - - /// - /// Normalizes a TF*IDF vector using L2-Norm. - /// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - /// - /// List - /// List - public static List Normalize(List vector) - { - List result = new List(); - - double sumSquared = 0; - foreach (var value in vector) - { - sumSquared += value * value; - } - - double SqrtSumSquared = Math.Sqrt(sumSquared); - - foreach (var value in vector) - { - // L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2) - result.Add(value / SqrtSumSquared); - } - return result; - } - } -}