From 30f0eea87cc18f7fafa225360bca50fbada481ba Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 12 Sep 2018 15:31:20 -0500 Subject: [PATCH] Intergate NBClassifier into NLP pipeline. --- .../Bayes/MultinomiaNaiveBayesModel.cs | 2 + .../BotSharp/BotSharpNBayesClassifier.cs | 41 +++++++--- .../NaiveBayesClassifierTest.cs | 7 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 12 +-- BotSharp.NLP/Classify/IClassifier.cs | 12 ++- BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 76 +++++++++++++++---- BotSharp.NLP/Classify/SVMClassifier.cs | 14 +++- BotSharp.NLP/Txt2Vec/OneHotEncoder.cs | 23 +++--- 8 files changed, 137 insertions(+), 50 deletions(-) diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs index ecccfb43..17e47538 100644 --- a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs @@ -10,5 +10,7 @@ namespace BotSharp.Algorithm.Bayes public List LabelDist { get; set; } public Dictionary CondProbDictionary { get; set; } + + public List Values { get; set; } } } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs index 61418c4c..bc1380e5 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -22,25 +22,22 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { meta.Model = "classification-nb.model"; - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - var encoder = new OneHotEncoder(); - encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence + var options = new ClassifyOptions + { + ModelFilePath = modelFileName + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var sentences = doc.Sentences.Select(x => new Sentence { Label = x.Intent.Label, Text = x.Text, Words = x.Tokens }).ToList(); - encoder.EncodeAll(); - var options = new ClassifyOptions - { - TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") - }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); - - classifier.Train(encoder.Sentences); + classifier.Train(sentences); Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); @@ -51,6 +48,28 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { + var options = new ClassifyOptions + { + ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model) + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var sentence = doc.Sentences.Select(s => new Sentence + { + Text = s.Text, + Words = s.Tokens + }).First(); + + + var result = classifier.Classify(sentence); + + doc.Sentences[0].Intent = new TextClassificationResult + { + Classifier = "BotSharpNBayesClassifier", + Label = result.First().Item1, + Confidence = (decimal)result.First().Item2 + }; + return true; } } diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index be599c14..2b8c2e0b 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -36,17 +36,14 @@ namespace BotSharp.NLP.UnitTest sentences.Shuffle(); - var encoder = new OneHotEncoder(); - encoder.Sentences = sentences; - encoder.EncodeAll(); - var options = new ClassifyOptions { + ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"), TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(0.7M); + var dataset = sentences.Split(1M); classifier.Train(dataset.Item1); int correct = 0; diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index 824c9eed..5f91b735 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -29,20 +29,20 @@ namespace BotSharp.NLP.Classify public void Train(List sentences) { - var vectors = new List>(); - - var sents = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); - - _classifier.Train(sents, new double[] { 0, 1 }, _options); + _classifier.Train(sentences, _options); + _classifier.SaveModel(_options); } public List> Classify(Sentence sentence) { var options = new ClassifyOptions { + ModelFilePath = _options.ModelFilePath }; - var classes = _classifier.Classify(sentence.Vector, options); + _classifier.LoadModel(options); + + var classes = _classifier.Classify(sentence, options); return classes.OrderByDescending(x => x.Item2).ToList(); } diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs index d84c0ab0..f0aa48b7 100644 --- a/BotSharp.NLP/Classify/IClassifier.cs +++ b/BotSharp.NLP/Classify/IClassifier.cs @@ -10,16 +10,20 @@ namespace BotSharp.NLP.Classify /// /// Training by feature vector /// - /// + /// /// - void Train(List> featureSets, double[] values, ClassifyOptions options); + void Train(List sentences, ClassifyOptions options); /// /// Predict by feature vector /// - /// + /// /// /// - List> Classify(double[] features, ClassifyOptions options); + List> Classify(Sentence sentence, ClassifyOptions options); + + String SaveModel(ClassifyOptions options); + + Object LoadModel(ClassifyOptions options); } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 1d81aac8..e29a881d 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -22,6 +22,8 @@ using BotSharp.Algorithm.Estimators; using BotSharp.Algorithm.Extensions; using BotSharp.Algorithm.Features; using BotSharp.Algorithm.Statistics; +using BotSharp.NLP.Txt2Vec; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; @@ -46,14 +48,24 @@ namespace BotSharp.NLP.Classify private Dictionary condProbDictionary = new Dictionary(); - public void Train(List> featureSets, double[] values, ClassifyOptions options) + private List words; + private double[] features = new double[] { 0, 1 }; + + public void Train(List sentences, ClassifyOptions options) { + var encoder = new OneHotEncoder(); + encoder.Sentences = sentences; + words = encoder.EncodeAll(); + + var featureSets = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); + labelDist = featureSets.GroupBy(x => x.Item1) .Select(x => new Probability { Value = x.Key, Freq = x.Count() }) + .OrderBy(x => x.Value) .ToList(); nb.LabelDist = labelDist; @@ -70,30 +82,27 @@ namespace BotSharp.NLP.Classify { for (int x = 0; x < featureCount; x++) { - for (int v = 0; v < values.Length; v++) + for (int v = 0; v < features.Length; v++) { - string key = $"{label.Value} f{x} {values[v]}"; - condProbDictionary[key] = nb.CalCondProb(x, label.Value, values[v]); + string key = $"{label.Value} f{x} {features[v]}"; + condProbDictionary[key] = nb.CalCondProb(x, label.Value, features[v]); } } }); - - // save the model - var model = new MultinomiaNaiveBayesModel - { - LabelDist = labelDist, - CondProbDictionary = condProbDictionary - }; } - public List> Classify(double[] features, ClassifyOptions options) + public List> Classify(Sentence sentence, ClassifyOptions options) { + var encoder = new OneHotEncoder(); + encoder.Words = words; + encoder.Encode(sentence); + var results = new List>(); // calculate prop labelDist.ForEach(lf => { - var prob = nb.CalPosteriorProb(lf.Value, features, lf.Prob, condProbDictionary); + var prob = nb.CalPosteriorProb(lf.Value, sentence.Vector, lf.Prob, condProbDictionary); results.Add(new Tuple(lf.Value, prob)); }); @@ -105,6 +114,47 @@ namespace BotSharp.NLP.Classify return results; } + + public string SaveModel(ClassifyOptions options) + { + // save the model + var model = new MultinomiaNaiveBayesModel + { + LabelDist = labelDist, + CondProbDictionary = condProbDictionary, + Values = words + }; + + //save the file + using (var bw = new BinaryWriter(new FileStream(options.ModelFilePath, FileMode.Create))) + { + var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model)); + bw.Write(bytes); + } + + return options.ModelFilePath; + } + + public Object LoadModel(ClassifyOptions options) + { + string json = String.Empty; + + //read the file + using (var br = new BinaryReader(new FileStream(options.ModelFilePath, FileMode.Open))) + { + byte[] bytes = br.ReadBytes((int)br.BaseStream.Length); + + json = Encoding.UTF8.GetString(bytes); + } + + var model = JsonConvert.DeserializeObject(json); + + labelDist = model.LabelDist; + condProbDictionary = model.CondProbDictionary; + words = model.Values; + + return model; + } } public class FeaturesWithLabel diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index cf56e333..01ae8cec 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -48,12 +48,12 @@ namespace BotSharp.NLP.Classify return Prediction.PredictLabelsProbability(options.Model, scaled); } - public void Train(List> featureSets, double[] values, ClassifyOptions options) + public void Train(List sentences, ClassifyOptions options) { // SVMClassifierTrain(featureSets, options); } - public List> Classify(double[] features, ClassifyOptions options) + public List> Classify(Sentence sentence, ClassifyOptions options) { throw new NotImplementedException(); } @@ -154,5 +154,15 @@ namespace BotSharp.NLP.Classify return labeledFeatureSet; } + + public string SaveModel(ClassifyOptions options) + { + throw new NotImplementedException(); + } + + object IClassifier.LoadModel(ClassifyOptions options) + { + throw new NotImplementedException(); + } } } diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 9e4f51d5..6c3152a0 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -15,17 +15,17 @@ namespace BotSharp.NLP.Txt2Vec { public List Sentences { get; set; } - private List words; + public List Words { get; set; } public void Encode(Sentence sentence) { InitDictionary(); - var vector = words.Select(x => 0D).ToArray(); + var vector = Words.Select(x => 0D).ToArray(); sentence.Words.ForEach(w => { - int index = words.IndexOf(w.Text.ToLower()); + int index = Words.IndexOf(w.Text.ToLower()); if(index > 0) { vector[index] = 1; @@ -35,24 +35,29 @@ namespace BotSharp.NLP.Txt2Vec sentence.Vector = vector; } - public void EncodeAll() + public List EncodeAll() { InitDictionary(); + Sentences.ForEach(sent => Encode(sent)); //Parallel.ForEach(Sentences, sent => Encode(sent)); + + return Words; } - private void InitDictionary() + private List InitDictionary() { - if (words == null) + if (Words == null) { - words = new List(); + Words = new List(); Sentences.ForEach(x => { - words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); + Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower())); }); - words = words.Distinct().OrderBy(x => x).ToList(); + Words = Words.Distinct().OrderBy(x => x).ToList(); } + + return Words; } } }