From 930cc700b9b6aa6974df9faddc474291f16d7032 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 26 Sep 2018 06:45:35 -0500 Subject: [PATCH] Abstractor BotSharpIntentClassifier --- ...ssifier.cs => BotSharpIntentClassifier.cs} | 47 ++++++++++++------- .../NaiveBayesClassifierTest.cs | 6 +-- BotSharp.NLP/Classify/ClassifierFactory.cs | 23 +++++++-- BotSharp.NLP/Classify/SVMClassifier.cs | 28 ++++++++++- BotSharp.WebHost/Settings/bot.json | 31 ++++++------ 5 files changed, 93 insertions(+), 42 deletions(-) rename BotSharp.Core/Engines/BotSharp/{BotSharpNBayesClassifier.cs => BotSharpIntentClassifier.cs} (56%) diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs similarity index 56% rename from BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs rename to BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs index 11845549..1672aad3 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs @@ -14,21 +14,15 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines.BotSharp { - public class BotSharpNBayesClassifier : INlpTrain, INlpPredict + public class BotSharpIntentClassifier : INlpTrain, INlpPredict { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } + private ClassifierFactory _classifier; public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { - meta.Model = "classification-nb.model"; - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - - var options = new ClassifyOptions - { - ModelFilePath = modelFileName - }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + Init(meta); var sentences = doc.Sentences.Select(x => new Sentence { @@ -37,20 +31,16 @@ namespace BotSharp.Core.Engines.BotSharp Words = x.Tokens }).ToList(); - classifier.Train(sentences); + _classifier.Train(sentences); - Console.WriteLine($"Saved model to {modelFileName}"); + Console.WriteLine($"Saved model to {Settings.ModelDir}"); return true; } public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { - var options = new ClassifyOptions - { - ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model) - }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + Init(meta); var sentence = doc.Sentences.Select(s => new Sentence { @@ -59,16 +49,37 @@ namespace BotSharp.Core.Engines.BotSharp }).First(); - var result = classifier.Classify(sentence); + var result = _classifier.Classify(sentence); doc.Sentences[0].Intent = new TextClassificationResult { - Classifier = "BotSharpNBayesClassifier", + Classifier = "BotSharpIntentClassifier", Label = result.First().Item1, Confidence = (decimal)result.First().Item2 }; return true; } + + private void Init(PipeModel meta) + { + if (_classifier == null) + { + meta.Model = "intent.model"; + + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + + var options = new ClassifyOptions + { + ModelFilePath = modelFileName + }; + + _classifier = new ClassifierFactory(options, SupportedLanguage.English); + + string classifierName = Configuration.GetValue($"classifer"); + + _classifier.GetClassifer(classifierName); + } + } } } diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index cabbeb71..b72fbc58 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -44,7 +44,7 @@ namespace BotSharp.NLP.UnitTest TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"), Dimension = 100 }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + var classifier = new ClassifierFactory(options, SupportedLanguage.English); var dataset = sentences.Split(0.7M); classifier.Train(dataset.Item1); @@ -73,7 +73,7 @@ namespace BotSharp.NLP.UnitTest { TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Gender") }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + var classifier = new ClassifierFactory(options, SupportedLanguage.English); var corpus = GetLabeledCorpus(options); @@ -159,7 +159,7 @@ namespace BotSharp.NLP.UnitTest ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"), TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify") }; - var classifier = new ClassifierFactory(options, SupportedLanguage.English); + var classifier = new ClassifierFactory(options, SupportedLanguage.English); var dataset = sentences.Split(0.7M); classifier.Train(dataset.Item1); diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index cc754488..f1a9d479 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -3,17 +3,17 @@ using BotSharp.NLP.Tokenize; using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Text; namespace BotSharp.NLP.Classify { - public class ClassifierFactory - where IClassify : IClassifier, new() + public class ClassifierFactory where IFeatureExtractor : ITextFeatureExtractor, new() { private SupportedLanguage _lang; - private IClassify _classifier; + private IClassifier _classifier; private ClassifyOptions _options; @@ -23,10 +23,25 @@ namespace BotSharp.NLP.Classify { _lang = lang; _options = options; - _classifier = new IClassify(); featureExtractor = new IFeatureExtractor(); } + public IClassifier GetClassifer(string name) + { + List types = new List(); + + types.AddRange(Assembly.Load(new AssemblyName("BotSharp.Core")) + .GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList()); + + types.AddRange(Assembly.Load(new AssemblyName("BotSharp.NLP")) + .GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList()); + + Type type = types.FirstOrDefault(x => x.Name == name); + var instance = (IClassifier)Activator.CreateInstance(type); + + return _classifier = instance; + } + public void Train(List sentences) { _classifier.Train(sentences, _options); diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 01ae8cec..7bbf8429 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -22,6 +22,8 @@ using System.IO; using System.Linq; using System.Text; using BotSharp.Algorithm.Features; +using BotSharp.NLP.Featuring; +using BotSharp.NLP.Txt2Vec; using SVM.BotSharp.MachineLearning; using Txt2Vec; @@ -32,6 +34,8 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { + private List words; + public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options) { Problem predict = new Problem(); @@ -50,7 +54,26 @@ namespace BotSharp.NLP.Classify public void Train(List sentences, ClassifyOptions options) { - // SVMClassifierTrain(featureSets, options); + var tfidf = new TfIdfFeatureExtractor(); + tfidf.Dimension = options.Dimension; + tfidf.Sentences = sentences; + tfidf.CalBasedOnCategory(); + + var encoder = new OneHotEncoder(); + encoder.Sentences = sentences; + encoder.Words = tfidf.Keywords(); + words = encoder.EncodeAll(); + + var featureSets = new List(); + sentences.ForEach(sent => + { + var fl = new FeaturesWithLabel(); + fl.Label = sent.Label; + fl.Features = sent.Words.Select(x => new Feature(words.IndexOf(x.Lemma).ToString(), words.Contains(x.Lemma) ? "1" : "0")).ToList(); + featureSets.Add(fl); + }); + + SVMClassifierTrain(featureSets, options); } public List> Classify(Sentence sentence, ClassifyOptions options) @@ -93,10 +116,11 @@ namespace BotSharp.NLP.Classify public List GetLabels(List featureSets) { + var categories = featureSets.Select(x => x.Label).Distinct().OrderBy(x => x).ToList(); List labels = new List(); foreach (var labelFeatureSet in featureSets) { - labels.Add(double.Parse(labelFeatureSet.Label)); + labels.Add(double.Parse(categories.IndexOf(labelFeatureSet.Label).ToString())); } return labels; diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index b3119328..7ec9c497 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -7,28 +7,29 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier" }, - - "BotSharpTokenizer": { - "tokenizer": "TreebankTokenizer" - }, - - "BotSharpNBayesClassifier": { - }, - + + "BotSharpTokenizer": { + "tokenizer": "TreebankTokenizer" + }, + + "BotSharpIntentClassifier": { + "classifer": "NaiveBayesClassifier" + }, + "BotSharpSVMClassifier": { "wordvec": "" }, "BotSharpTagger": { - "tagger": "NGramTagger" + "tagger": "NGramTagger" + }, + + "BotSharpCRFNer": { + "template": "|App_Data|CRFLite/template.en" }, - - "BotSharpCRFNer": { - "template": "|App_Data|CRFLite/template.en" - }, "CRFsuiteEntityRecognizer": { "fields": "y w pos chk",