From 3347881031533095c92503df91507c2cf74a7bec Mon Sep 17 00:00:00 2001 From: Bolo Peng Date: Wed, 5 Sep 2018 14:38:37 -0500 Subject: [PATCH] SVMClassifier pipeline dev finish --- .../Engines/BotSharp/BotSharpSVMClassifier.cs | 32 +++++++++++++------ BotSharp.NLP/BotSharp.NLP.csproj | 4 +++ BotSharp.NLP/Classify/ClassifyOptions.cs | 2 ++ BotSharp.NLP/Classify/SVMClassifier.cs | 3 +- BotSharp.WebHost/Settings/bot.json | 4 +-- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs index 87a0ab3f..7e97210e 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -33,17 +33,32 @@ namespace BotSharp.Core.Engines.BotSharp LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), ""); ClassifyOptions classifyOptions = new ClassifyOptions(); - classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model")); + classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model")); + classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data")); double[][] d = svmClassifier.Predict(featureSet, classifyOptions); + string intent = null; + decimal confidence = 0; + double max = Double.MinValue; + for (int i = 0; i < d[0].Count(); i++) + { + if (d[0][i] > max) + { + max = d[0][i]; + intent = agent.Intents[i].Name; + confidence = (decimal)d[0][i]; + } + } + + File.Delete(predictFileName); - //doc.Sentences[0].Intent = new TextClassificationResult - //{ - // Classifier = "FasttextClassifier", - // Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], - // Confidence = decimal.Parse(output.Split(' ')[1]) - //}; + doc.Sentences[0].Intent = new TextClassificationResult + { + Classifier = "SVMClassifier", + Label = intent, + Confidence = confidence + }; return true; } @@ -73,12 +88,11 @@ namespace BotSharp.Core.Engines.BotSharp List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels); ClassifyOptions classifyOptions = new ClassifyOptions(); classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); + classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data"); svmClassifier.Train(featureSetList, classifyOptions); meta.Meta = new JObject(); meta.Meta["compiled at"] = "Aug 31, 2018"; - - return true; } } diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj index 98f53e11..4c6b55c3 100644 --- a/BotSharp.NLP/BotSharp.NLP.csproj +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -39,6 +39,10 @@ Mark milestone 0.2.3. Tokenizer, POS Tagger, NER and Text Classifier modules can DEBUG;TRACE + + + + diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs index 8ac0d1a3..fce017fd 100644 --- a/BotSharp.NLP/Classify/ClassifyOptions.cs +++ b/BotSharp.NLP/Classify/ClassifyOptions.cs @@ -11,5 +11,7 @@ namespace BotSharp.NLP.Classify public string ModelFilePath { get; set; } public Model Model { get; set; } public string PrediceOutputFile { get; set; } + public string TransformFilePath { get; set; } + public RangeTransform Transform { get; set; } } } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 81651088..3cc61996 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -46,7 +46,7 @@ namespace BotSharp.NLP.Classify predict.Count = predict.X.Count(); predict.MaxIndex = 200; - RangeTransform transform = RangeTransform.Compute(predict); + RangeTransform transform = options.Transform; Problem scaled = transform.Scale(predict); return Prediction.PredictLabelsProbability(options.Model, scaled); @@ -85,6 +85,7 @@ namespace BotSharp.NLP.Classify param.Weights[i] = 1; } var model = Training.Train(scaled, param); + RangeTransform.Write(options.TransformFilePath, transform); SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model); Console.Write("Training finished!"); } diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index 013beac1..30701332 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -7,8 +7,8 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpCBOWClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpCBOWClassifier" }, "BotSharpSVMClassifier": {