diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs index 87a0ab3f..d0dfcd34 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -3,7 +3,9 @@ using BotSharp.Core.Agents; using BotSharp.NLP.Classify; using DotNetToolkit; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using RestSharp; using System; using System.Collections.Generic; using System.Diagnostics; @@ -31,19 +33,46 @@ namespace BotSharp.Core.Engines.BotSharp Args args = new Args(); args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), ""); + /* + // + var client = new RestClient("http://10.2.21.200:5005"); + var request = new RestRequest("doc2vec", Method.GET); + request.AddParameter("text", doc.Sentences[0].Text); + var response = client.Execute(request); + PredResult pred = JsonConvert.DeserializeObject(response.Content); + Vec vec = new Vec(); + vec.VecNodes = pred.Doc2Vec; + + LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(vec, ""); + // + */ ClassifyOptions classifyOptions = new ClassifyOptions(); - classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model")); + classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model")); + classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data")); double[][] d = svmClassifier.Predict(featureSet, classifyOptions); + string intent = null; + decimal confidence = 0; + double max = Double.MinValue; + for (int i = 0; i < d[0].Count(); i++) + { + if (d[0][i] > max) + { + max = d[0][i]; + intent = agent.Intents[i].Name; + confidence = (decimal)d[0][i]; + } + } + File.Delete(predictFileName); - //doc.Sentences[0].Intent = new TextClassificationResult - //{ - // Classifier = "FasttextClassifier", - // Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], - // Confidence = decimal.Parse(output.Split(' ')[1]) - //}; + doc.Sentences[0].Intent = new TextClassificationResult + { + Classifier = "SVMClassifier", + Label = intent, + Confidence = confidence + }; return true; } @@ -71,15 +100,48 @@ namespace BotSharp.Core.Engines.BotSharp Args args = new Args(); args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels); + + /* + // try using spacy doc2vec + var client = new RestClient("http://10.2.21.200:5005"); + var request = new RestRequest("batchdoc2vec", Method.POST); + request.RequestFormat = DataFormat.Json; + + request.AddParameter("application/json", JsonConvert.SerializeObject(new {Sentences = sentences}), ParameterType.RequestBody); + + var response = client.Execute(request); + Result res = JsonConvert.DeserializeObject(response.Content); + + List vecs = new List(); + foreach (List cur in res.Doc2vecList) + { + Vec vec = new Vec(); + vec.VecNodes = cur; + vecs.Add(vec); + } + List featureSetList = svmClassifier.FeatureSetsGenerator(vecs, labels); + // + */ + + + ClassifyOptions classifyOptions = new ClassifyOptions(); classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); + classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data"); svmClassifier.Train(featureSetList, classifyOptions); meta.Meta = new JObject(); meta.Meta["compiled at"] = "Aug 31, 2018"; - - return true; } } + public class Result + { + public List> Doc2vecList { get; set; } + } + + public class PredResult + { + public List Doc2Vec{ get; set; } + } } diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj index 98f53e11..4c6b55c3 100644 --- a/BotSharp.NLP/BotSharp.NLP.csproj +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -39,6 +39,10 @@ Mark milestone 0.2.3. Tokenizer, POS Tagger, NER and Text Classifier modules can DEBUG;TRACE + + + + diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs index 8ac0d1a3..fce017fd 100644 --- a/BotSharp.NLP/Classify/ClassifyOptions.cs +++ b/BotSharp.NLP/Classify/ClassifyOptions.cs @@ -11,5 +11,7 @@ namespace BotSharp.NLP.Classify public string ModelFilePath { get; set; } public Model Model { get; set; } public string PrediceOutputFile { get; set; } + public string TransformFilePath { get; set; } + public RangeTransform Transform { get; set; } } } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 81651088..433b1c83 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -44,9 +44,9 @@ namespace BotSharp.NLP.Classify predict.X = GetData(featureSets).ToArray(); predict.Y = new double[1]; predict.Count = predict.X.Count(); - predict.MaxIndex = 200; + predict.MaxIndex = 300; - RangeTransform transform = RangeTransform.Compute(predict); + RangeTransform transform = options.Transform; Problem scaled = transform.Scale(predict); return Prediction.PredictLabelsProbability(options.Model, scaled); @@ -58,13 +58,13 @@ namespace BotSharp.NLP.Classify } public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) - { + { // copy test multiclass Model Problem train = new Problem(); train.X = GetData(featureSets).ToArray(); train.Y = GetLabels(featureSets).ToArray(); train.Count = train.X.Count(); - train.MaxIndex = 200;//int.MaxValue; + train.MaxIndex = 300;//int.MaxValue; Parameter param = new Parameter(); RangeTransform transform = RangeTransform.Compute(train); @@ -85,6 +85,7 @@ namespace BotSharp.NLP.Classify param.Weights[i] = 1; } var model = Training.Train(scaled, param); + RangeTransform.Write(options.TransformFilePath, transform); SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model); Console.Write("Training finished!"); }