SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

try doc2vec using spacy, the svm classification based on that does NOThave a good performance

Browse source
This commit is contained in:
Bolo Peng 2018-09-05 17:10:47 -05:00
parent 42e62f0d00
commit ee27d804c2
3 changed files with 54 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
View file

@ -3,7 +3,9 @@ using BotSharp.Core.Agents;
using BotSharp.NLP.Classify;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Diagnostics;
@ -31,7 +33,20 @@ namespace BotSharp.Core.Engines.BotSharp
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
/*
//
var client = new RestClient("http://10.2.21.200:5005");
var request = new RestRequest("doc2vec", Method.GET);
request.AddParameter("text", doc.Sentences[0].Text);
var response = client.Execute<PredResult>(request);
PredResult pred = JsonConvert.DeserializeObject<PredResult>(response.Content);
Vec vec = new Vec();
vec.VecNodes = pred.Doc2Vec;
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(vec, "");
//
*/
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data"));
@ -50,7 +65,6 @@ namespace BotSharp.Core.Engines.BotSharp
}
}
File.Delete(predictFileName);
doc.Sentences[0].Intent = new TextClassificationResult
@ -86,6 +100,31 @@ namespace BotSharp.Core.Engines.BotSharp
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
/*
// try using spacy doc2vec
var client = new RestClient("http://10.2.21.200:5005");
var request = new RestRequest("batchdoc2vec", Method.POST);
request.RequestFormat = DataFormat.Json;
request.AddParameter("application/json", JsonConvert.SerializeObject(new {Sentences = sentences}), ParameterType.RequestBody);
var response = client.Execute<Result>(request);
Result res = JsonConvert.DeserializeObject<Result>(response.Content);
List<Vec> vecs = new List<Vec>();
foreach (List<double> cur in res.Doc2vecList)
{
Vec vec = new Vec();
vec.VecNodes = cur;
vecs.Add(vec);
}
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(vecs, labels);
//
*/
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model");
classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data");
@ -96,4 +135,13 @@ namespace BotSharp.Core.Engines.BotSharp
return true;
}
}
public class Result
{
public List<List<double>> Doc2vecList { get; set; }
}
public class PredResult
{
public List<double> Doc2Vec{ get; set; }
}
}

6
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -44,7 +44,7 @@ namespace BotSharp.NLP.Classify
predict.X = GetData(featureSets).ToArray();
predict.Y = new double[1];
predict.Count = predict.X.Count();
predict.MaxIndex = 200;
predict.MaxIndex = 300;
RangeTransform transform = options.Transform;
Problem scaled = transform.Scale(predict);
@ -58,13 +58,13 @@ namespace BotSharp.NLP.Classify
}
public void SVMClassifierTrain(List<LabeledFeatureSet> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
{
// copy test multiclass Model
Problem train = new Problem();
train.X = GetData(featureSets).ToArray();
train.Y = GetLabels(featureSets).ToArray();
train.Count = train.X.Count();
train.MaxIndex = 200;//int.MaxValue;
train.MaxIndex = 300;//int.MaxValue;
Parameter param = new Parameter();
RangeTransform transform = RangeTransform.Compute(train);

4
BotSharp.WebHost/Settings/bot.json
View file

@ -7,8 +7,8 @@
},
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpCBOWClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpCBOWClassifier"
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier"
},
"BotSharpSVMClassifier": {