SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/Oceania2018/BotSharp

Browse source
This commit is contained in:
Oceania2018 2018-09-05 17:33:50 -05:00
parent f61859e5a4 ee27d804c2
commit e80084068f
4 changed files with 82 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
View file

@ -3,7 +3,9 @@ using BotSharp.Core.Agents;
using BotSharp.NLP.Classify;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Diagnostics;
@ -31,19 +33,46 @@ namespace BotSharp.Core.Engines.BotSharp
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
/*
//
var client = new RestClient("http://10.2.21.200:5005");
var request = new RestRequest("doc2vec", Method.GET);
request.AddParameter("text", doc.Sentences[0].Text);
var response = client.Execute<PredResult>(request);
PredResult pred = JsonConvert.DeserializeObject<PredResult>(response.Content);
Vec vec = new Vec();
vec.VecNodes = pred.Doc2Vec;
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(vec, "");
//
*/
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data"));
double[][] d = svmClassifier.Predict(featureSet, classifyOptions);
string intent = null;
decimal confidence = 0;
double max = Double.MinValue;
for (int i = 0; i < d[0].Count(); i++)
{
if (d[0][i] > max)
{
max = d[0][i];
intent = agent.Intents[i].Name;
confidence = (decimal)d[0][i];
}
}
File.Delete(predictFileName);
//doc.Sentences[0].Intent = new TextClassificationResult
//{
// Classifier = "FasttextClassifier",
// Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1],
// Confidence = decimal.Parse(output.Split(' ')[1])
//};
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "SVMClassifier",
Label = intent,
Confidence = confidence
};
return true;
}
@ -71,15 +100,48 @@ namespace BotSharp.Core.Engines.BotSharp
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
/*
// try using spacy doc2vec
var client = new RestClient("http://10.2.21.200:5005");
var request = new RestRequest("batchdoc2vec", Method.POST);
request.RequestFormat = DataFormat.Json;
request.AddParameter("application/json", JsonConvert.SerializeObject(new {Sentences = sentences}), ParameterType.RequestBody);
var response = client.Execute<Result>(request);
Result res = JsonConvert.DeserializeObject<Result>(response.Content);
List<Vec> vecs = new List<Vec>();
foreach (List<double> cur in res.Doc2vecList)
{
Vec vec = new Vec();
vec.VecNodes = cur;
vecs.Add(vec);
}
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(vecs, labels);
//
*/
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model");
classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data");
svmClassifier.Train(featureSetList, classifyOptions);
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 31, 2018";
return true;
}
}
public class Result
{
public List<List<double>> Doc2vecList { get; set; }
}
public class PredResult
{
public List<double> Doc2Vec{ get; set; }
}
}

4
BotSharp.NLP/BotSharp.NLP.csproj
View file

@ -39,6 +39,10 @@ Mark milestone 0.2.3. Tokenizer, POS Tagger, NER and Text Classifier modules can
<DefineConstants>DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
</ItemGroup>

2
BotSharp.NLP/Classify/ClassifyOptions.cs
View file

@ -11,5 +11,7 @@ namespace BotSharp.NLP.Classify
public string ModelFilePath { get; set; }
public Model Model { get; set; }
public string PrediceOutputFile { get; set; }
public string TransformFilePath { get; set; }
public RangeTransform Transform { get; set; }
}
}

9
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -44,9 +44,9 @@ namespace BotSharp.NLP.Classify
predict.X = GetData(featureSets).ToArray();
predict.Y = new double[1];
predict.Count = predict.X.Count();
predict.MaxIndex = 200;
predict.MaxIndex = 300;
RangeTransform transform = RangeTransform.Compute(predict);
RangeTransform transform = options.Transform;
Problem scaled = transform.Scale(predict);
return Prediction.PredictLabelsProbability(options.Model, scaled);
@ -58,13 +58,13 @@ namespace BotSharp.NLP.Classify
}
public void SVMClassifierTrain(List<LabeledFeatureSet> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
{
// copy test multiclass Model
Problem train = new Problem();
train.X = GetData(featureSets).ToArray();
train.Y = GetLabels(featureSets).ToArray();
train.Count = train.X.Count();
train.MaxIndex = 200;//int.MaxValue;
train.MaxIndex = 300;//int.MaxValue;
Parameter param = new Parameter();
RangeTransform transform = RangeTransform.Compute(train);
@ -85,6 +85,7 @@ namespace BotSharp.NLP.Classify
param.Weights[i] = 1;
}
var model = Training.Train(scaled, param);
RangeTransform.Write(options.TransformFilePath, transform);
SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model);
Console.Write("Training finished!");
}