SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

SVMClassifier pipeline dev finish

Browse source
This commit is contained in:
Bolo Peng 2018-09-05 14:38:37 -05:00
parent a05968292f
commit 3347881031
5 changed files with 33 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
View file

@ -33,17 +33,32 @@ namespace BotSharp.Core.Engines.BotSharp
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data"));
double[][] d = svmClassifier.Predict(featureSet, classifyOptions);
string intent = null;
decimal confidence = 0;
double max = Double.MinValue;
for (int i = 0; i < d[0].Count(); i++)
{
if (d[0][i] > max)
{
max = d[0][i];
intent = agent.Intents[i].Name;
confidence = (decimal)d[0][i];
}
}
File.Delete(predictFileName);
//doc.Sentences[0].Intent = new TextClassificationResult
//{
// Classifier = "FasttextClassifier",
// Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1],
// Confidence = decimal.Parse(output.Split(' ')[1])
//};
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "SVMClassifier",
Label = intent,
Confidence = confidence
};
return true;
}
@ -73,12 +88,11 @@ namespace BotSharp.Core.Engines.BotSharp
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model");
classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data");
svmClassifier.Train(featureSetList, classifyOptions);
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 31, 2018";
return true;
}
}

4
BotSharp.NLP/BotSharp.NLP.csproj
View file

@ -39,6 +39,10 @@ Mark milestone 0.2.3. Tokenizer, POS Tagger, NER and Text Classifier modules can
<DefineConstants>DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
</ItemGroup>

2
BotSharp.NLP/Classify/ClassifyOptions.cs
View file

@ -11,5 +11,7 @@ namespace BotSharp.NLP.Classify
public string ModelFilePath { get; set; }
public Model Model { get; set; }
public string PrediceOutputFile { get; set; }
public string TransformFilePath { get; set; }
public RangeTransform Transform { get; set; }
}
}

3
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -46,7 +46,7 @@ namespace BotSharp.NLP.Classify
predict.Count = predict.X.Count();
predict.MaxIndex = 200;
RangeTransform transform = RangeTransform.Compute(predict);
RangeTransform transform = options.Transform;
Problem scaled = transform.Scale(predict);
return Prediction.PredictLabelsProbability(options.Model, scaled);
@ -85,6 +85,7 @@ namespace BotSharp.NLP.Classify
param.Weights[i] = 1;
}
var model = Training.Train(scaled, param);
RangeTransform.Write(options.TransformFilePath, transform);
SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model);
Console.Write("Training finished!");
}

4
BotSharp.WebHost/Settings/bot.json
View file

@ -7,8 +7,8 @@
},
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier"
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpCBOWClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpCBOWClassifier"
},
"BotSharpSVMClassifier": {