SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

developed labeledFeatureSet API and build the SVMClassifier in pipline

Browse source
This commit is contained in:
Bolo Peng 2018-08-31 16:20:12 -05:00
parent d4dc07171f
commit 1d16d9d55d
3 changed files with 106 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
BotSharp.Core/BotSharp.Core.csproj
View file

@ -62,4 +62,8 @@
<Folder Include="Engines\CoreNlp\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\BotSharp.NLP\BotSharp.NLP\BotSharp.NLP.csproj" />
</ItemGroup>
</Project>

92
BotSharp.Core/Engines/Classifiers/SVMClassifier.cs Normal file
View file

@ -0,0 +1,92 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP.Classify;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Txt2Vec;
namespace BotSharp.Core.Engines.Classifiers
{
public class SVMClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt");
File.WriteAllText(predictFileName, doc.Sentences[0].Text);
var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\"");
File.Delete(predictFileName);
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "FasttextClassifier",
Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1],
Confidence = decimal.Parse(output.Split(' ')[1])
};
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-fasttext.model";
string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
// assemble corpus
StringBuilder corpus = new StringBuilder();
agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}"));
List<string> labels = new List<string>();
List<string> sentences = new List<string>();
agent.Corpus.UserSays.ForEach(x =>{
labels.Add(x.Intent);
sentences.Add(x.Text);
});
Dictionary<string, string> labelDic = new Dictionary<string, string>();
int num = 0;
foreach (string label in labels)
{
if (labelDic.ContainsKey(label))
{
continue;
}
labelDic.Add(label, num++.ToString());
};
List<string> labelNums = new List<string>();
foreach (string label in labels)
{
labelNums.Add(labelDic[label]);
}
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
args.WordDecoderModelFile = Path.Combine(Settings.ModelDir, "wordvec_enu.bin");
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labelNums);
svmClassifier.Train(featureSetList, new ClassifyOptions(Path.Combine(Settings.ModelDir, "svm_classifier_model")));
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 31, 2018";
return true;
}
}
}

10
BotSharp.sln
View file

@ -11,6 +11,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.WebHost", "BotShar
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.UnitTest", "BotSharp.Core.UnitTest\BotSharp.Core.UnitTest.csproj", "{A31A6853-DFB8-477D-8F09-8E6E3D166102}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "..\BotSharp.NLP\BotSharp.NLP\BotSharp.NLP.csproj", "{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -51,6 +53,14 @@ Global
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.Build.0 = Release|Any CPU
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|x64.ActiveCfg = Release|x64
{A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|x64.Build.0 = Release|x64
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|Any CPU.Build.0 = Debug|Any CPU
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|x64.ActiveCfg = Debug|x64
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|x64.Build.0 = Debug|x64
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|Any CPU.ActiveCfg = Release|Any CPU
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|Any CPU.Build.0 = Release|Any CPU
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|x64.ActiveCfg = Release|x64
{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE