From 1d16d9d55df81548ecb1dcb25526f4b86dbb79ec Mon Sep 17 00:00:00 2001 From: Bolo Peng Date: Fri, 31 Aug 2018 16:20:12 -0500 Subject: [PATCH] developed labeledFeatureSet API and build the SVMClassifier in pipline --- BotSharp.Core/BotSharp.Core.csproj | 4 + .../Engines/Classifiers/SVMClassifier.cs | 92 +++++++++++++++++++ BotSharp.sln | 10 ++ 3 files changed, 106 insertions(+) create mode 100644 BotSharp.Core/Engines/Classifiers/SVMClassifier.cs diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index 2c5cf3ad..4ff12b5f 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -62,4 +62,8 @@ + + + + diff --git a/BotSharp.Core/Engines/Classifiers/SVMClassifier.cs b/BotSharp.Core/Engines/Classifiers/SVMClassifier.cs new file mode 100644 index 00000000..db473d95 --- /dev/null +++ b/BotSharp.Core/Engines/Classifiers/SVMClassifier.cs @@ -0,0 +1,92 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP.Classify; +using DotNetToolkit; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Txt2Vec; + +namespace BotSharp.Core.Engines.Classifiers +{ + public class SVMClassifier : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt"); + File.WriteAllText(predictFileName, doc.Sentences[0].Text); + + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\""); + + File.Delete(predictFileName); + + doc.Sentences[0].Intent = new TextClassificationResult + { + Classifier = "FasttextClassifier", + Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], + Confidence = decimal.Parse(output.Split(' ')[1]) + }; + + return true; + } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + meta.Model = "classification-fasttext.model"; + + string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt"); + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + + // assemble corpus + StringBuilder corpus = new StringBuilder(); + agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}")); + + List labels = new List(); + List sentences = new List(); + + + agent.Corpus.UserSays.ForEach(x =>{ + labels.Add(x.Intent); + sentences.Add(x.Text); + }); + + Dictionary labelDic = new Dictionary(); + int num = 0; + foreach (string label in labels) + { + if (labelDic.ContainsKey(label)) + { + continue; + } + labelDic.Add(label, num++.ToString()); + }; + List labelNums = new List(); + foreach (string label in labels) + { + labelNums.Add(labelDic[label]); + } + NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier(); + Args args = new Args(); + args.WordDecoderModelFile = Path.Combine(Settings.ModelDir, "wordvec_enu.bin"); + List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labelNums); + svmClassifier.Train(featureSetList, new ClassifyOptions(Path.Combine(Settings.ModelDir, "svm_classifier_model"))); + + meta.Meta = new JObject(); + meta.Meta["compiled at"] = "Aug 31, 2018"; + + + return true; + } + } +} diff --git a/BotSharp.sln b/BotSharp.sln index 0e04dc70..fb940ac0 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -11,6 +11,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.WebHost", "BotShar EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.UnitTest", "BotSharp.Core.UnitTest\BotSharp.Core.UnitTest.csproj", "{A31A6853-DFB8-477D-8F09-8E6E3D166102}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "..\BotSharp.NLP\BotSharp.NLP\BotSharp.NLP.csproj", "{ECAC1C50-5D12-496F-A5CC-7CF006EA9105}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +53,14 @@ Global {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|Any CPU.Build.0 = Release|Any CPU {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|x64.ActiveCfg = Release|x64 {A31A6853-DFB8-477D-8F09-8E6E3D166102}.Release|x64.Build.0 = Release|x64 + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|x64.ActiveCfg = Debug|x64 + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Debug|x64.Build.0 = Debug|x64 + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|Any CPU.Build.0 = Release|Any CPU + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|x64.ActiveCfg = Release|x64 + {ECAC1C50-5D12-496F-A5CC-7CF006EA9105}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE