diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs similarity index 95% rename from BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs rename to BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs index 0f2ae607..fa276a49 100644 --- a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs @@ -11,9 +11,9 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -namespace BotSharp.Core.Engines.Classifiers +namespace BotSharp.Core.Engines.BotSharp { - public class FasttextClassifier : INlpTrain, INlpPredict + public class BotSharpCBOWClassifier : INlpTrain, INlpPredict { public IConfiguration Configuration { get; set; } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs new file mode 100644 index 00000000..87a0ab3f --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs @@ -0,0 +1,85 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP.Classify; +using DotNetToolkit; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Txt2Vec; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpSVMClassifier : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + string predictFileName = Path.Combine(Settings.TempDir, "svm-predict-tempfile.txt"); + File.WriteAllText(predictFileName, doc.Sentences[0].Text); + + NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier(); + Args args = new Args(); + args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); + LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), ""); + + ClassifyOptions classifyOptions = new ClassifyOptions(); + classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model")); + double[][] d = svmClassifier.Predict(featureSet, classifyOptions); + + File.Delete(predictFileName); + + //doc.Sentences[0].Intent = new TextClassificationResult + //{ + // Classifier = "FasttextClassifier", + // Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], + // Confidence = decimal.Parse(output.Split(' ')[1]) + //}; + + return true; + } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + meta.Model = "classification-svm.model"; + string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-svm.parsed.txt"); + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + + List labels = new List(); + List sentences = new List(); + + agent.Corpus.UserSays.ForEach(x =>{ + agent.Intents.ForEach(intent => { + if (intent.Name == x.Intent) + { + labels.Add(agent.Intents.IndexOf(intent).ToString()); + } + }); + sentences.Add(x.Text); + }); + + NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier(); + Args args = new Args(); + args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); + List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels); + ClassifyOptions classifyOptions = new ClassifyOptions(); + classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); + svmClassifier.Train(featureSetList, classifyOptions); + + meta.Meta = new JObject(); + meta.Meta["compiled at"] = "Aug 31, 2018"; + + + return true; + } + } +} diff --git a/BotSharp.Core/Engines/Classifiers/SVMClassifier.cs b/BotSharp.Core/Engines/Classifiers/SVMClassifier.cs deleted file mode 100644 index deb2f013..00000000 --- a/BotSharp.Core/Engines/Classifiers/SVMClassifier.cs +++ /dev/null @@ -1,92 +0,0 @@ -using BotSharp.Core.Abstractions; -using BotSharp.Core.Agents; -using BotSharp.NLP.Classify; -using DotNetToolkit; -using Microsoft.Extensions.Configuration; -using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Txt2Vec; - -namespace BotSharp.Core.Engines.Classifiers -{ - public class SVMClassifier : INlpTrain, INlpPredict - { - public IConfiguration Configuration { get; set; } - public PipeSettings Settings { get; set; } - - public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) - { - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt"); - File.WriteAllText(predictFileName, doc.Sentences[0].Text); - - var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\""); - - File.Delete(predictFileName); - - doc.Sentences[0].Intent = new TextClassificationResult - { - Classifier = "FasttextClassifier", - Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], - Confidence = decimal.Parse(output.Split(' ')[1]) - }; - - return true; - } - - public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) - { - meta.Model = "classification-fasttext.model"; - - string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt"); - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - - // assemble corpus - StringBuilder corpus = new StringBuilder(); - agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}")); - - List labels = new List(); - List sentences = new List(); - - - agent.Corpus.UserSays.ForEach(x =>{ - labels.Add(x.Intent); - sentences.Add(x.Text); - }); - - Dictionary labelDic = new Dictionary(); - int num = 0; - foreach (string label in labels) - { - if (labelDic.ContainsKey(label)) - { - continue; - } - labelDic.Add(label, num++.ToString()); - }; - List labelNums = new List(); - foreach (string label in labels) - { - labelNums.Add(labelDic[label]); - } - NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier(); - Args args = new Args(); - //args.WordDecoderModelFile = Path.Combine(Settings.ModelDir, "wordvec_enu.bin"); - //List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labelNums); - //svmClassifier.Train(featureSetList, new ClassifyOptions(Path.Combine(Settings.ModelDir, "svm_classifier_model"))); - - meta.Meta = new JObject(); - meta.Meta["compiled at"] = "Aug 31, 2018"; - - - return true; - } - } -} diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 80b8e2cb..f6620b7d 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -94,6 +94,10 @@ namespace BotSharp.NLP.Classify { public List Features { get; set; } public string Label { get; set; } + public LabeledFeatureSet() + { + this.Features = new List(); + } } public class Feature diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index a2105962..81651088 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -22,6 +22,7 @@ using System.IO; using System.Linq; using System.Text; using SVM.BotSharp.MachineLearning; +using Txt2Vec; namespace BotSharp.NLP.Classify { @@ -32,17 +33,23 @@ namespace BotSharp.NLP.Classify { public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options) { - Problem test = new Problem(); + + } + + public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options) + { + Problem predict = new Problem(); List featureSets = new List(); featureSets.Add(featureSet); - test.X = GetData(featureSets).ToArray(); - test.Y = GetLabels(featureSets).ToArray(); - test.Count = test.Y.Distinct().Count(); - test.MaxIndex = int.MaxValue; + predict.X = GetData(featureSets).ToArray(); + predict.Y = new double[1]; + predict.Count = predict.X.Count(); + predict.MaxIndex = 200; - RangeTransform transform = RangeTransform.Compute(test); - Problem scaled = transform.Scale(test); - double d = Prediction.Predict(scaled, options.PrediceOutputFile, options.Model, false); + RangeTransform transform = RangeTransform.Compute(predict); + Problem scaled = transform.Scale(predict); + + return Prediction.PredictLabelsProbability(options.Model, scaled); } public void Train(List featureSets, ClassifyOptions options) @@ -50,14 +57,14 @@ namespace BotSharp.NLP.Classify SVMClassifierTrain(featureSets, options); } - public void SVMClassifierTrain (List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) + public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) { // copy test multiclass Model Problem train = new Problem(); train.X = GetData(featureSets).ToArray(); train.Y = GetLabels(featureSets).ToArray(); - train.Count = train.Y.Distinct().Count(); - train.MaxIndex = int.MaxValue; + train.Count = train.X.Count(); + train.MaxIndex = 200;//int.MaxValue; Parameter param = new Parameter(); RangeTransform transform = RangeTransform.Compute(train); @@ -77,11 +84,12 @@ namespace BotSharp.NLP.Classify for (int i = 0; i < numberOfClasses; i++) param.Weights[i] = 1; } - Model model = Training.Train(scaled, param); - Model.Write(options.ModelFilePath, model); + var model = Training.Train(scaled, param); + SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model); + Console.Write("Training finished!"); } - public List GetLabels (List featureSets) + public List GetLabels(List featureSets) { List labels = new List(); foreach (LabeledFeatureSet labelFeatureSet in featureSets) @@ -108,7 +116,43 @@ namespace BotSharp.NLP.Classify } return datas; } + + public List FeatureSetsGenerator(List sentenceVectors, List labels) + { + List res = new List(); + int j; + for (int i = 0; i < labels.Count; i++) + { + string curLabel = labels[i]; + Vec curVec = sentenceVectors[i]; + LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet(); + j = 1; + foreach (double node in curVec.VecNodes) + { + Feature feature = new Feature((j++).ToString(), node.ToString()); + labeledFeatureSet.Features.Add(feature); + } + labeledFeatureSet.Label = curLabel; + res.Add(labeledFeatureSet); + } + + return res; + } + + public LabeledFeatureSet FeatureSetsGenerator(Vec sentenceVectors, String label) + { + LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet(); + int j = 1; + foreach (double node in sentenceVectors.VecNodes) + { + Feature feature = new Feature((j++).ToString(), node.ToString()); + labeledFeatureSet.Features.Add(feature); + } + labeledFeatureSet.Label = label; + + return labeledFeatureSet; + } } - + } diff --git a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs index 286b2b77..dba24342 100644 --- a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs +++ b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs @@ -34,7 +34,7 @@ namespace Txt2Vec model.LoadModel(strModelFileName, bTxtFormat); } - public List Sentence2Vec(List sentences, WeightingScheme weightingScheme = WeightingScheme.TFIDF) + public List Sentence2Vec(List sentences, WeightingScheme weightingScheme = WeightingScheme.AVG) { // Inplementing TF-IDF TFIDFGenerator tfidfGenerator = new TFIDFGenerator(); @@ -88,24 +88,48 @@ namespace Txt2Vec } for (int i = 0; i < vectorList.Count; i++) { - this.dict.Add(sentences[i], vectorList[i]); + if (this.dict.ContainsKey(sentences[i])) + { + continue; + } + else + { + this.dict.Add(sentences[i], vectorList[i]); + } + } return vectorList; } - public Vec SingleSentence2Vec(string sentence) + public Vec SingleSentence2Vec(string sentence, WeightingScheme weightingScheme = WeightingScheme.AVG) { - if (dict.ContainsKey(sentence)) + + Vec sentenceVector = new Vec(); + List sentenceVectorList = new List(); + string[] words = sentence.Split(' '); + foreach (string word in words) { - return this.dict[sentence]; + Vec vec = Word2Vec(word.ToLower()); + sentenceVectorList.Add(vec); } - Vec vec = new Vec(); - int dim = new Encoder().layer1_size; - for (int i = 0; i < dim; i++) + if (weightingScheme == WeightingScheme.AVG) { - vec.VecNodes.Add(1); + int dim = sentenceVectorList[0].VecNodes.Count; + double nodeTotalValue; + for (int k = 0; k < dim; k++) + { + nodeTotalValue = 0; + for (int j = 0; j < sentenceVectorList.Count; j++) + { + Vec curWordVec = sentenceVectorList[j]; + double curNodeVal = curWordVec.VecNodes[k]; + nodeTotalValue += curNodeVal; + } + sentenceVector.VecNodes.Add(nodeTotalValue / dim); + } + } - return vec; + return sentenceVector; } public Vec TFIDFMultiply(List curVecList, List weight) diff --git a/BotSharp.WebHost/Settings/app.json b/BotSharp.WebHost/Settings/app.json index edea0c56..e0736129 100644 --- a/BotSharp.WebHost/Settings/app.json +++ b/BotSharp.WebHost/Settings/app.json @@ -4,6 +4,6 @@ "Version": "0.1.0", "MachineLearning": { - "dataDir": "" + "dataDir": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" } } diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index 584cf6e9..013beac1 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -7,8 +7,12 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier" + }, + + "BotSharpSVMClassifier": { + "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" }, "BotSharpTagger": {