SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

merge NLP project back to BotSharp and keep on constructing SVM pipeline

Browse source
This commit is contained in:
Bolo Peng 2018-09-04 17:34:41 -05:00
parent 4037160abd
commit a05968292f
8 changed files with 191 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs → BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs
View file

@ -11,9 +11,9 @@ using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.Classifiers
namespace BotSharp.Core.Engines.BotSharp
{
public class FasttextClassifier : INlpTrain, INlpPredict
public class BotSharpCBOWClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }

85
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs Normal file
View file

@ -0,0 +1,85 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP.Classify;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Txt2Vec;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpSVMClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
string predictFileName = Path.Combine(Settings.TempDir, "svm-predict-tempfile.txt");
File.WriteAllText(predictFileName, doc.Sentences[0].Text);
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
double[][] d = svmClassifier.Predict(featureSet, classifyOptions);
File.Delete(predictFileName);
//doc.Sentences[0].Intent = new TextClassificationResult
//{
// Classifier = "FasttextClassifier",
// Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1],
// Confidence = decimal.Parse(output.Split(' ')[1])
//};
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-svm.model";
string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-svm.parsed.txt");
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
List<string> labels = new List<string>();
List<string> sentences = new List<string>();
agent.Corpus.UserSays.ForEach(x =>{
agent.Intents.ForEach(intent => {
if (intent.Name == x.Intent)
{
labels.Add(agent.Intents.IndexOf(intent).ToString());
}
});
sentences.Add(x.Text);
});
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model");
svmClassifier.Train(featureSetList, classifyOptions);
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 31, 2018";
return true;
}
}
}

92
BotSharp.Core/Engines/Classifiers/SVMClassifier.cs
View file

@ -1,92 +0,0 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP.Classify;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Txt2Vec;
namespace BotSharp.Core.Engines.Classifiers
{
public class SVMClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt");
File.WriteAllText(predictFileName, doc.Sentences[0].Text);
var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\"");
File.Delete(predictFileName);
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "FasttextClassifier",
Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1],
Confidence = decimal.Parse(output.Split(' ')[1])
};
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-fasttext.model";
string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
// assemble corpus
StringBuilder corpus = new StringBuilder();
agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}"));
List<string> labels = new List<string>();
List<string> sentences = new List<string>();
agent.Corpus.UserSays.ForEach(x =>{
labels.Add(x.Intent);
sentences.Add(x.Text);
});
Dictionary<string, string> labelDic = new Dictionary<string, string>();
int num = 0;
foreach (string label in labels)
{
if (labelDic.ContainsKey(label))
{
continue;
}
labelDic.Add(label, num++.ToString());
};
List<string> labelNums = new List<string>();
foreach (string label in labels)
{
labelNums.Add(labelDic[label]);
}
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
//args.WordDecoderModelFile = Path.Combine(Settings.ModelDir, "wordvec_enu.bin");
//List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labelNums);
//svmClassifier.Train(featureSetList, new ClassifyOptions(Path.Combine(Settings.ModelDir, "svm_classifier_model")));
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 31, 2018";
return true;
}
}
}

4
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -94,6 +94,10 @@ namespace BotSharp.NLP.Classify
{
public List<Feature> Features { get; set; }
public string Label { get; set; }
public LabeledFeatureSet()
{
this.Features = new List<Feature>();
}
}
public class Feature

74
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -22,6 +22,7 @@ using System.IO;
using System.Linq;
using System.Text;
using SVM.BotSharp.MachineLearning;
using Txt2Vec;
namespace BotSharp.NLP.Classify
{
@ -32,17 +33,23 @@ namespace BotSharp.NLP.Classify
{
public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
{
Problem test = new Problem();
}
public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options)
{
Problem predict = new Problem();
List<LabeledFeatureSet> featureSets = new List<LabeledFeatureSet>();
featureSets.Add(featureSet);
test.X = GetData(featureSets).ToArray();
test.Y = GetLabels(featureSets).ToArray();
test.Count = test.Y.Distinct().Count();
test.MaxIndex = int.MaxValue;
predict.X = GetData(featureSets).ToArray();
predict.Y = new double[1];
predict.Count = predict.X.Count();
predict.MaxIndex = 200;
RangeTransform transform = RangeTransform.Compute(test);
Problem scaled = transform.Scale(test);
double d = Prediction.Predict(scaled, options.PrediceOutputFile, options.Model, false);
RangeTransform transform = RangeTransform.Compute(predict);
Problem scaled = transform.Scale(predict);
return Prediction.PredictLabelsProbability(options.Model, scaled);
}
public void Train(List<LabeledFeatureSet> featureSets, ClassifyOptions options)
@ -50,14 +57,14 @@ namespace BotSharp.NLP.Classify
SVMClassifierTrain(featureSets, options);
}
public void SVMClassifierTrain (List<LabeledFeatureSet> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
public void SVMClassifierTrain(List<LabeledFeatureSet> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
// copy test multiclass Model
Problem train = new Problem();
train.X = GetData(featureSets).ToArray();
train.Y = GetLabels(featureSets).ToArray();
train.Count = train.Y.Distinct().Count();
train.MaxIndex = int.MaxValue;
train.Count = train.X.Count();
train.MaxIndex = 200;//int.MaxValue;
Parameter param = new Parameter();
RangeTransform transform = RangeTransform.Compute(train);
@ -77,11 +84,12 @@ namespace BotSharp.NLP.Classify
for (int i = 0; i < numberOfClasses; i++)
param.Weights[i] = 1;
}
Model model = Training.Train(scaled, param);
Model.Write(options.ModelFilePath, model);
var model = Training.Train(scaled, param);
SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model);
Console.Write("Training finished!");
}
public List<double> GetLabels (List<LabeledFeatureSet> featureSets)
public List<double> GetLabels(List<LabeledFeatureSet> featureSets)
{
List<double> labels = new List<double>();
foreach (LabeledFeatureSet labelFeatureSet in featureSets)
@ -108,7 +116,43 @@ namespace BotSharp.NLP.Classify
}
return datas;
}
public List<LabeledFeatureSet> FeatureSetsGenerator(List<Vec> sentenceVectors, List<String> labels)
{
List<LabeledFeatureSet> res = new List<LabeledFeatureSet>();
int j;
for (int i = 0; i < labels.Count; i++)
{
string curLabel = labels[i];
Vec curVec = sentenceVectors[i];
LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet();
j = 1;
foreach (double node in curVec.VecNodes)
{
Feature feature = new Feature((j++).ToString(), node.ToString());
labeledFeatureSet.Features.Add(feature);
}
labeledFeatureSet.Label = curLabel;
res.Add(labeledFeatureSet);
}
return res;
}
public LabeledFeatureSet FeatureSetsGenerator(Vec sentenceVectors, String label)
{
LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet();
int j = 1;
foreach (double node in sentenceVectors.VecNodes)
{
Feature feature = new Feature((j++).ToString(), node.ToString());
labeledFeatureSet.Features.Add(feature);
}
labeledFeatureSet.Label = label;
return labeledFeatureSet;
}
}
}

44
BotSharp.NLP/Txt2Vec/VectorGenerator.cs
View file

@ -34,7 +34,7 @@ namespace Txt2Vec
model.LoadModel(strModelFileName, bTxtFormat);
}
public List<Vec> Sentence2Vec(List<string> sentences, WeightingScheme weightingScheme = WeightingScheme.TFIDF)
public List<Vec> Sentence2Vec(List<string> sentences, WeightingScheme weightingScheme = WeightingScheme.AVG)
{
// Inplementing TF-IDF
TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
@ -88,24 +88,48 @@ namespace Txt2Vec
}
for (int i = 0; i < vectorList.Count; i++)
{
this.dict.Add(sentences[i], vectorList[i]);
if (this.dict.ContainsKey(sentences[i]))
{
continue;
}
else
{
this.dict.Add(sentences[i], vectorList[i]);
}
}
return vectorList;
}
public Vec SingleSentence2Vec(string sentence)
public Vec SingleSentence2Vec(string sentence, WeightingScheme weightingScheme = WeightingScheme.AVG)
{
if (dict.ContainsKey(sentence))
Vec sentenceVector = new Vec();
List<Vec> sentenceVectorList = new List<Vec>();
string[] words = sentence.Split(' ');
foreach (string word in words)
{
return this.dict[sentence];
Vec vec = Word2Vec(word.ToLower());
sentenceVectorList.Add(vec);
}
Vec vec = new Vec();
int dim = new Encoder().layer1_size;
for (int i = 0; i < dim; i++)
if (weightingScheme == WeightingScheme.AVG)
{
vec.VecNodes.Add(1);
int dim = sentenceVectorList[0].VecNodes.Count;
double nodeTotalValue;
for (int k = 0; k < dim; k++)
{
nodeTotalValue = 0;
for (int j = 0; j < sentenceVectorList.Count; j++)
{
Vec curWordVec = sentenceVectorList[j];
double curNodeVal = curWordVec.VecNodes[k];
nodeTotalValue += curNodeVal;
}
sentenceVector.VecNodes.Add(nodeTotalValue / dim);
}
}
return vec;
return sentenceVector;
}
public Vec TFIDFMultiply(List<Vec> curVecList, List<double> weight)

2
BotSharp.WebHost/Settings/app.json
View file

@ -4,6 +4,6 @@
"Version": "0.1.0",
"MachineLearning": {
"dataDir": ""
"dataDir": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data"
}
}

8
BotSharp.WebHost/Settings/bot.json
View file

@ -7,8 +7,12 @@
},
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier"
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier"
},
"BotSharpSVMClassifier": {
"wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data"
},
"BotSharpTagger": {