SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from Oceania2018/master

Browse source 
merge newestcode
This commit is contained in:
geffzhang 2018-09-13 15:10:43 +08:00 committed by GitHub
parent 042cb91d3d a8c5242c7b
commit cc8ad7af25
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 594 additions and 509 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
public class BernoulliNaiveBayes
{
}
}

10
BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
public class GaussianNaiveBayes
{
}
}

119
BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs Normal file
View file

@ -0,0 +1,119 @@
/*
* BotSharp.Algorithm
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
/// <summary>
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
/// </summary>
public class MultinomiaNaiveBayes
{
public List<Probability> LabelDist { get; set; }
public List<Tuple<string, double[]>> FeatureSet { get; set; }
private double alpha { get; set; }
public MultinomiaNaiveBayes(double alpha = 0.5)
{
this.alpha = alpha;
}
/// <summary>
/// prior probability
/// </summary>
/// <param name="Y"></param>
/// <returns></returns>
public double CalPriorProb(string Y)
{
int N = FeatureSet.Count;
int k = LabelDist.Count;
int Nyk = LabelDist.First(x => x.Value == Y).Freq;
return (Nyk + alpha) / (N + k * alpha);
}
public double CalCondProb(int x, string Y, double feature)
{
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList();
var matrix = ConstructMatrix(featuresIfY);
int freq = 0;
for (int y = 0; y < featuresIfY.Count; y++)
{
if (matrix[y, x] == feature)
{
freq++;
}
}
int Nyk = featuresIfY.Count;
int n = featuresIfY.Count;
int Nykx = freq;
return Math.Log((Nykx + alpha) / (Nyk + n * alpha));
}
/// <summary>
/// calculate posterior probability P(Y|X)
/// X is feature set, Y is label
/// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
/// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
/// </summary>
public double CalPosteriorProb(string Y, double[] features, double priorProb, Dictionary<string, double> condProbDictionary)
{
int featureCount = features.Length;
double postProb = priorProb;
// loop features
for (int x = 0; x < featureCount; x++)
{
string key = $"{Y} f{x} {features[x]}";
postProb += condProbDictionary[key];
}
return Math.Pow(2, postProb);
}
private double[,] ConstructMatrix(List<Tuple<string, double[]>> featuresIfY)
{
var featureCount = featuresIfY[0].Item2.Length;
double[,] matrix = new double[featuresIfY.Count, featureCount];
for (int y = 0; y < featuresIfY.Count; y++)
{
for (int x = 0; x < featureCount; x++)
{
matrix[y, x] = featuresIfY[y].Item2[x];
}
}
return matrix;
}
}
}

16
BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs Normal file
View file

@ -0,0 +1,16 @@
using BotSharp.Algorithm.Statistics;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
public class MultinomiaNaiveBayesModel
{
public List<Probability> LabelDist { get; set; }
public Dictionary<string, double> CondProbDictionary { get; set; }
public List<String> Values { get; set; }
}
}

65
BotSharp.Algorithm/Bayes/NaiveBayes.cs
View file

@ -1,65 +0,0 @@
using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
/// <summary>
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
/// </summary>
public class NaiveBayes<Estimator> where Estimator : IEstimator, new()
{
/// <summary>
/// smoothing function
/// </summary>
private Estimator estomator;
public List<FeaturesDistribution> FeaturesDist { get; set; }
public List<Probability> LabelDist { get; set; }
public NaiveBayes()
{
estomator = new Estimator();
}
/// <summary>
/// calculate posterior probability P(Y|X)
/// X is feature set, Y is label
/// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
/// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
/// </summary>
/// <param name="Y">label</param>
/// <param name="featureSet"></param>
/// <returns></returns>
public double PosteriorProb(string Y, List<Feature> features)
{
double prob = 0;
// prior probability
prob = Math.Log(estomator.Prob(LabelDist, Y), 2);
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList();
// loop features
for (int x = 0; x < features.Count; x++)
{
var Xn = features[x];
var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues;
if(fv != null)
{
// features are independent, so calculate every feature prob and sum them
prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
}
}
return prob;
}
}
}

25
BotSharp.Algorithm/Estimators/Lidstone.cs → BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs
View file

@ -31,10 +31,12 @@ namespace BotSharp.Algorithm.Estimators
/// https://en.wikipedia.org/wiki/Additive_smoothing
/// Used as Multinomial Naive Bayes
/// </summary>
public class Lidstone : IEstimator
public class AdditiveSmoothing : IEstimator
{
/// <summary>
/// α > 0 is the smoothing parameter
/// 1 > α > 0 is the smoothing parameter is Lidstone
/// α = 1 is Laplace
/// α = 0 no smoothing
/// </summary>
public double Alpha { get; set; }
@ -62,5 +64,24 @@ namespace BotSharp.Algorithm.Estimators
return (x + Alpha) / (_N + Alpha * _d);
}
public double Prob(List<Tuple<string, double>> dist, string sample)
{
if (Alpha == 0)
{
Alpha = 0.5D;
}
// observation x = (x1, ..., xd)
var p = dist.Find(f => f.Item1 == sample);
double x = p == null ? 0D : p.Item2;
// N trials
double _N = dist.Sum(f => f.Item2);
int _d = dist.Count;
return (x + Alpha) / (_N + Alpha * _d);
}
}
}

76
BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs Normal file
View file

@ -0,0 +1,76 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Classify;
using BotSharp.NLP.Txt2Vec;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpNBayesClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-nb.model";
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
var options = new ClassifyOptions
{
ModelFilePath = modelFileName
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var sentences = doc.Sentences.Select(x => new Sentence
{
Label = x.Intent.Label,
Text = x.Text,
Words = x.Tokens
}).ToList();
classifier.Train(sentences);
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Sep 12, 2018";
return true;
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model)
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var sentence = doc.Sentences.Select(s => new Sentence
{
Text = s.Text,
Words = s.Tokens
}).First();
var result = classifier.Classify(sentence);
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "BotSharpNBayesClassifier",
Label = result.First().Item1,
Confidence = (decimal)result.First().Item2
};
return true;
}
}
}

2
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
View file

@ -129,7 +129,7 @@ namespace BotSharp.Core.Engines.BotSharp
ClassifyOptions classifyOptions = new ClassifyOptions();
classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model");
classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data");
svmClassifier.Train(featureSetList, classifyOptions);
// svmClassifier.Train(featureSetList, classifyOptions);
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 31, 2018";

10
BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Intents;
using BotSharp.NLP;
using BotSharp.NLP.Tokenize;
using Microsoft.Extensions.Configuration;
@ -14,14 +15,12 @@ namespace BotSharp.Core.Engines.BotSharp
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
private TokenizerFactory<RegexTokenizer> _tokenizer;
private TokenizerFactory<TreebankTokenizer> _tokenizer;
public BotSharpTokenizer()
{
_tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
_tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC,
SpecialWords = new List<string> { "'s" }
}, SupportedLanguage.English);
}
@ -48,7 +47,8 @@ namespace BotSharp.Core.Engines.BotSharp
doc.Sentences.Add(new NlpDocSentence
{
Tokens = _tokenizer.Tokenize(say.Text),
Text = say.Text
Text = say.Text,
Intent = new TextClassificationResult { Label = say.Intent }
});
});

17
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -36,32 +36,31 @@ namespace BotSharp.NLP.UnitTest
sentences.Shuffle();
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
encoder.EncodeAll();
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.9M);
classifier.TrainInVector(dataset.Item1);
var dataset = sentences.Split(1M);
classifier.Train(dataset.Item1);
int correct = 0;
dataset.Item1.ToList().ForEach(td =>
int total = 0;
dataset.Item1.ForEach(td =>
{
var classes = classifier.Classify(td);
if (td.Label == classes[0].Item1)
{
correct++;
}
total++;
});
var accuracy = (float)correct / dataset.Item2.Count;
var accuracy = (float)correct / total;
Assert.IsTrue(accuracy > 0.5);
}
[TestMethod]

5
BotSharp.NLP.UnitTest/SVMClassifierTest.cs
View file

@ -1,6 +1,5 @@
using BotSharp.NLP.Classify;
using BotSharp.NLP.Corpus;
using BotSharp.NLP.Models.TF_IDF;
using BotSharp.NLP.Tokenize;
using Microsoft.Extensions.Configuration;
using Microsoft.VisualStudio.TestTools.UnitTesting;
@ -29,8 +28,8 @@ namespace BotSharp.NLP.UnitTest
"see you Bolo",
"byebye Haiping"
};
TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
List<List<double>> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents);
/*TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
List<List<double>> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(documents);*/
}
[TestMethod]

31
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -27,37 +27,24 @@ namespace BotSharp.NLP.Classify
featureExtractor = new IFeatureExtractor();
}
public void Train(List<Sentence> sentences)
{
_classifier.Train(sentences, _options);
_classifier.SaveModel(_options);
}
public List<Tuple<string, double>> Classify(Sentence sentence)
{
var options = new ClassifyOptions
{
ModelFilePath = _options.ModelFilePath
};
var features = featureExtractor.GetFeatures(sentence.Words);
_classifier.LoadModel(options);
var classes = _classifier.Classify(features, options);
var classes = _classifier.Classify(sentence, options);
return classes.OrderByDescending(x => x.Item2).ToList();
}
public void Train(List<Sentence> sentences)
{
var sents = sentences.Select(x => new FeaturesWithLabel
{
Label = x.Label,
Features = featureExtractor.GetFeatures(x.Words)
}).ToList();
_classifier.Train(sents, _options);
}
public void TrainInVector(List<Sentence> sentences)
{
var vectors = new List<Tuple<string, double[]>>();
var sents = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
_classifier.Train(sents, _options);
}
}
}

16
BotSharp.NLP/Classify/IClassifier.cs
View file

@ -7,23 +7,23 @@ namespace BotSharp.NLP.Classify
{
public interface IClassifier
{
void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options);
List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options);
/// <summary>
/// Training by feature vector
/// </summary>
/// <param name="featureSets"></param>
/// <param name="sentences"></param>
/// <param name="options"></param>
void Train(List<Tuple<string, double[]>> featureSets, ClassifyOptions options);
void Train(List<Sentence> sentences, ClassifyOptions options);
/// <summary>
/// Predict by feature vector
/// </summary>
/// <param name="features"></param>
/// <param name="sentence"></param>
/// <param name="options"></param>
/// <returns></returns>
List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options);
List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options);
String SaveModel(ClassifyOptions options);
Object LoadModel(ClassifyOptions options);
}
}

203
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -22,6 +22,9 @@ using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Extensions;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using BotSharp.NLP.Featuring;
using BotSharp.NLP.Txt2Vec;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
@ -40,119 +43,123 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class NaiveBayesClassifier : IClassifier
{
private List<FeaturesDistribution> featuresDist;
private List<Probability> labelDist;
public void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options)
private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes();
private Dictionary<string, double> condProbDictionary = new Dictionary<string, double>();
private List<string> words;
private double[] features = new double[] { 0, 1 };
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
labelDist = featureSets.GroupBy(x => x.Label)
.Select(x => new Probability
{
Value = x.Key,
Freq = x.Count()
})
.ToList();
var tfidf = new TfIdfFeatureExtractor();
tfidf.Sentences = sentences;
tfidf.CalBasedOnCategory();
var keyWords = tfidf.Features();
string keywords2 = String.Join(",", keyWords.ToArray());
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
words = encoder.EncodeAll();
var fNames = new List<string>();
var featureSets = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name)));
fNames = fNames.OrderBy(x => x).Distinct().ToList();
var featureValues = new Dictionary<string, List<Feature>>();
for (int i = 0; i < featureSets.Count; i++)
{
var fs = featureSets[i];
featureValues[fs.Label] = new List<Feature>();
fNames.ForEach(fn =>
{
Feature feature = null;
for (int j = 0; j < fs.Features.Count; j++)
{
if (fs.Features[j].Name == fn)
{
feature = fs.Features[j];
break;
}
}
var fv = new Feature(fn, feature == null ? "False" : feature.Value);
featureValues[fs.Label].Add(fv);
});
}
featuresDist = new List<FeaturesDistribution>();
labelDist.Select(x => x.Value).ToList().ForEach(label =>
{
var fSets = featureValues[label];
fNames.ForEach(fName =>
{
var fsv = fSets.Where(fs => fs.Name == fName)
.GroupBy(fs => fs.Value)
.Select(fs => new Probability
{
Value = fs.Key,
Freq = fs.Count()
})
.OrderBy(fs => fs.Value)
.ToList();
featuresDist.Add(new FeaturesDistribution
{
Label = label,
FeatureName = fName,
FeatureValues = fsv
});
});
});
}
public List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options)
{
// calculate prop
var nb = new NaiveBayes<Lidstone>();
nb.LabelDist = labelDist;
nb.FeaturesDist = featuresDist;
Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features));
// add log
double[] logs = labelDist.Select(x => x.Prob).ToArray();
var sumLogs = logs.Reduce((log1, next) =>
{
double min = log1;
if (next < log1)
{
min = next;
}
return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2);
});
labelDist.ForEach(d => d.Prob -= sumLogs);
return labelDist.Select(x => new Tuple<string, double>(x.Value, x.Prob)).ToList();
}
public void Train(List<Tuple<string, double[]>> featureSets, ClassifyOptions options)
{
labelDist = featureSets.GroupBy(x => x.Item1)
.Select(x => new Probability
{
Value = x.Key,
Freq = x.Count()
})
.OrderBy(x => x.Value)
.ToList();
nb.LabelDist = labelDist;
nb.FeatureSet = featureSets;
// calculate prior prob
labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value));
// calculate posterior prob
// loop features
var featureCount = nb.FeatureSet[0].Item2.Length;
labelDist.ForEach(label =>
{
for (int x = 0; x < featureCount; x++)
{
for (int v = 0; v < features.Length; v++)
{
string key = $"{label.Value} f{x} {features[v]}";
condProbDictionary[key] = nb.CalCondProb(x, label.Value, features[v]);
}
}
});
}
public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
{
throw new NotImplementedException();
var encoder = new OneHotEncoder();
encoder.Words = words;
encoder.Encode(sentence);
var results = new List<Tuple<string, double>>();
// calculate prop
labelDist.ForEach(lf =>
{
var prob = nb.CalPosteriorProb(lf.Value, sentence.Vector, lf.Prob, condProbDictionary);
results.Add(new Tuple<string, double>(lf.Value, prob));
});
/*Parallel.ForEach(labelDist, (lf) =>
{
nb.Y = lf.Value;
lf.Prob = nb.PosteriorProb();
});*/
return results;
}
public string SaveModel(ClassifyOptions options)
{
// save the model
var model = new MultinomiaNaiveBayesModel
{
LabelDist = labelDist,
CondProbDictionary = condProbDictionary,
Values = words
};
//save the file
using (var bw = new BinaryWriter(new FileStream(options.ModelFilePath, FileMode.Create)))
{
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model));
bw.Write(bytes);
}
return options.ModelFilePath;
}
public Object LoadModel(ClassifyOptions options)
{
string json = String.Empty;
//read the file
using (var br = new BinaryReader(new FileStream(options.ModelFilePath, FileMode.Open)))
{
byte[] bytes = br.ReadBytes((int)br.BaseStream.Length);
json = Encoding.UTF8.GetString(bytes);
}
var model = JsonConvert.DeserializeObject<MultinomiaNaiveBayesModel>(json);
labelDist = model.LabelDist;
condProbDictionary = model.CondProbDictionary;
words = model.Values;
return model;
}
}

20
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -32,11 +32,6 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class SVMClassifier : IClassifier
{
public List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options)
{
return null;
}
public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options)
{
Problem predict = new Problem();
@ -53,9 +48,14 @@ namespace BotSharp.NLP.Classify
return Prediction.PredictLabelsProbability(options.Model, scaled);
}
public void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options)
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
SVMClassifierTrain(featureSets, options);
// SVMClassifierTrain(featureSets, options);
}
public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
{
throw new NotImplementedException();
}
public void SVMClassifierTrain(List<FeaturesWithLabel> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
@ -155,16 +155,14 @@ namespace BotSharp.NLP.Classify
return labeledFeatureSet;
}
public void Train(List<Tuple<string, double[]>> featureSets, ClassifyOptions options)
public string SaveModel(ClassifyOptions options)
{
throw new NotImplementedException();
}
public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
object IClassifier.LoadModel(ClassifyOptions options)
{
throw new NotImplementedException();
}
}
}

10
BotSharp.NLP/Featuring/IFeatureExtractor.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.NLP.Featuring
{
public interface IFeatureExtractor
{
}
}

155
BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs Normal file
View file

@ -0,0 +1,155 @@
/*
* BotSharp.NLP Library
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Featuring
{
public class TfIdfFeatureExtractor : IFeatureExtractor
{
public List<Sentence> Sentences { get; set; }
private List<Tuple<String, double>> tfs;
private List<string> Categories { get; set; }
public void Extract(Sentence sentence)
{
}
public List<string> Features()
{
var tfs2 = tfs.OrderByDescending(x => x.Item2)
.Select(x => x.Item1)
.Distinct()
.Take(Sentences.Count / Categories.Count)
.ToList();
return tfs2;
}
public void CalBasedOnSentence()
{
Categories = Sentences.Select(x => x.Label).Distinct().ToList();
tfs = new List<Tuple<String, double>>();
Sentences.ForEach(sent =>
{
sent.Words.ForEach(word =>
{
// TF
int c1 = sent.Words.Count(x => x.Lemma == word.Lemma);
double tf = (c1 + 1.0) / sent.Words.Count();
// IDF
var c2 = Sentences.Count(s => s.Words.Select(x => x.Lemma).Contains(word.Lemma));
double idf = Math.Log(Sentences.Count / (c2 + 1.0));
word.Vector = tf * idf;
tfs.Add(new Tuple<string, double>(word.Lemma, word.Vector));
});
});
}
public void CalBasedOnCategory()
{
tfs = new List<Tuple<String, double>>();
Categories = Sentences.Select(x => x.Label).Distinct().ToList();
Categories.ForEach(label =>
{
var allTokens = new List<Token>();
Sentences.Where(x => x.Label == label)
.ToList()
.ForEach(s => allTokens.AddRange(s.Words));
allTokens.Select(x => x.Lemma).Distinct()
.ToList()
.ForEach(word =>
{
// TF
int c1 = allTokens.Count(x => x.Lemma == word);
double tf = (c1 + 1.0) / allTokens.Count();
// IDF
var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word))
.GroupBy(x => x.Label).Count();
double idf = Math.Log(Categories.Count / (c2 + 1.0));
tfs.Add(new Tuple<string, double>(word, tf * idf));
});
});
}
/// <summary>
/// Normalizes a TF*IDF array of vectors using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors">List<List<double>></param>
/// <returns>List<List<double>></returns>
public static List<List<double>> Normalize(List<List<double>> vectors)
{
// Normalize the vectors using L2-Norm.
List<List<double>> normalizedVectors = new List<List<double>>();
foreach (var vector in vectors)
{
var normalized = Normalize(vector);
normalizedVectors.Add(normalized);
}
return normalizedVectors;
}
/// <summary>
/// Normalizes a TF*IDF vector using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors"> List<double> </param>
/// <returns> List<double> </returns>
public static List<double> Normalize(List<double> vector)
{
List<double> result = new List<double>();
double sumSquared = 0;
foreach (var value in vector)
{
sumSquared += value * value;
}
double SqrtSumSquared = Math.Sqrt(sumSquared);
foreach (var value in vector)
{
// L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
result.Add(value / SqrtSumSquared);
}
return result;
}
}
}

232
BotSharp.NLP/Models/TF-IDF/TFIDF.cs
View file

@ -1,232 +0,0 @@
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Models.TF_IDF
{
/// <summary>
/// Copyright (c) 2018 Bo Peng
///
/// Permission is hereby granted, free of charge, to any person obtaining
/// a copy of this software and associated documentation files (the
/// "Software"), to deal in the Software without restriction, including
/// without limitation the rights to use, copy, modify, merge, publish,
/// distribute, sublicense, and/or sell copies of the Software, and to
/// permit persons to whom the Software is furnished to do so, subject to
/// the following conditions:
///
/// The above copyright notice and this permission notice shall be
/// included in all copies or substantial portions of the Software.
/// </summary>
public class TFIDF
{
List<string> vocabulary { get; set; }
public TFIDF()
{
}
/// <summary>
/// Document vocabulary, containing each word's IDF value.
/// </summary>
private static Dictionary<string, double> _vocabularyIDF = new Dictionary<string, double>();
public static List<List<double>> GetTFIDFWeightsVectors(string[] documents, int vocabularyThreshold = 1)
{
List<List<string>> stemmedDocs;
List<string> vocabulary;
// Get the vocabulary and stem the documents at the same time.
vocabulary = GetVocabulary(documents, out stemmedDocs, vocabularyThreshold);
if (_vocabularyIDF.Count == 0)
{
// Calculate the IDF for each vocabulary term.
foreach (var term in vocabulary)
{
double numberOfDocsContainingTerm = stemmedDocs.Where(d => d.Contains(term)).Count();
_vocabularyIDF[term] = Math.Log((double)stemmedDocs.Count / ((double)1 + numberOfDocsContainingTerm));
}
}
// Transform each document into a vector of tfidf values.
List<List<double>> vectors = new List<List<double>>();
foreach (var doc in stemmedDocs)
{
List<double> vector = new List<double>();
foreach (string word in doc)
{
double tf = doc.Where(d => d == word).Count();
double tfidf = tf * _vocabularyIDF[word];
vector.Add(tfidf);
}
vectors.Add(vector);
}
return vectors;
}
/// <summary>
/// Normalizes a TF*IDF array of vectors using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors">List<List<double>></param>
/// <returns>List<List<double>></returns>
public static List<List<double>> Normalize(List<List<double>> vectors)
{
// Normalize the vectors using L2-Norm.
List<List<double>> normalizedVectors = new List<List<double>>();
foreach (var vector in vectors)
{
var normalized = Normalize(vector);
normalizedVectors.Add(normalized);
}
return normalizedVectors;
}
/// <summary>
/// Normalizes a TF*IDF vector using L2-Norm.
/// Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
/// </summary>
/// <param name="vectors"> List<double> </param>
/// <returns> List<double> </returns>
public static List<double> Normalize(List<double> vector)
{
List<double> result = new List<double>();
double sumSquared = 0;
foreach (var value in vector)
{
sumSquared += value * value;
}
double SqrtSumSquared = Math.Sqrt(sumSquared);
foreach (var value in vector)
{
// L2-norm: Xi = Xi / Sqrt(X0^2 + X1^2 + .. + Xn^2)
result.Add(value / SqrtSumSquared);
}
return result;
}
/// <summary>
/// Saves the TFIDF vocabulary to disk.
/// </summary>
/// <param name="filePath">File path</param>
public static void Save(string filePath = "vocabulary.dat")
{
// Save result to disk.
using (FileStream fs = new FileStream(filePath, FileMode.Create))
{
BinaryFormatter formatter = new BinaryFormatter();
formatter.Serialize(fs, _vocabularyIDF);
}
}
/// <summary>
/// Loads the TFIDF vocabulary from disk.
/// </summary>
/// <param name="filePath">File path</param>
public static void Load(string filePath = "vocabulary.dat")
{
// Load from disk.
using (FileStream fs = new FileStream(filePath, FileMode.Open))
{
BinaryFormatter formatter = new BinaryFormatter();
_vocabularyIDF = (Dictionary<string, double>)formatter.Deserialize(fs);
}
}
/// <summary>
/// Parses and tokenizes a list of documents, returning a vocabulary of words.
/// </summary>
/// <param name="docs">string[]</param>
/// <param name="stemmedDocs">List of List of string</param>
/// <returns>Vocabulary (list of strings)</returns>
private static List<string> GetVocabulary(string[] docs, out List<List<string>> stemmedDocs, int vocabularyThreshold)
{
List<string> vocabulary = new List<string>();
Dictionary<string, int> wordCountList = new Dictionary<string, int>();
stemmedDocs = new List<List<string>>();
int docIndex = 0;
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
{
Pattern = RegexTokenizer.WHITE_SPACE
}, SupportedLanguage.English);
foreach (var doc in docs)
{
List<string> stemmedDoc = new List<string>();
docIndex++;
if (docIndex % 100 == 0)
{
Console.WriteLine("Processing " + docIndex + "/" + docs.Length);
}
List<Token> tokens = tokenizer.Tokenize(doc);
List<string> list = new List<string>();
tokenizer.Tokenize(doc).ForEach( token => {
list.Add(token.Text.ToLower());
});
string[] parts2 = list.ToArray();
//string[] parts2 = Tokenize(doc);
List<string> words = new List<string>();
foreach (string part in parts2)
{
// Strip non-alphanumeric characters.
string stripped = Regex.Replace(part, "[^a-zA-Z0-9]", "");
try
{
var english = new EnglishWord(stripped);
string stem = english.Stem;
words.Add(stem);
if (stem.Length > 0)
{
// Build the word count list.
if (wordCountList.ContainsKey(stem))
{
wordCountList[stem]++;
}
else
{
wordCountList.Add(stem, 0);
}
stemmedDoc.Add(stem);
}
}
catch
{
}
}
stemmedDocs.Add(stemmedDoc);
}
// Get the top words.
var vocabList = wordCountList.Where(w => w.Value >= vocabularyThreshold);
foreach (var item in vocabList)
{
vocabulary.Add(item.Key);
}
return vocabulary;
}
}
public class EnglishWord
{
public EnglishWord(string input)
{
this.Original = input;
this.Stem = input;
this.Length = input.Length;
}
public string Stem { get; set; }
public string Original { get; }
public int Length { get; }
}
}

30
BotSharp.NLP/Models/TF-IDF/TFIDFGenerator.cs
View file

@ -1,30 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.NLP.Models.TF_IDF
{
/// <summary>
/// Copyright (c) 2018 Bo Peng
///
/// Permission is hereby granted, free of charge, to any person obtaining
/// a copy of this software and associated documentation files (the
/// "Software"), to deal in the Software without restriction, including
/// without limitation the rights to use, copy, modify, merge, publish,
/// distribute, sublicense, and/or sell copies of the Software, and to
/// permit persons to whom the Software is furnished to do so, subject to
/// the following conditions:
///
/// The above copyright notice and this permission notice shall be
/// included in all copies or substantial portions of the Software.
/// </summary>
public class TFIDFGenerator
{
public List<List<double>> TFIDFWeightVectorsForSentences(string[]documents)
{
List<List<double>> res = TFIDF.GetTFIDFWeightsVectors(documents, 0);
res = TFIDF.Normalize(res);
return res;
}
}
}

2
BotSharp.NLP/Tokenize/Token.cs
View file

@ -67,5 +67,7 @@ namespace BotSharp.NLP.Tokenize
{
return $"{Text} {Start} {Pos}";
}
public double Vector { get; set; }
}
}

5
BotSharp.NLP/Tokenize/TokenizerFactory.cs
View file

@ -29,7 +29,9 @@ namespace BotSharp.NLP.Tokenize
public List<Token> Tokenize(string sentence)
{
return _tokenizer.Tokenize(sentence, _options);
var tokens = _tokenizer.Tokenize(sentence, _options);
tokens.ForEach(x => x.Lemma = x.Text.ToLower());
return tokens;
}
public List<Sentence> Tokenize(List<String> sentences)
@ -39,6 +41,7 @@ namespace BotSharp.NLP.Tokenize
Parallel.ForEach(sents, (sentence) =>
{
sentence.Words = Tokenize(sentence.Text);
sentence.Words.ForEach(x => x.Lemma = x.Text.ToLower());
});
return sents;

30
BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
View file

@ -15,17 +15,17 @@ namespace BotSharp.NLP.Txt2Vec
{
public List<Sentence> Sentences { get; set; }
private List<string> words;
public List<string> Words { get; set; }
public void Encode(Sentence sentence)
{
InitDictionary();
var vector = words.Select(x => 0D).ToArray();
var vector = Words.Select(x => 0D).ToArray();
sentence.Words.ForEach(w =>
{
int index = words.IndexOf(w.Text.ToLower());
int index = Words.IndexOf(w.Lemma.ToLower());
if(index > 0)
{
vector[index] = 1;
@ -35,26 +35,24 @@ namespace BotSharp.NLP.Txt2Vec
sentence.Vector = vector;
}
public void EncodeAll()
public List<string> EncodeAll()
{
InitDictionary();
Parallel.ForEach(Sentences, sent =>
{
Encode(sent);
});
Sentences.ForEach(sent => Encode(sent));
//Parallel.ForEach(Sentences, sent => Encode(sent));
return Words;
}
private void InitDictionary()
private List<string> InitDictionary()
{
if (words == null)
if (Words == null)
{
words = new List<string>();
Sentences.ForEach(x =>
{
words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
});
words = words.Distinct().OrderBy(x => x).ToList();
// Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move".Split(',').ToList();
}
return Words;
}
}
}

5
BotSharp.NLP/Txt2Vec/VectorGenerator.cs
View file

@ -5,7 +5,6 @@ using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Threading;
using BotSharp.NLP.Models.TF_IDF;
//using AdvUtils;
namespace Txt2Vec
@ -37,8 +36,8 @@ namespace Txt2Vec
public List<Vec> Sentence2Vec(List<string> sentences, WeightingScheme weightingScheme = WeightingScheme.AVG)
{
// Inplementing TF-IDF
TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
List<List<double>> weights = tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray());
// TFIDFGenerator tfidfGenerator = new TFIDFGenerator();
List<List<double>> weights = null;// tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray());
List<List<Vec>> matixList = new List<List<Vec>>();

9
Settings/bot.json
View file

@ -7,10 +7,13 @@
},
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier"
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier"
},
"BotSharpNBayesClassifier": {
},
"BotSharpSVMClassifier": {
"wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data"
},