SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Encapsulate NaiveBayes' theorem and add optional smoother function.

Browse source
This commit is contained in:
botsharp2018 2018-09-09 10:32:10 -05:00
parent 1c47465878
commit 553d5c231e
8 changed files with 75 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
BotSharp.Algorithm/Bayes/NaiveBayes.cs
View file

@ -10,20 +10,20 @@ namespace BotSharp.Algorithm.Bayes
/// <summary>
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
/// </summary>
public class NaiveBayes
public class NaiveBayes<Smoother> where Smoother : ISmoother, new()
{
/// <summary>
/// smoothing function
/// </summary>
private Lidstone smoother;
private Smoother smoother;
public List<FeatureFrequencyDistribution> FeatureDist { get; set; }
public List<FeaturesDistribution> FeaturesDist { get; set; }
public List<Probability> LabelDist { get; set; }
public NaiveBayes()
{
smoother = new Lidstone();
smoother = new Smoother();
}
/// <summary>
@ -35,19 +35,25 @@ namespace BotSharp.Algorithm.Bayes
/// <param name="Y">label</param>
/// <param name="featureSet"></param>
/// <returns></returns>
public double PosteriorProb(string Y, LabeledFeatureSet featureSet)
public double PosteriorProb(string Y, List<Feature> features)
{
double prob = 0;
// prior probability
prob = smoother.Log2Prob(LabelDist, Y);
prob = Math.Log(smoother.Prob(LabelDist, Y), 2);
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
featureSet.Features.ForEach(f =>
var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList();
// loop features
for (int x = 0; x < features.Count; x++)
{
var fv = FeatureDist.Find(x => x.Label == Y && x.FeatureName == f.Name).FeatureValues;
prob += smoother.Log2Prob(fv, f.Value);
});
var Xn = features[x];
var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues;
// features are independent, so calculate every feature prob and sum them
prob += Math.Log(smoother.Prob(fv, Xn.Value), 2);
}
return prob;
}
@ -65,7 +71,17 @@ namespace BotSharp.Algorithm.Bayes
}
}
public class FeatureFrequencyDistribution
public class FeaturesWithLabel
{
public List<Feature> Features { get; set; }
public string Label { get; set; }
public FeaturesWithLabel()
{
this.Features = new List<Feature>();
}
}
public class FeaturesDistribution
{
public string Label { get; set; }
@ -78,14 +94,4 @@ namespace BotSharp.Algorithm.Bayes
return $"{Label} {FeatureName} {FeatureValues.Count}";
}
}
public class LabeledFeatureSet
{
public List<Feature> Features { get; set; }
public string Label { get; set; }
public LabeledFeatureSet()
{
this.Features = new List<Feature>();
}
}
}

40
BotSharp.Algorithm/Formulas/Lidstone.cs
View file

@ -29,17 +29,12 @@ namespace BotSharp.Algorithm.Formulas
/// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator.
/// https://en.wikipedia.org/wiki/Additive_smoothing
/// </summary>
public class Lidstone
public class Lidstone : ISmoother
{
/// <summary>
/// α > 0 is the smoothing parameter
/// </summary>
private double _a;
public Lidstone(double alpha = 0.5D)
{
_a = alpha;
}
public double Alpha { get; set; }
/// <summary>
/// Probability
@ -49,6 +44,11 @@ namespace BotSharp.Algorithm.Formulas
/// <returns></returns>
public double Prob(List<Probability> dist, string sample)
{
if(Alpha == 0)
{
Alpha = 0.5D;
}
// observation x = (x1, ..., xd)
var p = dist.Find(f => f.Value == sample);
int x = p == null ? 0 : p.Freq;
@ -58,31 +58,7 @@ namespace BotSharp.Algorithm.Formulas
int _d = dist.Count;
return (x + _a) / (_N + _a * _d);
}
/// <summary>
/// 2 based Log probability
/// </summary>
/// <param name="dist">distribution</param>
/// <param name="sample">sample value</param>
/// <returns></returns>
public double Log2Prob(List<Probability> dist, string sample)
{
var d = Prob(dist, sample);
return Math.Log(d, 2);
}
/// <summary>
/// 10 based Log probability
/// </summary>
/// <param name="dist">distribution</param>
/// <param name="sample">sample value</param>
/// <returns></returns>
public double Log10Prob(List<Probability> dist, string sample)
{
var d = Prob(dist, sample);
return Math.Log(d, 10);
return (x + Alpha) / (_N + Alpha * _d);
}
}
}

11
BotSharp.Algorithm/ISmoother.cs Normal file
View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm
{
public interface ISmoother
{
double Prob(List<Probability> dist, string sample);
}
}

2
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
View file

@ -100,7 +100,7 @@ namespace BotSharp.Core.Engines.BotSharp
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
List<LabeledFeatureSet> featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
var featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
/*
// try using spacy doc2vec

7
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -25,10 +25,7 @@ namespace BotSharp.NLP.Classify
public List<Tuple<string, double>> Classify(Sentence sentence)
{
var classes = _classifier.Classify(new LabeledFeatureSet
{
Features = GetFeatures(sentence.Words)
}, new ClassifyOptions
var classes = _classifier.Classify(GetFeatures(sentence.Words), new ClassifyOptions
{
});
@ -37,7 +34,7 @@ namespace BotSharp.NLP.Classify
public void Train(List<Sentence> sentences)
{
_classifier.Train(sentences.Select(x => new LabeledFeatureSet
_classifier.Train(sentences.Select(x => new FeaturesWithLabel
{
Label = x.Label,
Features = GetFeatures(x.Words)

4
BotSharp.NLP/Classify/IClassifier.cs
View file

@ -7,8 +7,8 @@ namespace BotSharp.NLP.Classify
{
public interface IClassifier
{
void Train(List<LabeledFeatureSet> featureSets, ClassifyOptions options);
void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options);
List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options);
List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options);
}
}

23
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -37,11 +37,11 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class NaiveBayesClassifier : IClassifier
{
private List<FeatureFrequencyDistribution> featureDist;
private List<FeaturesDistribution> featuresDist;
private List<Probability> labelDist;
public void Train(List<LabeledFeatureSet> featureSets, ClassifyOptions options)
public void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options)
{
labelDist = featureSets.GroupBy(x => x.Label)
.Select(x => new Probability
@ -65,7 +65,7 @@ namespace BotSharp.NLP.Classify
Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList()
}).ToList();
featureDist = new List<FeatureFrequencyDistribution>();
featuresDist = new List<FeaturesDistribution>();
labelDist.Select(x => x.Value).ToList().ForEach(label =>
{
@ -82,7 +82,7 @@ namespace BotSharp.NLP.Classify
.OrderBy(f => f.Value)
.ToList();
featureDist.Add(new FeatureFrequencyDistribution
featuresDist.Add(new FeaturesDistribution
{
Label = label,
FeatureName = fName,
@ -92,17 +92,14 @@ namespace BotSharp.NLP.Classify
});
}
public List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
public List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options)
{
var nb = new NaiveBayes();
// calculate prop
var nb = new NaiveBayes<Lidstone>();
nb.LabelDist = labelDist;
nb.FeatureDist = featureDist;
labelDist.ForEach(lf =>
{
// prior probability
lf.Prob = nb.PosteriorProb(lf.Value, featureSet);
});
nb.FeaturesDist = featuresDist;
labelDist.ForEach(lf => lf.Prob = nb.PosteriorProb(lf.Value, features));
// add log
double[] logs = labelDist.Select(x => x.Prob).ToArray();

28
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -32,15 +32,15 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class SVMClassifier : IClassifier
{
public List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
public List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options)
{
return null;
}
public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options)
public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options)
{
Problem predict = new Problem();
List<LabeledFeatureSet> featureSets = new List<LabeledFeatureSet>();
List<FeaturesWithLabel> featureSets = new List<FeaturesWithLabel>();
featureSets.Add(featureSet);
predict.X = GetData(featureSets).ToArray();
predict.Y = new double[1];
@ -53,12 +53,12 @@ namespace BotSharp.NLP.Classify
return Prediction.PredictLabelsProbability(options.Model, scaled);
}
public void Train(List<LabeledFeatureSet> featureSets, ClassifyOptions options)
public void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options)
{
SVMClassifierTrain(featureSets, options);
}
public void SVMClassifierTrain(List<LabeledFeatureSet> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
public void SVMClassifierTrain(List<FeaturesWithLabel> featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
// copy test multiclass Model
Problem train = new Problem();
@ -91,10 +91,10 @@ namespace BotSharp.NLP.Classify
Console.Write("Training finished!");
}
public List<double> GetLabels(List<LabeledFeatureSet> featureSets)
public List<double> GetLabels(List<FeaturesWithLabel> featureSets)
{
List<double> labels = new List<double>();
foreach (LabeledFeatureSet labelFeatureSet in featureSets)
foreach (var labelFeatureSet in featureSets)
{
labels.Add(double.Parse(labelFeatureSet.Label));
}
@ -102,11 +102,11 @@ namespace BotSharp.NLP.Classify
return labels;
}
public List<Node[]> GetData(List<LabeledFeatureSet> featureSets)
public List<Node[]> GetData(List<FeaturesWithLabel> featureSets)
{
List<Node[]> datas = new List<Node[]>();
foreach (LabeledFeatureSet labelFeatureSet in featureSets)
foreach (var labelFeatureSet in featureSets)
{
List<Node> curNodes = new List<Node>();
labelFeatureSet.Features.ForEach(features => {
@ -119,15 +119,15 @@ namespace BotSharp.NLP.Classify
return datas;
}
public List<LabeledFeatureSet> FeatureSetsGenerator(List<Vec> sentenceVectors, List<String> labels)
public List<FeaturesWithLabel> FeatureSetsGenerator(List<Vec> sentenceVectors, List<String> labels)
{
List<LabeledFeatureSet> res = new List<LabeledFeatureSet>();
var res = new List<FeaturesWithLabel>();
int j;
for (int i = 0; i < labels.Count; i++)
{
string curLabel = labels[i];
Vec curVec = sentenceVectors[i];
LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet();
var labeledFeatureSet = new FeaturesWithLabel();
j = 1;
foreach (double node in curVec.VecNodes)
{
@ -141,9 +141,9 @@ namespace BotSharp.NLP.Classify
return res;
}
public LabeledFeatureSet FeatureSetsGenerator(Vec sentenceVectors, String label)
public FeaturesWithLabel FeatureSetsGenerator(Vec sentenceVectors, String label)
{
LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet();
var labeledFeatureSet = new FeaturesWithLabel();
int j = 1;
foreach (double node in sentenceVectors.VecNodes)
{