diff --git a/BotSharp.Algorithm/Bayes/NaiveBayes.cs b/BotSharp.Algorithm/Bayes/NaiveBayes.cs
index 764b14f9..e2217130 100644
--- a/BotSharp.Algorithm/Bayes/NaiveBayes.cs
+++ b/BotSharp.Algorithm/Bayes/NaiveBayes.cs
@@ -10,20 +10,20 @@ namespace BotSharp.Algorithm.Bayes
///
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
///
- public class NaiveBayes
+ public class NaiveBayes where Smoother : ISmoother, new()
{
///
/// smoothing function
///
- private Lidstone smoother;
+ private Smoother smoother;
- public List FeatureDist { get; set; }
+ public List FeaturesDist { get; set; }
public List LabelDist { get; set; }
public NaiveBayes()
{
- smoother = new Lidstone();
+ smoother = new Smoother();
}
///
@@ -35,19 +35,25 @@ namespace BotSharp.Algorithm.Bayes
/// label
///
///
- public double PosteriorProb(string Y, LabeledFeatureSet featureSet)
+ public double PosteriorProb(string Y, List features)
{
double prob = 0;
// prior probability
- prob = smoother.Log2Prob(LabelDist, Y);
+ prob = Math.Log(smoother.Prob(LabelDist, Y), 2);
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
- featureSet.Features.ForEach(f =>
+ var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList();
+
+ // loop features
+ for (int x = 0; x < features.Count; x++)
{
- var fv = FeatureDist.Find(x => x.Label == Y && x.FeatureName == f.Name).FeatureValues;
- prob += smoother.Log2Prob(fv, f.Value);
- });
+ var Xn = features[x];
+ var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues;
+
+ // features are independent, so calculate every feature prob and sum them
+ prob += Math.Log(smoother.Prob(fv, Xn.Value), 2);
+ }
return prob;
}
@@ -65,7 +71,17 @@ namespace BotSharp.Algorithm.Bayes
}
}
- public class FeatureFrequencyDistribution
+ public class FeaturesWithLabel
+ {
+ public List Features { get; set; }
+ public string Label { get; set; }
+ public FeaturesWithLabel()
+ {
+ this.Features = new List();
+ }
+ }
+
+ public class FeaturesDistribution
{
public string Label { get; set; }
@@ -78,14 +94,4 @@ namespace BotSharp.Algorithm.Bayes
return $"{Label} {FeatureName} {FeatureValues.Count}";
}
}
-
- public class LabeledFeatureSet
- {
- public List Features { get; set; }
- public string Label { get; set; }
- public LabeledFeatureSet()
- {
- this.Features = new List();
- }
- }
}
diff --git a/BotSharp.Algorithm/Formulas/Lidstone.cs b/BotSharp.Algorithm/Formulas/Lidstone.cs
index 8596c894..78bcc3a3 100644
--- a/BotSharp.Algorithm/Formulas/Lidstone.cs
+++ b/BotSharp.Algorithm/Formulas/Lidstone.cs
@@ -29,17 +29,12 @@ namespace BotSharp.Algorithm.Formulas
/// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator.
/// https://en.wikipedia.org/wiki/Additive_smoothing
///
- public class Lidstone
+ public class Lidstone : ISmoother
{
///
/// α > 0 is the smoothing parameter
///
- private double _a;
-
- public Lidstone(double alpha = 0.5D)
- {
- _a = alpha;
- }
+ public double Alpha { get; set; }
///
/// Probability
@@ -49,6 +44,11 @@ namespace BotSharp.Algorithm.Formulas
///
public double Prob(List dist, string sample)
{
+ if(Alpha == 0)
+ {
+ Alpha = 0.5D;
+ }
+
// observation x = (x1, ..., xd)
var p = dist.Find(f => f.Value == sample);
int x = p == null ? 0 : p.Freq;
@@ -58,31 +58,7 @@ namespace BotSharp.Algorithm.Formulas
int _d = dist.Count;
- return (x + _a) / (_N + _a * _d);
- }
-
- ///
- /// 2 based Log probability
- ///
- /// distribution
- /// sample value
- ///
- public double Log2Prob(List dist, string sample)
- {
- var d = Prob(dist, sample);
- return Math.Log(d, 2);
- }
-
- ///
- /// 10 based Log probability
- ///
- /// distribution
- /// sample value
- ///
- public double Log10Prob(List dist, string sample)
- {
- var d = Prob(dist, sample);
- return Math.Log(d, 10);
+ return (x + Alpha) / (_N + Alpha * _d);
}
}
}
diff --git a/BotSharp.Algorithm/ISmoother.cs b/BotSharp.Algorithm/ISmoother.cs
new file mode 100644
index 00000000..322dd6ef
--- /dev/null
+++ b/BotSharp.Algorithm/ISmoother.cs
@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Algorithm
+{
+ public interface ISmoother
+ {
+ double Prob(List dist, string sample);
+ }
+}
diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
index b7dcc0ac..9def3cc4 100644
--- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
+++ b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
@@ -100,7 +100,7 @@ namespace BotSharp.Core.Engines.BotSharp
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
- List featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
+ var featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
/*
// try using spacy doc2vec
diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs
index 4cf2247a..a90d2109 100644
--- a/BotSharp.NLP/Classify/ClassifierFactory.cs
+++ b/BotSharp.NLP/Classify/ClassifierFactory.cs
@@ -25,10 +25,7 @@ namespace BotSharp.NLP.Classify
public List> Classify(Sentence sentence)
{
- var classes = _classifier.Classify(new LabeledFeatureSet
- {
- Features = GetFeatures(sentence.Words)
- }, new ClassifyOptions
+ var classes = _classifier.Classify(GetFeatures(sentence.Words), new ClassifyOptions
{
});
@@ -37,7 +34,7 @@ namespace BotSharp.NLP.Classify
public void Train(List sentences)
{
- _classifier.Train(sentences.Select(x => new LabeledFeatureSet
+ _classifier.Train(sentences.Select(x => new FeaturesWithLabel
{
Label = x.Label,
Features = GetFeatures(x.Words)
diff --git a/BotSharp.NLP/Classify/IClassifier.cs b/BotSharp.NLP/Classify/IClassifier.cs
index 833a2732..e52b995c 100644
--- a/BotSharp.NLP/Classify/IClassifier.cs
+++ b/BotSharp.NLP/Classify/IClassifier.cs
@@ -7,8 +7,8 @@ namespace BotSharp.NLP.Classify
{
public interface IClassifier
{
- void Train(List featureSets, ClassifyOptions options);
+ void Train(List featureSets, ClassifyOptions options);
- List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options);
+ List> Classify(List features, ClassifyOptions options);
}
}
diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs
index afbb51d1..8ef10aaa 100644
--- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs
+++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs
@@ -37,11 +37,11 @@ namespace BotSharp.NLP.Classify
///
public class NaiveBayesClassifier : IClassifier
{
- private List featureDist;
+ private List featuresDist;
private List labelDist;
- public void Train(List featureSets, ClassifyOptions options)
+ public void Train(List featureSets, ClassifyOptions options)
{
labelDist = featureSets.GroupBy(x => x.Label)
.Select(x => new Probability
@@ -65,7 +65,7 @@ namespace BotSharp.NLP.Classify
Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList()
}).ToList();
- featureDist = new List();
+ featuresDist = new List();
labelDist.Select(x => x.Value).ToList().ForEach(label =>
{
@@ -82,7 +82,7 @@ namespace BotSharp.NLP.Classify
.OrderBy(f => f.Value)
.ToList();
- featureDist.Add(new FeatureFrequencyDistribution
+ featuresDist.Add(new FeaturesDistribution
{
Label = label,
FeatureName = fName,
@@ -92,17 +92,14 @@ namespace BotSharp.NLP.Classify
});
}
- public List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
+ public List> Classify(List features, ClassifyOptions options)
{
- var nb = new NaiveBayes();
+ // calculate prop
+ var nb = new NaiveBayes();
nb.LabelDist = labelDist;
- nb.FeatureDist = featureDist;
-
- labelDist.ForEach(lf =>
- {
- // prior probability
- lf.Prob = nb.PosteriorProb(lf.Value, featureSet);
- });
+ nb.FeaturesDist = featuresDist;
+
+ labelDist.ForEach(lf => lf.Prob = nb.PosteriorProb(lf.Value, features));
// add log
double[] logs = labelDist.Select(x => x.Prob).ToArray();
diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs
index 26bd1dd4..4bbc4711 100644
--- a/BotSharp.NLP/Classify/SVMClassifier.cs
+++ b/BotSharp.NLP/Classify/SVMClassifier.cs
@@ -32,15 +32,15 @@ namespace BotSharp.NLP.Classify
///
public class SVMClassifier : IClassifier
{
- public List> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
+ public List> Classify(List features, ClassifyOptions options)
{
return null;
}
- public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options)
+ public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options)
{
Problem predict = new Problem();
- List featureSets = new List();
+ List featureSets = new List();
featureSets.Add(featureSet);
predict.X = GetData(featureSets).ToArray();
predict.Y = new double[1];
@@ -53,12 +53,12 @@ namespace BotSharp.NLP.Classify
return Prediction.PredictLabelsProbability(options.Model, scaled);
}
- public void Train(List featureSets, ClassifyOptions options)
+ public void Train(List featureSets, ClassifyOptions options)
{
SVMClassifierTrain(featureSets, options);
}
- public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
+ public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
// copy test multiclass Model
Problem train = new Problem();
@@ -91,10 +91,10 @@ namespace BotSharp.NLP.Classify
Console.Write("Training finished!");
}
- public List GetLabels(List featureSets)
+ public List GetLabels(List featureSets)
{
List labels = new List();
- foreach (LabeledFeatureSet labelFeatureSet in featureSets)
+ foreach (var labelFeatureSet in featureSets)
{
labels.Add(double.Parse(labelFeatureSet.Label));
}
@@ -102,11 +102,11 @@ namespace BotSharp.NLP.Classify
return labels;
}
- public List GetData(List featureSets)
+ public List GetData(List featureSets)
{
List datas = new List();
- foreach (LabeledFeatureSet labelFeatureSet in featureSets)
+ foreach (var labelFeatureSet in featureSets)
{
List curNodes = new List();
labelFeatureSet.Features.ForEach(features => {
@@ -119,15 +119,15 @@ namespace BotSharp.NLP.Classify
return datas;
}
- public List FeatureSetsGenerator(List sentenceVectors, List labels)
+ public List FeatureSetsGenerator(List sentenceVectors, List labels)
{
- List res = new List();
+ var res = new List();
int j;
for (int i = 0; i < labels.Count; i++)
{
string curLabel = labels[i];
Vec curVec = sentenceVectors[i];
- LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet();
+ var labeledFeatureSet = new FeaturesWithLabel();
j = 1;
foreach (double node in curVec.VecNodes)
{
@@ -141,9 +141,9 @@ namespace BotSharp.NLP.Classify
return res;
}
- public LabeledFeatureSet FeatureSetsGenerator(Vec sentenceVectors, String label)
+ public FeaturesWithLabel FeatureSetsGenerator(Vec sentenceVectors, String label)
{
- LabeledFeatureSet labeledFeatureSet = new LabeledFeatureSet();
+ var labeledFeatureSet = new FeaturesWithLabel();
int j = 1;
foreach (double node in sentenceVectors.VecNodes)
{