SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fix MultinomiaNaiveBayes bug.

Browse source
This commit is contained in:
Oceania2018 2018-09-11 16:17:12 -05:00
parent b89ced3321
commit f87e506417
10 changed files with 203 additions and 201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
BotSharp.Algorithm/Bayes/BernoulliNaiveBayes.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
public class BernoulliNaiveBayes
{
}
}

10
BotSharp.Algorithm/Bayes/GaussianNaiveBayes.cs Normal file
View file

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
public class GaussianNaiveBayes
{
}
}

110
BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs Normal file
View file

@ -0,0 +1,110 @@
/*
* BotSharp.Algorithm
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
/// <summary>
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
/// </summary>
public class MultinomiaNaiveBayes
{
public List<Probability> LabelDist { get; set; }
public List<Tuple<string, double[]>> FeatureSet { get; set; }
public double Alpha { get; set; }
/// <summary>
/// prior probability
/// </summary>
/// <param name="Y"></param>
/// <returns></returns>
public double CalPriorProb(string Y)
{
int N = FeatureSet.Count;
int k = LabelDist.Count;
int Nyk = LabelDist.First(x => x.Value == Y).Freq;
return (Nyk + Alpha) / (N + k * Alpha);
}
/// <summary>
/// calculate posterior probability P(Y|X)
/// X is feature set, Y is label
/// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
/// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
/// </summary>
public double PosteriorProb(string Y, double[] features, double priorProb)
{
Alpha = 0.5;
int featureCount = features.Length;
double postProb = priorProb;
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
var featuresIfY = FeatureSet.Where(fd => fd.Item1 == Y).ToList();
var matrix = ConstructMatrix(featuresIfY);
// loop features
for (int x = 0; x < featureCount; x++)
{
int freq = 0;
for (int y = 0; y < featuresIfY.Count; y++)
{
if(matrix[y, x] == features[x])
{
freq++;
}
}
int Nyk = featuresIfY.Count;
int n = featureCount;
int Nykx = freq;
postProb += Math.Log((Nykx + Alpha) / (Nyk + n * Alpha));
}
return Math.Pow(2, postProb);
}
private double[,] ConstructMatrix(List<Tuple<string, double[]>> featuresIfY)
{
var featureCount = featuresIfY[0].Item2.Length;
double[,] matrix = new double[featuresIfY.Count, featureCount];
for (int y = 0; y < featuresIfY.Count; y++)
{
for (int x = 0; x < featureCount; x++)
{
matrix[y, x] = featuresIfY[y].Item2[x];
}
}
return matrix;
}
}
}

65
BotSharp.Algorithm/Bayes/NaiveBayes.cs
View file

@ -1,65 +0,0 @@
using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
/// <summary>
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
/// </summary>
public class NaiveBayes<Estimator> where Estimator : IEstimator, new()
{
/// <summary>
/// smoothing function
/// </summary>
private Estimator estomator;
public List<FeaturesDistribution> FeaturesDist { get; set; }
public List<Probability> LabelDist { get; set; }
public NaiveBayes()
{
estomator = new Estimator();
}
/// <summary>
/// calculate posterior probability P(Y|X)
/// X is feature set, Y is label
/// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
/// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
/// </summary>
/// <param name="Y">label</param>
/// <param name="featureSet"></param>
/// <returns></returns>
public double PosteriorProb(string Y, List<Feature> features)
{
double prob = 0;
// prior probability
prob = Math.Log(estomator.Prob(LabelDist, Y), 2);
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
var featuresIfY = FeaturesDist.Where(fd => fd.Label == Y).ToList();
// loop features
for (int x = 0; x < features.Count; x++)
{
var Xn = features[x];
var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues;
if(fv != null)
{
// features are independent, so calculate every feature prob and sum them
prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
}
}
return prob;
}
}
}

25
BotSharp.Algorithm/Estimators/Lidstone.cs → BotSharp.Algorithm/Estimators/AdditiveSmoothing.cs
View file

@ -31,10 +31,12 @@ namespace BotSharp.Algorithm.Estimators
/// https://en.wikipedia.org/wiki/Additive_smoothing
/// Used as Multinomial Naive Bayes
/// </summary>
public class Lidstone : IEstimator
public class AdditiveSmoothing : IEstimator
{
/// <summary>
/// α > 0 is the smoothing parameter
/// 1 > α > 0 is the smoothing parameter is Lidstone
/// α = 1 is Laplace
/// α = 0 no smoothing
/// </summary>
public double Alpha { get; set; }
@ -62,5 +64,24 @@ namespace BotSharp.Algorithm.Estimators
return (x + Alpha) / (_N + Alpha * _d);
}
public double Prob(List<Tuple<string, double>> dist, string sample)
{
if (Alpha == 0)
{
Alpha = 0.5D;
}
// observation x = (x1, ..., xd)
var p = dist.Find(f => f.Item1 == sample);
double x = p == null ? 0D : p.Item2;
// N trials
double _N = dist.Sum(f => f.Item2);
int _d = dist.Count;
return (x + Alpha) / (_N + Alpha * _d);
}
}
}

13
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -32,9 +32,9 @@ namespace BotSharp.NLP.UnitTest
{
newSentences[i].Label = sentences[i].Label;
}
sentences = newSentences.ToList();
sentences = newSentences.Take(10).ToList();
sentences.Shuffle();
//sentences.Shuffle();
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
@ -46,9 +46,7 @@ namespace BotSharp.NLP.UnitTest
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.9M);
classifier.TrainInVector(dataset.Item1);
var dataset = sentences.Split(1M);
classifier.Train(dataset.Item1);
int correct = 0;
@ -58,10 +56,13 @@ namespace BotSharp.NLP.UnitTest
if (td.Label == classes[0].Item1)
{
correct++;
}
});
var accuracy = (float)correct / dataset.Item2.Count;
var accuracy = (float)correct / dataset.Item1.Count;
Assert.IsTrue(accuracy > 0.8);
}
[TestMethod]

35
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -27,31 +27,7 @@ namespace BotSharp.NLP.Classify
featureExtractor = new IFeatureExtractor();
}
public List<Tuple<string, double>> Classify(Sentence sentence)
{
var options = new ClassifyOptions
{
};
var features = featureExtractor.GetFeatures(sentence.Words);
var classes = _classifier.Classify(features, options);
return classes.OrderByDescending(x => x.Item2).ToList();
}
public void Train(List<Sentence> sentences)
{
var sents = sentences.Select(x => new FeaturesWithLabel
{
Label = x.Label,
Features = featureExtractor.GetFeatures(x.Words)
}).ToList();
_classifier.Train(sents, _options);
}
public void TrainInVector(List<Sentence> sentences)
{
var vectors = new List<Tuple<string, double[]>>();
@ -59,5 +35,16 @@ namespace BotSharp.NLP.Classify
_classifier.Train(sents, _options);
}
public List<Tuple<string, double>> Classify(Sentence sentence)
{
var options = new ClassifyOptions
{
};
var classes = _classifier.Classify(sentence.Vector, options);
return classes.OrderByDescending(x => x.Item2).ToList();
}
}
}

4
BotSharp.NLP/Classify/IClassifier.cs
View file

@ -7,10 +7,6 @@ namespace BotSharp.NLP.Classify
{
public interface IClassifier
{
void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options);
List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options);
/// <summary>
/// Training by feature vector
/// </summary>

126
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -40,104 +40,14 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class NaiveBayesClassifier : IClassifier
{
private List<FeaturesDistribution> featuresDist;
private List<Probability> labelDist;
public void Train(List<FeaturesWithLabel> featureSets, ClassifyOptions options)
{
labelDist = featureSets.GroupBy(x => x.Label)
.Select(x => new Probability
{
Value = x.Key,
Freq = x.Count()
})
.ToList();
private MultinomiaNaiveBayes nb = new MultinomiaNaiveBayes();
var fNames = new List<string>();
featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name)));
fNames = fNames.OrderBy(x => x).Distinct().ToList();
var featureValues = new Dictionary<string, List<Feature>>();
for (int i = 0; i < featureSets.Count; i++)
{
var fs = featureSets[i];
featureValues[fs.Label] = new List<Feature>();
fNames.ForEach(fn =>
{
Feature feature = null;
for (int j = 0; j < fs.Features.Count; j++)
{
if (fs.Features[j].Name == fn)
{
feature = fs.Features[j];
break;
}
}
var fv = new Feature(fn, feature == null ? "False" : feature.Value);
featureValues[fs.Label].Add(fv);
});
}
featuresDist = new List<FeaturesDistribution>();
labelDist.Select(x => x.Value).ToList().ForEach(label =>
{
var fSets = featureValues[label];
fNames.ForEach(fName =>
{
var fsv = fSets.Where(fs => fs.Name == fName)
.GroupBy(fs => fs.Value)
.Select(fs => new Probability
{
Value = fs.Key,
Freq = fs.Count()
})
.OrderBy(fs => fs.Value)
.ToList();
featuresDist.Add(new FeaturesDistribution
{
Label = label,
FeatureName = fName,
FeatureValues = fsv
});
});
});
}
public List<Tuple<string, double>> Classify(List<Feature> features, ClassifyOptions options)
{
// calculate prop
var nb = new NaiveBayes<Lidstone>();
nb.LabelDist = labelDist;
nb.FeaturesDist = featuresDist;
Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features));
// add log
double[] logs = labelDist.Select(x => x.Prob).ToArray();
var sumLogs = logs.Reduce((log1, next) =>
{
double min = log1;
if (next < log1)
{
min = next;
}
return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2);
});
labelDist.ForEach(d => d.Prob -= sumLogs);
return labelDist.Select(x => new Tuple<string, double>(x.Value, x.Prob)).ToList();
}
/// <summary>
/// Cache all categories' prior probability
/// </summary>
private Dictionary<string, double> PriorPropDictionary = new Dictionary<string, double>();
public void Train(List<Tuple<string, double[]>> featureSets, ClassifyOptions options)
{
@ -148,11 +58,35 @@ namespace BotSharp.NLP.Classify
Freq = x.Count()
})
.ToList();
nb.LabelDist = labelDist;
nb.FeatureSet = featureSets;
// calculate prior prob
labelDist.ForEach(l => l.Prob = nb.CalPriorProb(l.Value));
// calculate posterior prob
}
public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
{
throw new NotImplementedException();
var results = new List<Tuple<string, double>>();
// calculate prop
labelDist.ForEach(lf =>
{
var prob = nb.PosteriorProb(lf.Value, features, lf.Prob);
results.Add(new Tuple<string, double>(lf.Value, prob));
});
/*Parallel.ForEach(labelDist, (lf) =>
{
nb.Y = lf.Value;
lf.Prob = nb.PosteriorProb();
});*/
return results;
}
}

6
BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
View file

@ -38,10 +38,8 @@ namespace BotSharp.NLP.Txt2Vec
public void EncodeAll()
{
InitDictionary();
Parallel.ForEach(Sentences, sent =>
{
Encode(sent);
});
Sentences.ForEach(sent => Encode(sent));
//Parallel.ForEach(Sentences, sent => Encode(sent));
}
private void InitDictionary()