SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Change Lidstone smooth estimator for NB. Still in progres.

Browse source
This commit is contained in:
Oceania2018 2018-09-07 17:24:57 -05:00
parent fde6508fea
commit 3c2755f579
6 changed files with 167 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
BotSharp.Algorithm/Formulas/Lidstone.cs Normal file
View file

@ -0,0 +1,87 @@
/*
* BotSharp.Algorithm
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.Formulas
{
/// <summary>
/// Lidstone smoothing is a technique used to smooth categorical data.
/// In statistics, it's called additive smoothing or Laplace smoothing.
/// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator.
/// https://en.wikipedia.org/wiki/Additive_smoothing
/// </summary>
public class Lidstone
{
/// <summary>
/// α > 0 is the smoothing parameter
/// </summary>
private double _a;
public Lidstone(double alpha = 0.5D)
{
_a = alpha;
}
/// <summary>
/// Probability
/// </summary>
/// <param name="dist">distribution</param>
/// <param name="sample">sample value</param>
/// <returns></returns>
public double Prob(List<Probability> dist, string sample)
{
// observation x = (x1, ..., xd)
int x = dist.Find(f => f.Value == sample).Freq;
// N trials
int _N = dist.Sum(f => f.Freq);
int _d = dist.Count;
return (x + _a) / (_N + _a * _d);
}
/// <summary>
/// 2 based Log probability
/// </summary>
/// <param name="dist">distribution</param>
/// <param name="sample">sample value</param>
/// <returns></returns>
public double Log2Prob(List<Probability> dist, string sample)
{
var d = Prob(dist, sample);
return Math.Log(d, 2);
}
/// <summary>
/// 10 based Log probability
/// </summary>
/// <param name="dist">distribution</param>
/// <param name="sample">sample value</param>
/// <returns></returns>
public double Log10Prob(List<Probability> dist, string sample)
{
var d = Prob(dist, sample);
return Math.Log(d, 10);
}
}
}

34
BotSharp.Algorithm/Probability.cs Normal file
View file

@ -0,0 +1,34 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Algorithm
{
/// <summary>
/// In probability theory and statistics, a probability distribution is a mathematical function
/// that provides the probabilities of occurrence of different possible outcomes in an experiment.
/// https://en.wikipedia.org/wiki/Probability_distribution
/// </summary>
public class Probability
{
/// <summary>
/// one value of all samples
/// </summary>
public string Value { get; set; }
/// <summary>
/// the number of times that something happens within a particular period of time
/// </summary>
public int Freq { get; set; }
/// <summary>
/// how likely something is, sometimes calculated in a mathematical way
/// </summary>
public double Prob { get; set; }
public override string ToString()
{
return $"{Value} {Freq} {Prob}";
}
}
}

3
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -32,6 +32,9 @@ namespace BotSharp.NLP.UnitTest
corpus.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
classifier.Train(corpus);
string text = "Aamir";
classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) });
}
private List<Sentence> GetLabeledCorpus(ClassifyOptions options)

7
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -24,7 +24,12 @@ namespace BotSharp.NLP.Classify
public void Classify(Sentence sentence)
{
_classifier.Classify(new LabeledFeatureSet
{
Features = GetFeatures(sentence.Words)
}, new ClassifyOptions
{
});
}
public void Train(List<Sentence> sentences)

53
BotSharp.NLP/Classify/Lidstone.cs
View file

@ -1,53 +0,0 @@
/*
* BotSharp.NLP Library
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.NLP.Classify
{
/// <summary>
/// Lidstone smoothing, is a technique used to smooth categorical data.
/// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator:
/// Refer https://en.wikipedia.org/wiki/Additive_smoothing
/// </summary>
public class Lidstone : IEstimator
{
/// <summary>
/// x = (x1, …, xd)
/// </summary>
private int _d;
/// <summary>
/// α > 0 is the smoothing parameter
/// </summary>
private float _a;
/// <summary>
/// N trials
/// </summary>
private int _N;
public Lidstone(float alpha, int bins)
{
_a = alpha;
_d = bins;
}
}
}

50
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -16,6 +16,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using BotSharp.Algorithm;
using BotSharp.Algorithm.Formulas;
using System;
using System.Collections.Generic;
using System.IO;
@ -33,15 +35,18 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class NaiveBayesClassifier : IClassifier
{
public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
{
throw new NotImplementedException();
}
private List<FeatureFrequencyDistribution> featureDist;
private List<Probability> labelDist;
public void Train(List<LabeledFeatureSet> featureSets, ClassifyOptions options)
{
var labelFreqDist = featureSets.GroupBy(x => x.Label)
.Select(x => new { Label = x.Key, Count = x.Count() })
labelDist = featureSets.GroupBy(x => x.Label)
.Select(x => new Probability
{
Value = x.Key,
Freq = x.Count()
})
.ToList();
var fNames = featureSets[0].Features.Select(x => x.Name).ToList();
@ -56,20 +61,24 @@ namespace BotSharp.NLP.Classify
Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList()
}).ToList();
var featureFreqDist = new List<FeatureFrequencyDistribution>();
featureDist = new List<FeatureFrequencyDistribution>();
labelFreqDist.Select(x => x.Label).ToList().ForEach(label =>
labelDist.Select(x => x.Value).ToList().ForEach(label =>
{
var fSets = featureSets.Where(x => x.Label == label);
fNames.ForEach(fName =>
{
var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName))
.GroupBy(f => f.Value)
.Select(f => new Tuple<string, int>(f.Key, f.Count()))
.OrderBy(f => f.Item1)
.Select(f => new Probability
{
Value = f.Key,
Freq = f.Count()
})
.OrderBy(f => f.Value)
.ToList();
featureFreqDist.Add(new FeatureFrequencyDistribution
featureDist.Add(new FeatureFrequencyDistribution
{
Label = label,
FeatureName = fName,
@ -77,11 +86,26 @@ namespace BotSharp.NLP.Classify
});
});
});
}
featureFreqDist.ForEach(ffd =>
public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
{
var estimator = new Lidstone();
labelDist.ForEach(lf =>
{
lf.Prob = estimator.Log2Prob(labelDist, lf.Value);
});
featureDist.ForEach(fd =>
{
fd.FeatureValues.ForEach(fv =>
{
fv.Prob = estimator.Log2Prob(fd.FeatureValues, fv.Value);
var p = labelDist.Find(l => l.Value == fd.Label);
p.Prob += fv.Prob;
});
});
}
}
@ -128,7 +152,7 @@ namespace BotSharp.NLP.Classify
public string FeatureName { get; set; }
public List<Tuple<string, int>> FeatureValues { get; set; }
public List<Probability> FeatureValues { get; set; }
public override string ToString()
{