SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Abstract Bayes algorithm structure.

Browse source
This commit is contained in:
botsharp2018 2018-09-09 09:36:22 -05:00
parent a26c4fa1da
commit 1c47465878
28 changed files with 109 additions and 1926 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

201
BotSharp.Algorithm.UnitTest/BayesianTest.cs
View file

@ -1,201 +0,0 @@
using BotSharp.Algorithm.Bayesian;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.UnitTest
{
[TestClass]
public class BayesianTest
{
/// <summary>
/// The training set
/// </summary>
private ITrainingSet _trainingSet;
/// <summary>
/// The classifier
/// </summary>
private IClassifier _classifier;
/// <summary>
/// The spam class
/// </summary>
private static IClass _spamClass;
/// <summary>
/// The ham class
/// </summary>
private static IClass _hamClass;
/// <summary>
/// Sets up.
/// </summary>
public void SetUp()
{
_trainingSet = BuildTrainingSet();
_classifier = BuildClassifier(_trainingSet);
}
/// <summary>
/// Builds the classifier.
/// </summary>
/// <returns>Classifier&lt;StringClass, StringToken&gt;.</returns>
private IClassifier BuildClassifier(ITrainingSetAccessor trainingSet)
{
var classifier = new NaiveClassifier(trainingSet)
{
// disable smoothing for exact probabilities
SmoothingAlpha = 0.0D
};
return classifier;
}
/// <summary>
/// Builds the training set.
/// </summary>
/// <returns>ITrainingSet&lt;StringClass, StringToken&gt;.</returns>
private static ITrainingSet BuildTrainingSet()
{
var trainingSet = new TrainingSet();
// build data sets
var spamSet = BuildSpamDataSet();
var hamSet = BuildHamDataSet();
// monkey test
//spamSet.SetSize.Should()
//.Be(hamSet.SetSize, "because this test relies on identical set sizes for exact probability testing");
// register classes
_spamClass = spamSet.Class;
_hamClass = hamSet.Class;
// add the sets and return
trainingSet.Add(spamSet, hamSet);
return trainingSet;
}
/// <summary>
/// Builds the spam data set.
/// </summary>
/// <returns>IDataSet&lt;StringClass, StringToken&gt;.</returns>
private static IDataSet BuildSpamDataSet()
{
return BuildDataSet("spam", 0.5D, "rolex", "watches", "viagra", "prince", "money", "send", "xyzzy");
}
/// <summary>
/// Builds the spam data set.
/// </summary>
/// <returns>IDataSet&lt;StringClass, StringToken&gt;.</returns>
private static IDataSet BuildHamDataSet()
{
return BuildDataSet("ham", 0.5D, "love", "flowers", "unicorn", "friendship", "money", "send", "send");
}
/// <summary>
/// Builds the data set.
/// </summary>
/// <param name="className">Name of the class.</param>
/// <param name="classProbability">The class probability.</param>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <returns>IDataSet&lt;StringClass, StringToken&gt;.</returns>
private static IDataSet BuildDataSet(string className, double classProbability, string token, params string[] additionalTokens)
{
var @class = new StringClass(className, classProbability);
var dataSet = new DataSet(@class);
dataSet.AddToken(new StringToken(token));
dataSet.AddToken(additionalTokens.Select(t => new StringToken(t)));
return dataSet;
}
[TestMethod]
public void CalculateProbabilityReturnsOneHundredPercentForAKnownSpamWord()
{
var token = new StringToken("rolex");
var probability = _classifier.CalculateProbability(_spamClass, token);
//probability.Should().BeApproximately(1.0D, 0.0001D, "because the word is known the be a spam word");
}
[TestMethod]
public void CalculateProbabilityReturnsOneHundredPercentForAKnownHamWord()
{
var token = new StringToken("unicorn");
var probability = _classifier.CalculateProbability(_hamClass, token);
//probability.Should().BeApproximately(1.0D, 0.0001D, "because the word is known the be a ham word");
}
[TestMethod]
public void CalculateProbabilitiesWithHamWordReturnsProbabilitiesForAllClasses()
{
var token = new StringToken("unicorn");
var probabilities = _classifier.CalculateProbabilities(token).ToList();
/*probabilities.Single(p => p.Class.Equals(_spamClass))
.Probability.Should()
.BeApproximately(0D, 0.000001D, "because the token is known to be a ham word");
probabilities.Single(p => p.Class.Equals(_hamClass))
.Probability.Should()
.BeApproximately(1D, 0.000001D, "because the token is known to be a ham word");*/
}
[TestMethod]
public void CalculateProbabilitiesWithMixedWordReturnsProbabilitiesForAllClasses()
{
var token = new StringToken("money");
var probabilities = _classifier.CalculateProbabilities(token).ToList();
/*probabilities.Single(p => p.Class.Equals(_spamClass))
.Probability.Should()
.BeApproximately(0.5D, 0.000001D, "because the token is known to be a ham and spam word");
probabilities.Single(p => p.Class.Equals(_hamClass))
.Probability.Should()
.BeApproximately(0.5D, 0.000001D, "because the token is known to be a ham and spam word");*/
}
[TestMethod]
public void CalculateProbabilitiesWithMixedWordThatIsMoreLikelyHamThanSpamReturnsProbabilitiesForAllClasses()
{
var token = new StringToken("send");
var probabilities = _classifier.CalculateProbabilities(token).ToList();
/*probabilities.Single(p => p.Class.Equals(_spamClass))
.Probability.Should()
.BeApproximately(1 / 3D, 0.000001D, "because the token is more likely to be a ham than spam word");
probabilities.Single(p => p.Class.Equals(_hamClass))
.Probability.Should()
.BeApproximately(2 / 3D, 0.000001D, "because the token is more likely to be a ham than spam word");*/
}
[TestMethod]
public void CalculateProbabilitiesWithRareTokensAndSmoothingAlphaIsUnambiguous()
{
var token1 = new StringToken("rolex");
var token2 = new StringToken("unicorn");
var token3 = new StringToken("send");
const double smoothingAlpha = 1.0D;
var probabilities = _classifier.CalculateProbabilities(new IToken[] { token1, token2, token3 }, smoothingAlpha).ToList();
/*probabilities.Single(p => p.Class.Equals(_spamClass))
.Probability.Should()
.BeLessThan(0.5D, "because we used more ham than spam tokens");
probabilities.Single(p => p.Class.Equals(_hamClass))
.Probability.Should()
.BeGreaterThan(0.5D, "because we used more ham than spam tokens");*/
}
}
}

91
BotSharp.Algorithm/Bayes/NaiveBayes.cs Normal file
View file

@ -0,0 +1,91 @@
using BotSharp.Algorithm.Extensions;
using BotSharp.Algorithm.Formulas;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Algorithm.Bayes
{
/// <summary>
/// https://en.wikipedia.org/wiki/Bayes%27_theorem
/// </summary>
public class NaiveBayes
{
/// <summary>
/// smoothing function
/// </summary>
private Lidstone smoother;
public List<FeatureFrequencyDistribution> FeatureDist { get; set; }
public List<Probability> LabelDist { get; set; }
public NaiveBayes()
{
smoother = new Lidstone();
}
/// <summary>
/// calculate posterior probability P(Y|X)
/// X is feature set, Y is label
/// P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
/// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
/// </summary>
/// <param name="Y">label</param>
/// <param name="featureSet"></param>
/// <returns></returns>
public double PosteriorProb(string Y, LabeledFeatureSet featureSet)
{
double prob = 0;
// prior probability
prob = smoother.Log2Prob(LabelDist, Y);
// posterior probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
featureSet.Features.ForEach(f =>
{
var fv = FeatureDist.Find(x => x.Label == Y && x.FeatureName == f.Name).FeatureValues;
prob += smoother.Log2Prob(fv, f.Value);
});
return prob;
}
}
public class Feature
{
public string Name { get; set; }
public string Value { get; set; }
public Feature(string name, string value)
{
Name = name;
Value = value;
}
}
public class FeatureFrequencyDistribution
{
public string Label { get; set; }
public string FeatureName { get; set; }
public List<Probability> FeatureValues { get; set; }
public override string ToString()
{
return $"{Label} {FeatureName} {FeatureValues.Count}";
}
}
public class LabeledFeatureSet
{
public List<Feature> Features { get; set; }
public string Label { get; set; }
public LabeledFeatureSet()
{
this.Features = new List<Feature>();
}
}
}

52
BotSharp.Algorithm/Bayesian/ClassBase.cs
View file

@ -1,52 +0,0 @@
using System;
using System.Diagnostics;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class ClassBase.
/// </summary>
[DebuggerDisplay("Class {Name}, base P = {Probability}")]
public abstract class ClassBase : IClass
{
/// <summary>
/// Gets the name.
/// </summary>
/// <value>The name.</value>
public string Name { get; private set; }
/// <summary>
/// Gets the class' base probability.
/// </summary>
/// <value>The probability.</value>
public double Probability { get; set; }
/// <summary>
/// Initializes a new instance of the <see cref="ClassBase" /> class.
/// </summary>
/// <param name="name">The name.</param>
/// <param name="probability">The probability.</param>
/// <exception cref="System.ArgumentNullException">name</exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// probability;Class base probability must be greater than or equal to zero.
/// or
/// probability;Class base probability must be less than or equal to one.
/// </exception>
protected ClassBase(string name, double probability)
{
if (ReferenceEquals(name, null)) throw new ArgumentNullException("name");
if (probability < 0) throw new ArgumentOutOfRangeException("probability", "Class base probability must be greater than or equal to zero.");
if (probability > 1) throw new ArgumentOutOfRangeException("probability", "Class base probability must be less than or equal to one.");
Name = name;
Probability = probability;
}
/// <summary>
/// Indicates whether the current object is equal to another object of the same type.
/// </summary>
/// <param name="other">An object to compare with this object.</param>
/// <returns>true if the current object is equal to the <paramref name="other" /> parameter; otherwise, false.</returns>
public abstract bool Equals(IClass other);
}
}

46
BotSharp.Algorithm/Bayesian/CombinedConditionalProbabilities.cs
View file

@ -1,46 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Struct CombinedConditionalProbability
/// </summary>
[DebuggerDisplay("P({Class}|{TokenProbabilities.Count} tokens)={Probability}")]
public struct CombinedConditionalProbability
{
/// <summary>
/// The class
/// </summary>
public readonly IClass Class;
/// <summary>
/// The token probabilities
/// </summary>
public ICollection<ConditionalProbability> TokenProbabilities;
/// <summary>
/// The probability
/// </summary>
public double Probability;
/// <summary>
/// Initializes a new instance of the <see cref="CombinedConditionalProbability" /> struct.
/// </summary>
/// <param name="class">The class.</param>
/// <param name="probability">The probability.</param>
/// <param name="tokenProbabilities">The tokenProbabilities.</param>
public CombinedConditionalProbability(IClass @class, double probability, ICollection<ConditionalProbability> tokenProbabilities)
{
if (ReferenceEquals(@class, null)) throw new ArgumentNullException("class");
if (ReferenceEquals(tokenProbabilities, null)) throw new ArgumentNullException("tokenProbabilities");
if (probability < 0) throw new ArgumentOutOfRangeException("probability", probability, "Probability must greater than or equal to zero");
if (probability > 1) throw new ArgumentOutOfRangeException("probability", probability, "Probability must less than or equal to one");
Class = @class;
TokenProbabilities = tokenProbabilities;
Probability = probability;
}
}
}

104
BotSharp.Algorithm/Bayesian/ConditionalProbability.cs
View file

@ -1,104 +0,0 @@
using System;
using System.Diagnostics;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Struct ConditionalProbability
/// </summary>
[DebuggerDisplay("P({Class}|{Token})={Probability}")]
public struct ConditionalProbability : IEquatable<ConditionalProbability>
{
/// <summary>
/// The class
/// </summary>
public readonly IClass Class;
/// <summary>
/// The token
/// </summary>
public readonly IToken Token;
/// <summary>
/// The conditional probability
/// </summary>
public readonly double Probability;
/// <summary>
/// The occurrence of the token during the training phase.
/// </summary>
public readonly long Occurrence;
/// <summary>
/// Initializes a new instance of the <see cref="ConditionalProbability" /> struct.
/// </summary>
/// <param name="class">The class.</param>
/// <param name="token">The token.</param>
/// <param name="probability">The probability.</param>
/// <param name="occurrence">The occurrence.</param>
/// <exception cref="System.ArgumentNullException">@class
/// or
/// token</exception>
/// <exception cref="System.ArgumentOutOfRangeException">probability;Probability must greater than or equal to zero
/// or
/// probability;Probability must less than or equal to one</exception>
public ConditionalProbability(IClass @class, IToken token, double probability, long occurrence)
{
if (ReferenceEquals(@class, null)) throw new ArgumentNullException("class");
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (probability < 0) throw new ArgumentOutOfRangeException("probability", probability, "Probability must greater than or equal to zero");
if (probability > 1) throw new ArgumentOutOfRangeException("probability", probability, "Probability must less than or equal to one");
if (probability < 0) throw new ArgumentOutOfRangeException("occurrence", occurrence, "Occurrence must greater than or equal to zero");
Class = @class;
Token = token;
Probability = probability;
Occurrence = occurrence;
}
/// <summary>
/// Determines whether the specified <see cref="System.Object" /> is equal to this instance.
/// </summary>
/// <param name="obj">Another object to compare to.</param>
/// <returns><see langword="true" /> if the specified <see cref="System.Object" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
public override bool Equals(object obj)
{
if (ReferenceEquals(obj, null)) return false;
return obj is ConditionalProbability && Equals((ConditionalProbability) obj);
}
/// <summary>
/// Indicates whether the current object is equal to another object of the same type.
/// </summary>
/// <param name="other">An object to compare with this object.</param>
/// <returns>true if the current object is equal to the <paramref name="other" /> parameter; otherwise, false.</returns>
public bool Equals(ConditionalProbability other)
{
return Class.Equals(other.Class)
&& Token.Equals(other.Token)
&& Probability.Equals(other.Probability);
}
/// <summary>
/// Returns a hash code for this instance.
/// </summary>
/// <returns>A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table.</returns>
public override int GetHashCode()
{
var hash = 27;
hash = (13 * hash) + Class.GetHashCode();
hash = (13 * hash) + Token.GetHashCode();
hash = (13 * hash) + Probability.GetHashCode();
return hash;
}
/// <summary>
/// Returns a <see cref="System.String" /> that represents this instance.
/// </summary>
/// <returns>A <see cref="System.String" /> that represents this instance.</returns>
public override string ToString()
{
return String.Format("P({0}|{1})={2:P}", Class, Token, Probability);
}
}
}

358
BotSharp.Algorithm/Bayesian/DataSet.cs
View file

@ -1,358 +0,0 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Linq;
using System.Threading;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class DataSet.
/// </summary>
[DebuggerDisplay("Data set for class P({Class.Name})={Class.Probability}")]
public sealed class DataSet : IDataSet
{
/// <summary>
/// The default smoothing alpha
/// </summary>
public const double DefaultSmoothingAlpha = 0D;
/// <summary>
/// The token count
/// </summary>
private readonly ConcurrentDictionary<IToken, long> _tokenCount = new ConcurrentDictionary<IToken, long>();
/// <summary>
/// The set size, i.e. the number of all tokens
/// </summary>
private long _setSize;
/// <summary>
/// Gets the number of distinct tokens,
/// i.e. every token counted at exactly once.
/// </summary>
/// <value>The token count.</value>
/// <seealso cref="SetSize"/>
public long TokenCount
{
get { return _tokenCount.Count; }
}
/// <summary>
/// Gets the size of the set.
/// </summary>
/// <value>The size of the set.</value>
/// <seealso cref="TokenCount"/>
public long SetSize
{
get { return _setSize; }
}
/// <summary>
/// Gets the class.
/// </summary>
/// <value>The class.</value>
public IClass Class { get; private set; }
/// <summary>
/// Initializes a new instance of the <see cref="DataSet"/> class.
/// </summary>
/// <param name="class">The class.</param>
/// <exception cref="System.ArgumentNullException">@class</exception>
public DataSet(IClass @class)
{
if (ReferenceEquals(@class, null)) throw new ArgumentNullException("class");
Class = @class;
}
/// <summary>
/// Gets the <see cref="TokenInformation{IToken}" /> with the specified token.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>TokenInformation&lt;IToken&gt;.</returns>
/// <exception cref="System.ArgumentNullException">token</exception>
public TokenInformation<IToken> this[IToken token, double alpha = DefaultSmoothingAlpha]
{
get
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
long count;
if (!_tokenCount.TryGetValue(token, out count))
{
return new TokenInformation<IToken>(token, 0L, 0D);
}
var percentage = GetPercentage(count, alpha);
return new TokenInformation<IToken>(token, count, percentage);
}
}
/// <summary>
/// Gets the number of occurrences of the given token.
/// </summary>
/// <param name="token">The token.</param>
/// <returns>System.Int64.</returns>
/// <seealso cref="GetPercentage" />
public long GetCount(IToken token)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
long count;
return !_tokenCount.TryGetValue(token, out count) ? 0 : count;
}
/// <summary>
/// Gets the approximated percentage of the given
/// <see cref="IToken" /> in this data set
/// by determining its occurrence count over the whole population.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>System.Double.</returns>
/// <exception cref="System.ArgumentNullException">token</exception>
/// <seealso cref="GetCount" />
public double GetPercentage(IToken token, double alpha = DefaultSmoothingAlpha)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (alpha < 0) throw new ArgumentOutOfRangeException("alpha", alpha, "Smoothing parameter alpha must be greater than or equal to zero.");
var count = GetCount(token);
return GetPercentage(count, alpha);
}
/// <summary>
/// Gets the approximated percentage of the given
/// <see cref="IToken" /> in this data set
/// by determining its occurrence count over the whole population.
/// </summary>
/// <param name="tokenCount">The token count.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>System.Double.</returns>
/// <exception cref="System.ArgumentNullException">token</exception>
/// <seealso cref="GetCount" />
private double GetPercentage(long tokenCount, double alpha = DefaultSmoothingAlpha)
{
Debug.Assert(alpha >= 0, "alpha >= 0");
Debug.Assert(tokenCount >= 0, "tokenCount >= 0");
var totalCount = _setSize; // TODO: cache inverse set size
var vocabularySize = TokenCount;
return (double)(tokenCount + alpha)/(double)(totalCount + alpha*vocabularySize);
}
/// <summary>
/// Adds the given tokens a single time, incrementing the <see cref="SetSize"/>
/// and, at the first addition, the <see cref="TokenCount"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.ArgumentNullException">
/// token
/// or
/// additionalTokens
/// </exception>
public void AddToken(IToken token, params IToken[] additionalTokens)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (ReferenceEquals(additionalTokens, null)) throw new ArgumentNullException("additionalTokens");
_tokenCount.AddOrUpdate(token, AddFirsIToken, IncremenITokenCount);
Interlocked.Increment(ref _setSize);
AddToken(additionalTokens);
}
/// <summary>
/// Adds the given tokens a single time, incrementing the <see cref="SetSize"/>
/// and, at the first addition, the <see cref="TokenCount"/>.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.ArgumentNullException">tokens</exception>
public void AddToken(IEnumerable<IToken> tokens)
{
if (ReferenceEquals(tokens, null)) throw new ArgumentNullException("tokens");
foreach (var token in tokens)
{
_tokenCount.AddOrUpdate(token, AddFirsIToken, IncremenITokenCount);
Interlocked.Increment(ref _setSize);
}
}
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="SetSize"/> and,
/// eventually, the <see cref="TokenCount"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.ArgumentNullException">
/// token
/// or
/// additionalTokens
/// </exception>
/// <seealso cref="PurgeToken(IToken,IToken[])"/>
public void RemoveTokenOnce(IToken token, params IToken[] additionalTokens)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (ReferenceEquals(additionalTokens, null)) throw new ArgumentNullException("additionalTokens");
RemoveSingleTokenInternal(token);
RemoveTokenOnce(additionalTokens);
}
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="SetSize"/> and,
/// eventually, the <see cref="TokenCount"/>.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.ArgumentNullException">tokens</exception>
/// <seealso cref="PurgeToken(IEnumerable&lt;IToken&gt;)"/>
public void RemoveTokenOnce(IEnumerable<IToken> tokens)
{
if (ReferenceEquals(tokens, null)) throw new ArgumentNullException("tokens");
foreach (var token in tokens)
{
RemoveSingleTokenInternal(token);
}
}
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="SetSize"/> and,
/// eventually, the <see cref="TokenCount"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.ArgumentNullException">
/// token
/// or
/// additionalTokens
/// </exception>
/// <seealso cref="RemoveTokenOnce(IToken,IToken[])"/>
public void PurgeToken(IToken token, params IToken[] additionalTokens)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (ReferenceEquals(additionalTokens, null)) throw new ArgumentNullException("additionalTokens");
PurgeTokenInternal(token);
PurgeToken(additionalTokens);
}
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="SetSize"/> and,
/// eventually, the <see cref="TokenCount"/>.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.ArgumentNullException">tokens</exception>
/// <seealso cref="RemoveTokenOnce(IEnumerable&lt;IToken&gt;)"/>
public void PurgeToken(IEnumerable<IToken> tokens)
{
if (ReferenceEquals(tokens, null)) throw new ArgumentNullException("tokens");
foreach (var token in tokens)
{
PurgeTokenInternal(token);
}
}
/// <summary>
/// Purges the tokens fulfilling the given predicate.
/// </summary>
/// <param name="predicate">The predicate.</param>
public void PurgeWhere(Predicate<TokenCount> predicate)
{
var candidateForPurge = from pair in _tokenCount
let tokenCount = new TokenCount(pair.Key, pair.Value)
where predicate(tokenCount)
select pair.Key;
PurgeToken(candidateForPurge);
}
/// <summary>
/// Removes the single token internally.
/// </summary>
/// <param name="token">The token.</param>
private void RemoveSingleTokenInternal(IToken token)
{
long count;
while (_tokenCount.TryGetValue(token, out count))
{
var newValue = count - 1;
var collectionUpdated = _tokenCount.TryUpdate(token, newValue: newValue, comparisonValue: count);
if (!collectionUpdated) continue;
Interlocked.Decrement(ref _setSize);
if (newValue == 0)
{
// explicit removal if the count is zero
var collection = _tokenCount as ICollection<KeyValuePair<IToken, long>>;
collection.Remove(new KeyValuePair<IToken, long>(token, 0));
}
break;
}
}
/// <summary>
/// Purges a single token internally.
/// </summary>
/// <param name="token">The token.</param>
private void PurgeTokenInternal(IToken token)
{
long count;
if (!_tokenCount.TryRemove(token, out count)) return;
// decrement 'count' times
// TODO: use Interlocked.CompareExchange
for (int i = 0; i < count; ++i)
{
Interlocked.Decrement(ref _setSize);
}
}
/// <summary>
/// Factory to initialize the value in <see cref="_tokenCount"/> for the given <paramref name="token"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <returns>System.Int64.</returns>
private static long AddFirsIToken(IToken token)
{
return 1;
}
/// <summary>
/// Factory to increment the value in <see cref="_tokenCount"/> for the given <paramref name="token"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="count">The number of tokens.</param>
/// <returns>System.Int64.</returns>
private static long IncremenITokenCount(IToken token, long count)
{
return count + 1;
}
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
public IEnumerator<TokenCount> GetEnumerator()
{
return _tokenCount.Select(token => new TokenCount(token.Key, token.Value)).GetEnumerator();
}
/// <summary>
/// Returns an enumerator that iterates through a collection.
/// </summary>
/// <returns>An <see cref="T:System.Collections.IEnumerator" /> object that can be used to iterate through the collection.</returns>
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

148
BotSharp.Algorithm/Bayesian/EmptyDataSet.cs
View file

@ -1,148 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class EmptyDataSet. This class cannot be inherited.
/// </summary>
internal sealed class EmptyDataSet : IDataSet
{
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
public IEnumerator<TokenCount> GetEnumerator()
{
yield break;
}
/// <summary>
/// Returns an enumerator that iterates through a collection.
/// </summary>
/// <returns>An <see cref="T:System.Collections.IEnumerator" /> object that can be used to iterate through the collection.</returns>
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
/// <summary>
/// Gets the token count.
/// </summary>
/// <value>The token count.</value>
public long TokenCount { get { return 0; } }
/// <summary>
/// Gets the size of the set.
/// </summary>
/// <value>The size of the set.</value>
public long SetSize { get { return 0; } }
/// <summary>
/// Gets the class.
/// </summary>
/// <value>The class.</value>
public IClass Class { get; private set; }
/// <summary>
/// Gets the <see cref="TokenInformation{IToken}"/> with the specified token.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>TokenInformation&lt;IToken&gt;.</returns>
public TokenInformation<IToken> this[IToken token, double alpha = 0D]
{
get { return new TokenInformation<IToken>(token, 0L, 0D); }
}
/// <summary>
/// Initializes a new instance of the <see cref="EmptyDataSet"/> class.
/// </summary>
/// <param name="class">The class.</param>
/// <exception cref="System.ArgumentNullException">class</exception>
public EmptyDataSet(IClass @class)
{
if (ReferenceEquals(null, @class)) throw new ArgumentNullException("class");
Class = @class;
}
/// <summary>
/// Gets the count.
/// </summary>
/// <param name="token">The token.</param>
/// <returns>System.Int64.</returns>
public long GetCount(IToken token)
{
return 0L;
}
/// <summary>
/// Gets the percentage.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">The alpha.</param>
/// <returns>System.Double.</returns>
/// <seealso cref="GetCount" />
public double GetPercentage(IToken token, double alpha = 0)
{
return 0D;
}
/// <summary>
/// Adds the token.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.InvalidOperationException">Adding data to the empty data set is not allowed.</exception>
public void AddToken(IToken token, params IToken[] additionalTokens)
{
throw new InvalidOperationException("Adding data to the empty data set is not allowed.");
}
/// <summary>
/// Adds the token.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.InvalidOperationException">Adding data to the empty data set is not allowed.</exception>
public void AddToken(IEnumerable<IToken> tokens)
{
throw new InvalidOperationException("Adding data to the empty data set is not allowed.");
}
/// <summary>
/// Removes the token once.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
public void RemoveTokenOnce(IToken token, params IToken[] additionalTokens)
{
}
/// <summary>
/// Removes the token once.
/// </summary>
/// <param name="tokens">The tokens.</param>
public void RemoveTokenOnce(IEnumerable<IToken> tokens)
{
}
/// <summary>
/// Purges the token.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
public void PurgeToken(IToken token, params IToken[] additionalTokens)
{
}
/// <summary>
/// Purges the token.
/// </summary>
/// <param name="tokens">The tokens.</param>
public void PurgeToken(IEnumerable<IToken> tokens)
{
}
}
}

25
BotSharp.Algorithm/Bayesian/IClass.cs
View file

@ -1,25 +0,0 @@
using System;
using System.ComponentModel;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface IClass
/// </summary>
public interface IClass : IEquatable<IClass>
{
/// <summary>
/// Gets the name.
/// </summary>
/// <value>The name.</value>
string Name { get; }
/// <summary>
/// Gets or sets the class' base probability.
/// </summary>
/// <value>The probability.</value>
[DefaultValue(1)]
double Probability { get; set; }
}
}

55
BotSharp.Algorithm/Bayesian/IClassifier.cs
View file

@ -1,55 +0,0 @@
using System;
using System.Collections.Generic;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface IClassifier
/// </summary>
public interface IClassifier
{
/// <summary>
/// Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.
/// <para>
/// Laplace smoothing is required in the context or rare (i.e. untrained) tokens or tokens
/// that do not appear in some classes. With smoothing disabled, these tokens result
/// in a zero probability for the whole class. To counter that, a positive ("alpha")
/// value for smoothing can be set.
/// </para>
/// </summary>
double SmoothingAlpha { get; set; }
/// <summary>
/// Calculates the probability of having the <see cref="IClass"/>
/// given the occurrence of the <see cref="IToken"/>.
/// </summary>
/// <param name="classUnderTest">The class under test.</param>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied, setting to <see langword="null"/> defaults to the values set in <see cref="SmoothingAlpha"/>.</param>
/// <returns>System.Double.</returns>
double CalculateProbability(IClass classUnderTest, IToken token, double? alpha = null);
/// <summary>
/// Calculates the probability of having the
/// <see cref="IClass" />
/// given the occurrence of the
/// <see cref="IToken" />.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied, setting to <see langword="null"/> defaults to the values set in <see cref="SmoothingAlpha"/>.</param>
/// <returns>System.Double.</returns>
IEnumerable<ConditionalProbability> CalculateProbabilities(IToken token, double? alpha = null);
/// <summary>
/// Calculates the probability of having the
/// <see cref="IClass" />
/// given the occurrence of the
/// <see cref="IToken" />.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied, setting to <see langword="null"/> defaults to the values set in <see cref="SmoothingAlpha"/>.</param>
/// <returns>System.Double.</returns>
IEnumerable<CombinedConditionalProbability> CalculateProbabilities(ICollection<IToken> tokens, double? alpha = null);
}
}

9
BotSharp.Algorithm/Bayesian/IDataSet.cs
View file

@ -1,9 +0,0 @@
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface IDataSet
/// </summary>
public interface IDataSet : IDataSetAccessor, ITokenRegistration
{
}
}

63
BotSharp.Algorithm/Bayesian/IDataSetAccessor.cs
View file

@ -1,63 +0,0 @@
using System;
using System.Collections.Generic;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface IDataSetAccessor
/// </summary>
public interface IDataSetAccessor : IEnumerable<TokenCount>
{
/// <summary>
/// Gets the number of distinct tokens,
/// i.e. every token counted at exactly once.
/// </summary>
/// <value>The token count.</value>
/// <seealso cref="SetSize"/>
long TokenCount { get; }
/// <summary>
/// Gets the size of the set.
/// </summary>
/// <value>The size of the set.</value>
/// <seealso cref="TokenCount"/>
long SetSize { get; }
/// <summary>
/// Gets the class.
/// </summary>
/// <value>The class.</value>
IClass Class { get; }
/// <summary>
/// Gets the <see cref="TokenInformation{IToken}" /> with the specified token.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>TokenInformation&lt;IToken&gt;.</returns>
/// <exception cref="System.ArgumentNullException">token</exception>
TokenInformation<IToken> this[IToken token, double alpha] { get; }
/// <summary>
/// Gets the number of occurrences of the given token.
/// </summary>
/// <param name="token">The token.</param>
/// <returns>System.Int64.</returns>
/// <exception cref="System.ArgumentNullException">token</exception>
/// <seealso cref="GetPercentage"/>
long GetCount(IToken token);
/// <summary>
/// Gets the approximated percentage of the given
/// <see cref="IToken" /> in this data set
/// by determining its occurrence count over the whole population.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>System.Double.</returns>
/// <exception cref="System.ArgumentNullException">token</exception>
/// <seealso cref="GetCount" />
double GetPercentage(IToken token, double alpha);
}
}

11
BotSharp.Algorithm/Bayesian/IToken.cs
View file

@ -1,11 +0,0 @@
using System;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface IToken
/// </summary>
public interface IToken : IEquatable<IToken>
{
}
}

78
BotSharp.Algorithm/Bayesian/ITokenRegistration.cs
View file

@ -1,78 +0,0 @@
using System.Collections.Generic;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface ITokenRegistration
/// </summary>
public interface ITokenRegistration
{
/// <summary>
/// Adds the given tokens a single time, incrementing the <see cref="DataSet.SetSize"/>
/// and, at the first addition, the <see cref="DataSet.TokenCount"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.ArgumentNullException">
/// token
/// or
/// additionalTokens
/// </exception>
void AddToken(IToken token, params IToken[] additionalTokens);
/// <summary>
/// Adds the given tokens a single time, incrementing the <see cref="DataSet.SetSize"/>
/// and, at the first addition, the <see cref="DataSet.TokenCount"/>.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.ArgumentNullException">tokens</exception>
void AddToken(IEnumerable<IToken> tokens);
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="DataSet.SetSize"/> and,
/// eventually, the <see cref="DataSet.TokenCount"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.ArgumentNullException">
/// token
/// or
/// additionalTokens
/// </exception>
/// <seealso cref="PurgeToken(IToken,IToken[])"/>
void RemoveTokenOnce(IToken token, params IToken[] additionalTokens);
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="DataSet.SetSize"/> and,
/// eventually, the <see cref="DataSet.TokenCount"/>.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.ArgumentNullException">tokens</exception>
/// <seealso cref="PurgeToken(System.Collections.Generic.IEnumerable{IToken})"/>
void RemoveTokenOnce(IEnumerable<IToken> tokens);
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="IDataSet.SetSize"/> and,
/// eventually, the <see cref="IDataSet.TokenCount"/>.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="additionalTokens">The additional tokens.</param>
/// <exception cref="System.ArgumentNullException">
/// token
/// or
/// additionalTokens
/// </exception>
/// <seealso cref="RemoveTokenOnce(IToken,IToken[])"/>
void PurgeToken(IToken token, params IToken[] additionalTokens);
/// <summary>
/// Removes the given tokens a single time, decrementing the <see cref="IDataSet.SetSize"/> and,
/// eventually, the <see cref="IDataSet.TokenCount"/>.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <exception cref="System.ArgumentNullException">tokens</exception>
/// <seealso cref="RemoveTokenOnce(System.Collections.Generic.IEnumerable{IToken})"/>
void PurgeToken(IEnumerable<IToken> tokens);
}
}

28
BotSharp.Algorithm/Bayesian/ITrainingSet.cs
View file

@ -1,28 +0,0 @@
using System.Collections.Generic;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface ITrainingSet
/// </summary>
public interface ITrainingSet : ITrainingSetAccessor
{
/// <summary>
/// Adds the specified data set.
/// </summary>
/// <param name="dataSet">The data set.</param>
/// <param name="additionalDataSets">The additional data sets.</param>
/// <exception cref="System.ArgumentNullException">dataSet</exception>
/// <exception cref="System.ArgumentException">A data set for a given class was already registered.</exception>
void Add(IDataSet dataSet, params IDataSet[] additionalDataSets);
/// <summary>
/// Adds the specified data sets.
/// </summary>
/// <param name="dataSets">The data sets.</param>
/// <exception cref="System.ArgumentNullException">dataSets</exception>
/// <exception cref="System.ArgumentException">A data set for a given class was already registered.</exception>
void Add(IEnumerable<IDataSet> dataSets);
}
}

18
BotSharp.Algorithm/Bayesian/ITrainingSetAccessor.cs
View file

@ -1,18 +0,0 @@
using System.Collections.Generic;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Interface ITrainingSetAccerssor
/// </summary>
public interface ITrainingSetAccessor : IEnumerable<IDataSet>
{
/// <summary>
/// Gets the <see cref="IDataSet"/> with the specified class.
/// </summary>
/// <param name="class">The class.</param>
/// <returns>IDataSet&lt;TClass, TToken&gt;.</returns>
/// <exception cref="System.ArgumentException">No data set was registered for the given class;class</exception>
IDataSet this[IClass @class] { get; }
}
}

67
BotSharp.Algorithm/Bayesian/LinqExtensions.cs
View file

@ -1,67 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Reflection;
namespace BotSharp.Algorithm.Bayesian
{
internal static class LinqExtensions
{
/// <summary>
/// Converts an enumerable to a collection
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="enumerable">The enumerable.</param>
/// <returns>ICollection&lt;T&gt;.</returns>
public static ICollection<T> ToCollection<T>(this IEnumerable<T> enumerable)
{
var type = enumerable.GetType();
if (type.IsGenericCollectionType()) return (ICollection<T>)enumerable;
var collection = new Collection<T>();
foreach (var t in enumerable)
{
collection.Add(t);
}
return collection;
}
/// <summary>
/// Forces evaluation of the enumerable
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="enumerable">The enumerable.</param>
public static void Run<T>(this IEnumerable<T> enumerable)
{
var type = enumerable.GetType();
if (type.IsGenericCollectionType()) return;
foreach (var item in enumerable)
{
}
}
/// <summary>
/// The cache for <see cref="IsGenericCollectionType"/>
/// </summary>
private static readonly ConcurrentDictionary<Type, bool> IsGenericCollectionTypeCache = new ConcurrentDictionary<Type, bool>();
/// <summary>
/// Determines whether the specified type is a (generic) collection.
/// </summary>
/// <param name="type">The type.</param>
/// <returns><c>true</c> if the specified type is collection; otherwise, <c>false</c>.</returns>
public static bool IsGenericCollectionType(this Type type)
{
return IsGenericCollectionTypeCache.GetOrAdd(type, t => type.GetInterfaces()
.Any(ti => ti.IsGenericType
&&
ti.GetGenericTypeDefinition() ==
typeof (ICollection<>)));
}
}
}

201
BotSharp.Algorithm/Bayesian/NaiveClassifier.cs
View file

@ -1,201 +0,0 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class NaiveClassifier. This class cannot be inherited.
/// <para>
/// Assumes that all token occurrences are statistically independent.
/// </para>
/// </summary>
public sealed class NaiveClassifier : IClassifier
{
/// <summary>
/// The training sets
/// </summary>
private readonly ITrainingSetAccessor _trainingSets;
/// <summary>
/// Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.
/// </summary>
private double _smoothingAlpha = 0.01D;
/// <summary>
/// Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.
/// </summary>
[DefaultValue(0.01D)]
public double SmoothingAlpha
{
get { return _smoothingAlpha; }
set
{
if (value <= 0) throw new ArgumentOutOfRangeException("value", value, "Value must be greater than zero.");
_smoothingAlpha = value;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="NaiveClassifier"/> class.
/// </summary>
/// <param name="trainingSets">The training sets.</param>
/// <exception cref="System.ArgumentNullException">trainingSets</exception>
public NaiveClassifier(ITrainingSetAccessor trainingSets)
{
if (ReferenceEquals(trainingSets, null)) throw new ArgumentNullException("trainingSets");
_trainingSets = trainingSets;
}
/// <summary>
/// Calculates the probability of having the <see cref="IClass"/>
/// given the occurrence of the <see cref="IToken"/>.
/// </summary>
/// <param name="classUnderTest">The class under test.</param>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>System.Double.</returns>
public double CalculateProbability(IClass classUnderTest, IToken token, double? alpha = null)
{
var smoothingAlpha = alpha ?? _smoothingAlpha;
ICollection<IDataSetAccessor> remainingSets;
var setForClassUnderTest = SplitDataSets(classUnderTest, out remainingSets);
// calculate the token's probability in the class under test
var percentageInClassUnderTest = setForClassUnderTest.GetPercentage(token, smoothingAlpha);
var probabilityInClassUnderTest = percentageInClassUnderTest * classUnderTest.Probability;
// calculate the token's probabilities for the remaining classes
double sumOfRemainingProbabilites;
CalculateTokenProbabilityGivenClass(token, remainingSets, out sumOfRemainingProbabilites, smoothingAlpha).Run();
// calculate total probability
var totalProbability = probabilityInClassUnderTest + sumOfRemainingProbabilites;
// calculate the class' probability given the token
var probabilityForClass = probabilityInClassUnderTest/totalProbability;
// correct for rare words
return probabilityForClass;
}
/// <summary>
/// Calculates the probability of having the
/// <see cref="IClass" />
/// given the occurrence of the
/// <see cref="IToken" />.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>System.Double.</returns>
public IEnumerable<ConditionalProbability> CalculateProbabilities(IToken token, double? alpha = null)
{
var smoothingAlpha = alpha ?? _smoothingAlpha;
// calculate the token's probabilities for all classes
double totalProbability;
var probabilities = CalculateTokenProbabilityGivenClass(token, _trainingSets, out totalProbability, smoothingAlpha);
// apply Bayes theorem
var inverseOfTotalProbability = 1.0D/totalProbability;
return from cp in probabilities
let conditionalProbability = cp.Probability * inverseOfTotalProbability
select new ConditionalProbability(cp.Class, cp.Token, conditionalProbability, cp.Occurrence);
}
/// <summary>
/// Calculates the probability of having the
/// <see cref="IClass" />
/// given the occurrence of the
/// <see cref="IToken" />.
/// </summary>
/// <param name="tokens">The tokens.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>System.Double.</returns>
public IEnumerable<CombinedConditionalProbability> CalculateProbabilities(ICollection<IToken> tokens, double? alpha = null)
{
var smoothingAlpha = alpha ?? _smoothingAlpha;
var cpgs = tokens
.SelectMany(token => CalculateProbabilities(token, smoothingAlpha))
.GroupBy(cp => cp.Class)
.ToCollection();
return from @group in cpgs
let cps = @group.ToCollection()
let eta = cps.Select(cp => cp.Probability)
.Sum(p => Math.Log(1 - p) - Math.Log(p))
let probability = 1/(1 + Math.Exp(eta))
select new CombinedConditionalProbability(@group.Key, probability, cps);
}
/// <summary>
/// Calculates the token probabilities given a class.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="sets">The sets.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <returns>IEnumerable&lt;ConditionalProbability&lt;IClass, IToken&gt;&gt;.</returns>
private IEnumerable<ConditionalProbability> CalculateTokenProbabilityGivenClass(IToken token, IEnumerable<IDataSetAccessor> sets, double alpha)
{
return from set in sets
let @class = set.Class
let classProbability = @class.Probability
let percentageInClass = set.GetPercentage(token, alpha)
let countInClass = set.GetCount(token)
let probabilityInClass = percentageInClass*classProbability
select new ConditionalProbability(@class, token, probabilityInClass, countInClass);
}
/// <summary>
/// Calculates the token probabilities given a class.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="sets">The sets.</param>
/// <param name="alpha">Additive smoothing parameter. If set to zero, no Laplace smoothing will be applied.</param>
/// <param name="totalProbability">The total probability for the given classes.</param>
/// <returns>IEnumerable&lt;ConditionalProbability&lt;IClass, IToken&gt;&gt;.</returns>
private IEnumerable<ConditionalProbability> CalculateTokenProbabilityGivenClass(IToken token, IEnumerable<IDataSetAccessor> sets, out double totalProbability, double alpha)
{
var probabilities = CalculateTokenProbabilityGivenClass(token, sets, alpha).ToCollection();
totalProbability = probabilities.Sum(p => p.Probability);
return probabilities;
}
/// <summary>
/// Splits the data sets.
/// </summary>
/// <param name="classUnderTest">The class under test.</param>
/// <param name="remainingSets">The remaining sets.</param>
/// <returns>IDataSet&lt;IClass, IToken&gt;.</returns>
private IDataSetAccessor SplitDataSets(IClass classUnderTest, out ICollection<IDataSetAccessor> remainingSets)
{
IDataSet setForClassUnderTest = null;
remainingSets = new Collection<IDataSetAccessor>();
// split data sets by selected class and other classes
foreach (var trainingSet in _trainingSets)
{
// select the set for the class under test
if (trainingSet.Class.Equals(classUnderTest))
{
Debug.Assert(setForClassUnderTest == null,
"The class under test must not have multiple sets registered in the DataSet");
setForClassUnderTest = trainingSet;
continue;
}
// select remaining sets
remainingSets.Add(trainingSet);
}
// return the found set or an empty set
return setForClassUnderTest ?? new EmptyDataSet(classUnderTest);
}
}
}

84
BotSharp.Algorithm/Bayesian/StringClass.cs
View file

@ -1,84 +0,0 @@
using System;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class StringClass. This class cannot be inherited.
/// </summary>
public sealed class StringClass : ClassBase
{
/// <summary>
/// Initializes a new instance of the <see cref="StringClass" /> class.
/// </summary>
/// <param name="name">The name.</param>
/// <param name="probability">The probability.</param>
/// <exception cref="System.ArgumentNullException">name</exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// probability;Class base probability must be greater than or equal to zero.
/// or
/// probability;Class base probability must be less than or equal to one.
/// </exception>
public StringClass(string name, double probability)
: base(name, probability)
{
}
/// <summary>
/// Determines whether the specified <see cref="StringClass" /> is equal to this instance.
/// </summary>
/// <param name="other">The <see cref="T:System.Object" /> to compare with the current <see cref="T:StringClass" />.</param>
/// <returns><see langword="true" /> if the specified <see cref="System.Object" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
public override bool Equals(IClass other)
{
var otherAsObject = (object) other;
return Equals(otherAsObject);
}
/// <summary>
/// Determines whether the specified <see cref="StringClass" /> is equal to this instance.
/// </summary>
/// <param name="other">The <see cref="T:System.Object" /> to compare with the current <see cref="T:StringClass" />.</param>
/// <returns><see langword="true" /> if the specified <see cref="System.Object" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
private bool Equals(StringClass other)
{
if (ReferenceEquals(other, null)) return false;
if (ReferenceEquals(other, this)) return true;
return String.Equals(Name, other.Name)
&& Math.Abs(Probability - other.Probability) < 0.0001D;
}
/// <summary>
/// Determines whether the specified <see cref="System.Object" /> is equal to this instance.
/// </summary>
/// <param name="obj">The <see cref="T:System.Object" /> to compare with the current <see cref="T:System.Object" />.</param>
/// <returns><see langword="true" /> if the specified <see cref="System.Object" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
if (ReferenceEquals(this, obj)) return true;
return obj is StringClass && Equals((StringClass) obj);
}
/// <summary>
/// Returns a hash code for this instance.
/// </summary>
/// <returns>A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table.</returns>
public override int GetHashCode()
{
var hash = 27;
hash = (13 * hash) + Name.GetHashCode();
hash = (13 * hash) + Probability.GetHashCode();
return hash;
}
/// <summary>
/// Returns a <see cref="System.String" /> that represents this instance.
/// </summary>
/// <returns>A <see cref="System.String" /> that represents this instance.</returns>
public override string ToString()
{
return Name;
}
}
}

81
BotSharp.Algorithm/Bayesian/StringToken.cs
View file

@ -1,81 +0,0 @@
using System;
using System.Diagnostics;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class StringToken. This class cannot be inherited.
/// </summary>
[DebuggerDisplay("{Value}")]
public sealed class StringToken : IToken
{
/// <summary>
/// Gets the value.
/// </summary>
/// <value>The value.</value>
public string Value { get; private set; }
/// <summary>
/// Initializes a new instance of the <see cref="StringToken"/> class.
/// </summary>
/// <param name="value">The value.</param>
public StringToken(string value)
{
if (ReferenceEquals(value, null)) throw new ArgumentNullException("value");
Value = value;
}
/// <summary>
/// Determines whether the specified <see cref="StringToken" /> is equal to this instance.
/// </summary>
/// <param name="other">The <see cref="T:System.Object" /> to compare with the current <see cref="T:System.Object" />.</param>
/// <returns><see langword="true" /> if the specified <see cref="StringToken" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
private bool Equals(StringToken other)
{
return string.Equals(Value, other.Value);
}
/// <summary>
/// Determines whether the specified <see cref="IToken" /> is equal to this instance.
/// </summary>
/// <param name="other">The <see cref="T:System.Object" /> to compare with the current <see cref="T:System.Object" />.</param>
/// <returns><see langword="true" /> if the specified <see cref="StringToken" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
bool IEquatable<IToken>.Equals(IToken other)
{
var otherAsObject = (object)other;
return Equals(otherAsObject);
}
/// <summary>
/// Determines whether the specified <see cref="System.Object" /> is equal to this instance.
/// </summary>
/// <param name="obj">The <see cref="T:System.Object" /> to compare with the current <see cref="T:System.Object" />.</param>
/// <returns><see langword="true" /> if the specified <see cref="System.Object" /> is equal to this instance; otherwise, <see langword="false" />.</returns>
public override bool Equals(object obj)
{
// ReSharper disable once ConditionIsAlwaysTrueOrFalse
if (ReferenceEquals(null, obj)) return false;
if (ReferenceEquals(this, obj)) return true;
return obj is StringToken && Equals((StringToken) obj);
}
/// <summary>
/// Returns a hash code for this instance.
/// </summary>
/// <returns>A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table.</returns>
public override int GetHashCode()
{
return Value.GetHashCode();
}
/// <summary>
/// Returns a <see cref="System.String" /> that represents this instance.
/// </summary>
/// <returns>A <see cref="System.String" /> that represents this instance.</returns>
public override string ToString()
{
return Value;
}
}
}

37
BotSharp.Algorithm/Bayesian/TokenCount.cs
View file

@ -1,37 +0,0 @@
using System;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Struct TokenCount
/// </summary>
public struct TokenCount
{
/// <summary>
/// The token
/// </summary>
public readonly IToken Token;
/// <summary>
/// The number of occurrences
/// </summary>
public readonly long Count;
/// <summary>
/// Initializes a new instance of the <see cref="TokenCount" /> struct.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="count">The count.</param>
/// <exception cref="System.ArgumentNullException">token</exception>
/// <exception cref="System.ArgumentOutOfRangeException">count;Count must be positive or zero.</exception>
public TokenCount(IToken token, long count)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (count < 0) throw new ArgumentOutOfRangeException("count", count, "Count must be positive or zero.");
Token = token;
Count = count;
}
}
}

50
BotSharp.Algorithm/Bayesian/TokenInformation.cs
View file

@ -1,50 +0,0 @@
using System;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Struct TokenInformation
/// </summary>
public struct TokenInformation<TToken>
where TToken: IToken
{
/// <summary>
/// The token
/// </summary>
public readonly TToken Token;
/// <summary>
/// The count in the class
/// </summary>
public long Count;
/// <summary>
/// The occurrence percentage of the token in the class.
/// </summary>
public double Percentage;
/// <summary>
/// Initializes a new instance of the <see cref="TokenInformation{TToken}" /> struct.
/// </summary>
/// <param name="token">The token.</param>
/// <param name="count">The count.</param>
/// <param name="percentage">The percentage.</param>
/// <exception cref="System.ArgumentNullException">token</exception>
/// <exception cref="System.ArgumentOutOfRangeException">
/// count;Count must be positive or zero
/// or
/// percentage;Percentage must be positive or zero
/// </exception>
public TokenInformation(TToken token, long count, double percentage)
{
if (ReferenceEquals(token, null)) throw new ArgumentNullException("token");
if (count < 0) throw new ArgumentOutOfRangeException("count", count, "Count must be positive or zero");
if (percentage < 0) throw new ArgumentOutOfRangeException("percentage", percentage, "Percentage must be positive or zero");
Token = token;
Count = count;
Percentage = percentage;
}
}
}

141
BotSharp.Algorithm/Bayesian/TrainingSet.cs
View file

@ -1,141 +0,0 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
namespace BotSharp.Algorithm.Bayesian
{
/// <summary>
/// Class TrainingSet. This class cannot be inherited.
/// </summary>
public sealed class TrainingSet : ITrainingSet
{
/// <summary>
/// The data sets
/// </summary>
private readonly ConcurrentDictionary<IClass, IDataSet> _dataSets = new ConcurrentDictionary<IClass, IDataSet>();
/// <summary>
/// Initializes a new instance of the <see cref="TrainingSet"/> class.
/// </summary>
public TrainingSet()
{
}
/// <summary>
/// Initializes a new instance of the <see cref="TrainingSet"/> class.
/// </summary>
/// <param name="dataSets">The data sets.</param>
public TrainingSet(IEnumerable<IDataSet> dataSets)
{
Add(dataSets);
}
/// <summary>
/// Initializes a new instance of the <see cref="TrainingSet"/> class.
/// </summary>
/// <param name="dataSet">The data set.</param>
/// <param name="additionalDataSets">The additional data sets.</param>
public TrainingSet(IDataSet dataSet, params IDataSet[] additionalDataSets)
{
Add(dataSet, additionalDataSets);
}
/// <summary>
/// Gets the <see cref="IDataSet"/> with the specified class.
/// </summary>
/// <param name="class">The class.</param>
/// <returns>IDataSet&lt;IClass, IToken&gt;.</returns>
/// <exception cref="System.ArgumentException">No data set was registered for the given class;class</exception>
public IDataSet this[IClass @class]
{
get
{
IDataSet set;
if (_dataSets.TryGetValue(@class, out set)) return set;
throw new ArgumentException("No data set was registered for the given class", "class");
}
}
/// <summary>
/// Adds the specified data set.
/// </summary>
/// <param name="dataSet">The data set.</param>
/// <param name="additionalDataSets">The additional data sets.</param>
/// <exception cref="System.ArgumentNullException">dataSet</exception>
/// <exception cref="System.ArgumentException">A data set for a given class was already registered.</exception>
public void Add(IDataSet dataSet, params IDataSet[] additionalDataSets)
{
if (ReferenceEquals(dataSet, null)) throw new ArgumentNullException("dataSet");
try
{
AddInternal(dataSet);
}
catch (ArgumentException e)
{
throw new ArgumentException("A data set for a given class was already registered.", e);
}
// may throw, that's anticipated
Add(additionalDataSets);
}
/// <summary>
/// Adds the specified data sets.
/// </summary>
/// <param name="dataSets">The data sets.</param>
/// <exception cref="System.ArgumentNullException">dataSets</exception>
/// <exception cref="System.ArgumentException">A data set for a given class was already registered.</exception>
public void Add(IEnumerable<IDataSet> dataSets)
{
if (ReferenceEquals(dataSets, null)) throw new ArgumentNullException("dataSets");
try
{
foreach (var dataSet in dataSets)
{
AddInternal(dataSet);
}
}
catch (ArgumentException e)
{
throw new ArgumentException("A data set for a given class was already registered.", e);
}
}
/// <summary>
/// Adds the data set internally.
/// </summary>
/// <param name="dataSet">The data set.</param>
/// <exception cref="System.ArgumentException">Data set for the given class was already registered.</exception>
private void AddInternal(IDataSet dataSet)
{
if (!_dataSets.TryAdd(dataSet.Class, dataSet))
{
throw new ArgumentException("Data set for the given class was already registered.");
}
}
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
public IEnumerator<IDataSet> GetEnumerator()
{
return _dataSets.Select(dataSet => dataSet.Value).GetEnumerator();
}
/// <summary>
/// Returns an enumerator that iterates through a collection.
/// </summary>
/// <returns>An <see cref="T:System.Collections.IEnumerator" /> object that can be used to iterate through the collection.</returns>
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

7
BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Abstractions;
using BotSharp.Algorithm.Bayes;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP.Classify;
using DotNetToolkit;
@ -29,10 +30,10 @@ namespace BotSharp.Core.Engines.BotSharp
string predictFileName = Path.Combine(Settings.TempDir, "svm-predict-tempfile.txt");
File.WriteAllText(predictFileName, doc.Sentences[0].Text);
NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
var svmClassifier = new NLP.Classify.SVMClassifier();
Args args = new Args();
args.ModelFile = Path.Combine(Configuration.GetValue<String>("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
var featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
/*
//
var client = new RestClient("http://10.2.21.200:5005");

7
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -69,9 +69,10 @@ namespace BotSharp.NLP.UnitTest
corpus.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
// classifier.Train(corpus);
// string text = "Bridget";
// classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) });
classifier.Train(corpus);
string text = "Bridget";
classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) });
corpus.Shuffle();
var trainingData = corpus.Skip(2000).ToList();
classifier.Train(trainingData);

3
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -1,4 +1,5 @@
using BotSharp.NLP.Corpus;
using BotSharp.Algorithm.Bayes;
using BotSharp.NLP.Corpus;
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;

3
BotSharp.NLP/Classify/IClassifier.cs
View file

@ -1,4 +1,5 @@
using System;
using BotSharp.Algorithm.Bayes;
using System;
using System.Collections.Generic;
using System.Text;

66
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -17,6 +17,7 @@
*/
using BotSharp.Algorithm;
using BotSharp.Algorithm.Bayes;
using BotSharp.Algorithm.Extensions;
using BotSharp.Algorithm.Formulas;
using System;
@ -33,8 +34,6 @@ namespace BotSharp.NLP.Classify
/// This technique works well for topic classification;
/// say we have a set of academic papers, and we want to classify them into different topics (computer science, biology, mathematics).
/// Naive Bayes is best for Less training data
/// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
/// Y is label, X is features.
/// </summary>
public class NaiveBayesClassifier : IClassifier
{
@ -95,19 +94,14 @@ namespace BotSharp.NLP.Classify
public List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
{
var estimator = new Lidstone();
var nb = new NaiveBayes();
nb.LabelDist = labelDist;
nb.FeatureDist = featureDist;
labelDist.ForEach(lf =>
{
// prior probability
lf.Prob = estimator.Log2Prob(labelDist, lf.Value);
// post probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
featureSet.Features.ForEach(f =>
{
var fv = featureDist.Find(x => x.Label == lf.Value && x.FeatureName == f.Name).FeatureValues;
lf.Prob += estimator.Log2Prob(fv, f.Value);
});
lf.Prob = nb.PosteriorProb(lf.Value, featureSet);
});
// add log
@ -129,54 +123,4 @@ namespace BotSharp.NLP.Classify
return labelDist.Select(x => new Tuple<string, double>(x.Value, x.Prob)).ToList();
}
}
public class LabeledFeatureSet
{
public List<Feature> Features { get; set; }
public string Label { get; set; }
public LabeledFeatureSet()
{
this.Features = new List<Feature>();
}
}
public class Feature
{
public string Name { get; set; }
public string Value { get; set; }
public Feature(string name, string value)
{
Name = name;
Value = value;
}
}
public class FeatureProbabilityDistribution
{
public string Label { get; set; }
public string FeatureName { get; set; }
public int Count { get; set; }
public override string ToString()
{
return $"{Label} {FeatureName} {Count}";
}
}
public class FeatureFrequencyDistribution
{
public string Label { get; set; }
public string FeatureName { get; set; }
public List<Probability> FeatureValues { get; set; }
public override string ToString()
{
return $"{Label} {FeatureName} {FeatureValues.Count}";
}
}
}

1
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -21,6 +21,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using BotSharp.Algorithm.Bayes;
using SVM.BotSharp.MachineLearning;
using Txt2Vec;