SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Text classification performance upate.

Browse source
This commit is contained in:
Oceania2018 2018-09-10 17:25:41 -05:00
parent 1f5bdbbc82
commit d8afcb0692
6 changed files with 68 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
BotSharp.Algorithm/Bayes/NaiveBayes.cs
View file

@ -50,10 +50,13 @@ namespace BotSharp.Algorithm.Bayes
for (int x = 0; x < features.Count; x++)
{
var Xn = features[x];
var fv = featuresIfY.First(fd => fd.FeatureName == Xn.Name).FeatureValues;
var fv = featuresIfY.FirstOrDefault(fd => fd.FeatureName == Xn.Name)?.FeatureValues;
// features are independent, so calculate every feature prob and sum them
prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
if(fv != null)
{
// features are independent, so calculate every feature prob and sum them
prob += Math.Log(estomator.Prob(fv, Xn.Value), 2);
}
}
return prob;

9
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -31,8 +31,8 @@ namespace BotSharp.NLP.UnitTest
{
newSentences[i].Label = sentences[i].Label;
}
sentences = newSentences;
sentences = newSentences.Take(10).ToList();
sentences.Shuffle();
var options = new ClassifyOptions
@ -40,11 +40,12 @@ namespace BotSharp.NLP.UnitTest
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.7M);
var dataset = sentences.Split(0.9M);
classifier.Train(dataset.Item1);
int correct = 0;
dataset.Item2.ForEach(td =>
dataset.Item1.ToList().ForEach(td =>
{
var classes = classifier.Classify(td);
if (td.Label == classes[0].Item1)

14
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -29,20 +29,26 @@ namespace BotSharp.NLP.Classify
public List<Tuple<string, double>> Classify(Sentence sentence)
{
var classes = _classifier.Classify(featureExtractor.GetFeatures(sentence.Words), new ClassifyOptions
var options = new ClassifyOptions
{
});
};
var features = featureExtractor.GetFeatures(sentence.Words);
var classes = _classifier.Classify(features, options);
return classes.OrderByDescending(x => x.Item2).ToList();
}
public void Train(List<Sentence> sentences)
{
_classifier.Train(sentences.Select(x => new FeaturesWithLabel
var sents = sentences.Select(x => new FeaturesWithLabel
{
Label = x.Label,
Features = featureExtractor.GetFeatures(x.Words)
}).ToList(), _options);
}).ToList();
_classifier.Train(sents, _options);
}
}
}

56
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -27,6 +27,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.NLP.Classify
{
@ -53,36 +54,51 @@ namespace BotSharp.NLP.Classify
})
.ToList();
var fNames = featureSets[0].Features.Select(x => x.Name)
.Distinct()
.OrderBy(x => x)
.ToList();
var fNames = new List<string>();
// combine all features.
var allFeatureValues = new List<Feature>();
featureSets.ForEach(fs => fNames.ForEach(fName => allFeatureValues.Add(new Feature(fName, fs.Features.First(x => x.Name == fName).Value))));
featureSets.ForEach(fs => fNames.AddRange(fs.Features.Select(x => x.Name)));
fNames = fNames.OrderBy(x => x).Distinct().ToList();
var featureValues = fNames.Select(fn => new
var featureValues = new Dictionary<string, List<Feature>>();
for (int i = 0; i < featureSets.Count; i++)
{
Name = fn,
Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList()
}).ToList();
var fs = featureSets[i];
featureValues[fs.Label] = new List<Feature>();
fNames.ForEach(fn =>
{
Feature feature = null;
for (int j = 0; j < fs.Features.Count; j++)
{
if (fs.Features[j].Name == fn)
{
feature = fs.Features[j];
break;
}
}
var fv = new Feature(fn, feature == null ? "False" : feature.Value);
featureValues[fs.Label].Add(fv);
});
}
featuresDist = new List<FeaturesDistribution>();
labelDist.Select(x => x.Value).ToList().ForEach(label =>
{
var fSets = featureSets.Where(x => x.Label == label);
var fSets = featureValues[label];
fNames.ForEach(fName =>
{
var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName))
.GroupBy(f => f.Value)
.Select(f => new Probability
var fsv = fSets.Where(fs => fs.Name == fName)
.GroupBy(fs => fs.Value)
.Select(fs => new Probability
{
Value = f.Key,
Freq = f.Count()
Value = fs.Key,
Freq = fs.Count()
})
.OrderBy(f => f.Value)
.OrderBy(fs => fs.Value)
.ToList();
featuresDist.Add(new FeaturesDistribution
@ -101,8 +117,8 @@ namespace BotSharp.NLP.Classify
var nb = new NaiveBayes<Lidstone>();
nb.LabelDist = labelDist;
nb.FeaturesDist = featuresDist;
labelDist.ForEach(lf => lf.Prob = nb.PosteriorProb(lf.Value, features));
Parallel.ForEach(labelDist, (lf) => lf.Prob = nb.PosteriorProb(lf.Value, features));
// add log
double[] logs = labelDist.Select(x => x.Prob).ToArray();

5
BotSharp.NLP/Classify/SentenceFeatureExtractor.cs
View file

@ -13,9 +13,10 @@ namespace BotSharp.NLP.Classify
{
var features = new List<Feature>();
words.Where(x => x.Text.Length > 1)
words.Where(x => x.IsAlpha)
.Distinct()
.ToList()
.ForEach(w => features.Add(new Feature("contains", w.Text.ToLower())));
.ForEach(w => features.Add(new Feature($"contains {w.Text.ToLower()}", "True")));
return features;
}

9
BotSharp.NLP/Tokenize/Token.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Tokenize
{
@ -41,7 +42,13 @@ namespace BotSharp.NLP.Tokenize
/// <summary>
/// Is the token an alpha character?
/// </summary>
public bool IsAlpha { get; set; }
public bool IsAlpha
{
get
{
return Regex.IsMatch(Text, @"^[a-zA-Z]+$");
}
}
/// <summary>
/// Is the token part of a stop list, i.e. the most common words of the language?