SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Intergate NBClassifier into NLP pipeline.

Browse source
This commit is contained in:
Oceania2018 2018-09-12 15:31:20 -05:00
parent 347cd5cd9c
commit 30f0eea87c
8 changed files with 137 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.Algorithm/Bayes/MultinomiaNaiveBayesModel.cs
View file

@ -10,5 +10,7 @@ namespace BotSharp.Algorithm.Bayes
public List<Probability> LabelDist { get; set; }
public Dictionary<string, double> CondProbDictionary { get; set; }
public List<String> Values { get; set; }
}
}

41
BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs
View file

@ -22,25 +22,22 @@ namespace BotSharp.Core.Engines.BotSharp
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-nb.model";
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
var encoder = new OneHotEncoder();
encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence
var options = new ClassifyOptions
{
ModelFilePath = modelFileName
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var sentences = doc.Sentences.Select(x => new Sentence
{
Label = x.Intent.Label,
Text = x.Text,
Words = x.Tokens
}).ToList();
encoder.EncodeAll();
var options = new ClassifyOptions
{
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
classifier.Train(encoder.Sentences);
classifier.Train(sentences);
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
@ -51,6 +48,28 @@ namespace BotSharp.Core.Engines.BotSharp
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model)
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var sentence = doc.Sentences.Select(s => new Sentence
{
Text = s.Text,
Words = s.Tokens
}).First();
var result = classifier.Classify(sentence);
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "BotSharpNBayesClassifier",
Label = result.First().Item1,
Confidence = (decimal)result.First().Item2
};
return true;
}
}

7
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -36,17 +36,14 @@ namespace BotSharp.NLP.UnitTest
sentences.Shuffle();
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
encoder.EncodeAll();
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.7M);
var dataset = sentences.Split(1M);
classifier.Train(dataset.Item1);
int correct = 0;

12
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -29,20 +29,20 @@ namespace BotSharp.NLP.Classify
public void Train(List<Sentence> sentences)
{
var vectors = new List<Tuple<string, double[]>>();
var sents = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
_classifier.Train(sents, new double[] { 0, 1 }, _options);
_classifier.Train(sentences, _options);
_classifier.SaveModel(_options);
}
public List<Tuple<string, double>> Classify(Sentence sentence)
{
var options = new ClassifyOptions
{
ModelFilePath = _options.ModelFilePath
};
var classes = _classifier.Classify(sentence.Vector, options);
_classifier.LoadModel(options);
var classes = _classifier.Classify(sentence, options);
return classes.OrderByDescending(x => x.Item2).ToList();
}

12
BotSharp.NLP/Classify/IClassifier.cs
View file

@ -10,16 +10,20 @@ namespace BotSharp.NLP.Classify
/// <summary>
/// Training by feature vector
/// </summary>
/// <param name="featureSets"></param>
/// <param name="sentences"></param>
/// <param name="options"></param>
void Train(List<Tuple<string, double[]>> featureSets, double[] values, ClassifyOptions options);
void Train(List<Sentence> sentences, ClassifyOptions options);
/// <summary>
/// Predict by feature vector
/// </summary>
/// <param name="features"></param>
/// <param name="sentence"></param>
/// <param name="options"></param>
/// <returns></returns>
List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options);
List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options);
String SaveModel(ClassifyOptions options);
Object LoadModel(ClassifyOptions options);
}
}

76
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -22,6 +22,8 @@ using BotSharp.Algorithm.Estimators;
using BotSharp.Algorithm.Extensions;
using BotSharp.Algorithm.Features;
using BotSharp.Algorithm.Statistics;
using BotSharp.NLP.Txt2Vec;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
@ -46,14 +48,24 @@ namespace BotSharp.NLP.Classify
private Dictionary<string, double> condProbDictionary = new Dictionary<string, double>();
public void Train(List<Tuple<string, double[]>> featureSets, double[] values, ClassifyOptions options)
private List<string> words;
private double[] features = new double[] { 0, 1 };
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
words = encoder.EncodeAll();
var featureSets = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
labelDist = featureSets.GroupBy(x => x.Item1)
.Select(x => new Probability
{
Value = x.Key,
Freq = x.Count()
})
.OrderBy(x => x.Value)
.ToList();
nb.LabelDist = labelDist;
@ -70,30 +82,27 @@ namespace BotSharp.NLP.Classify
{
for (int x = 0; x < featureCount; x++)
{
for (int v = 0; v < values.Length; v++)
for (int v = 0; v < features.Length; v++)
{
string key = $"{label.Value} f{x} {values[v]}";
condProbDictionary[key] = nb.CalCondProb(x, label.Value, values[v]);
string key = $"{label.Value} f{x} {features[v]}";
condProbDictionary[key] = nb.CalCondProb(x, label.Value, features[v]);
}
}
});
// save the model
var model = new MultinomiaNaiveBayesModel
{
LabelDist = labelDist,
CondProbDictionary = condProbDictionary
};
}
public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
{
var encoder = new OneHotEncoder();
encoder.Words = words;
encoder.Encode(sentence);
var results = new List<Tuple<string, double>>();
// calculate prop
labelDist.ForEach(lf =>
{
var prob = nb.CalPosteriorProb(lf.Value, features, lf.Prob, condProbDictionary);
var prob = nb.CalPosteriorProb(lf.Value, sentence.Vector, lf.Prob, condProbDictionary);
results.Add(new Tuple<string, double>(lf.Value, prob));
});
@ -105,6 +114,47 @@ namespace BotSharp.NLP.Classify
return results;
}
public string SaveModel(ClassifyOptions options)
{
// save the model
var model = new MultinomiaNaiveBayesModel
{
LabelDist = labelDist,
CondProbDictionary = condProbDictionary,
Values = words
};
//save the file
using (var bw = new BinaryWriter(new FileStream(options.ModelFilePath, FileMode.Create)))
{
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(model));
bw.Write(bytes);
}
return options.ModelFilePath;
}
public Object LoadModel(ClassifyOptions options)
{
string json = String.Empty;
//read the file
using (var br = new BinaryReader(new FileStream(options.ModelFilePath, FileMode.Open)))
{
byte[] bytes = br.ReadBytes((int)br.BaseStream.Length);
json = Encoding.UTF8.GetString(bytes);
}
var model = JsonConvert.DeserializeObject<MultinomiaNaiveBayesModel>(json);
labelDist = model.LabelDist;
condProbDictionary = model.CondProbDictionary;
words = model.Values;
return model;
}
}
public class FeaturesWithLabel

14
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -48,12 +48,12 @@ namespace BotSharp.NLP.Classify
return Prediction.PredictLabelsProbability(options.Model, scaled);
}
public void Train(List<Tuple<string, double[]>> featureSets, double[] values, ClassifyOptions options)
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
// SVMClassifierTrain(featureSets, options);
}
public List<Tuple<string, double>> Classify(double[] features, ClassifyOptions options)
public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
{
throw new NotImplementedException();
}
@ -154,5 +154,15 @@ namespace BotSharp.NLP.Classify
return labeledFeatureSet;
}
public string SaveModel(ClassifyOptions options)
{
throw new NotImplementedException();
}
object IClassifier.LoadModel(ClassifyOptions options)
{
throw new NotImplementedException();
}
}
}

23
BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
View file

@ -15,17 +15,17 @@ namespace BotSharp.NLP.Txt2Vec
{
public List<Sentence> Sentences { get; set; }
private List<string> words;
public List<string> Words { get; set; }
public void Encode(Sentence sentence)
{
InitDictionary();
var vector = words.Select(x => 0D).ToArray();
var vector = Words.Select(x => 0D).ToArray();
sentence.Words.ForEach(w =>
{
int index = words.IndexOf(w.Text.ToLower());
int index = Words.IndexOf(w.Text.ToLower());
if(index > 0)
{
vector[index] = 1;
@ -35,24 +35,29 @@ namespace BotSharp.NLP.Txt2Vec
sentence.Vector = vector;
}
public void EncodeAll()
public List<string> EncodeAll()
{
InitDictionary();
Sentences.ForEach(sent => Encode(sent));
//Parallel.ForEach(Sentences, sent => Encode(sent));
return Words;
}
private void InitDictionary()
private List<string> InitDictionary()
{
if (words == null)
if (Words == null)
{
words = new List<string>();
Words = new List<string>();
Sentences.ForEach(x =>
{
words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
});
words = words.Distinct().OrderBy(x => x).ToList();
Words = Words.Distinct().OrderBy(x => x).ToList();
}
return Words;
}
}
}