SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Abstractor BotSharpIntentClassifier

Browse source
This commit is contained in:
Oceania2018 2018-09-26 06:45:35 -05:00
parent e20d7b48a3
commit 930cc700b9
5 changed files with 93 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs → BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
View file

@ -14,21 +14,15 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpNBayesClassifier : INlpTrain, INlpPredict
public class BotSharpIntentClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
private ClassifierFactory<SentenceFeatureExtractor> _classifier;
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-nb.model";
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
var options = new ClassifyOptions
{
ModelFilePath = modelFileName
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
Init(meta);
var sentences = doc.Sentences.Select(x => new Sentence
{
@ -37,20 +31,16 @@ namespace BotSharp.Core.Engines.BotSharp
Words = x.Tokens
}).ToList();
classifier.Train(sentences);
_classifier.Train(sentences);
Console.WriteLine($"Saved model to {modelFileName}");
Console.WriteLine($"Saved model to {Settings.ModelDir}");
return true;
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model)
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
Init(meta);
var sentence = doc.Sentences.Select(s => new Sentence
{
@ -59,16 +49,37 @@ namespace BotSharp.Core.Engines.BotSharp
}).First();
var result = classifier.Classify(sentence);
var result = _classifier.Classify(sentence);
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "BotSharpNBayesClassifier",
Classifier = "BotSharpIntentClassifier",
Label = result.First().Item1,
Confidence = (decimal)result.First().Item2
};
return true;
}
private void Init(PipeModel meta)
{
if (_classifier == null)
{
meta.Model = "intent.model";
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
var options = new ClassifyOptions
{
ModelFilePath = modelFileName
};
_classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
string classifierName = Configuration.GetValue<String>($"classifer");
_classifier.GetClassifer(classifierName);
}
}
}
}

6
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -44,7 +44,7 @@ namespace BotSharp.NLP.UnitTest
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
Dimension = 100
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.7M);
classifier.Train(dataset.Item1);
@ -73,7 +73,7 @@ namespace BotSharp.NLP.UnitTest
{
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Gender")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, WordFeatureExtractor>(options, SupportedLanguage.English);
var classifier = new ClassifierFactory<WordFeatureExtractor>(options, SupportedLanguage.English);
var corpus = GetLabeledCorpus(options);
@ -159,7 +159,7 @@ namespace BotSharp.NLP.UnitTest
ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"),
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.7M);
classifier.Train(dataset.Item1);

23
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -3,17 +3,17 @@ using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
namespace BotSharp.NLP.Classify
{
public class ClassifierFactory<IClassify, IFeatureExtractor>
where IClassify : IClassifier, new()
public class ClassifierFactory<IFeatureExtractor>
where IFeatureExtractor : ITextFeatureExtractor, new()
{
private SupportedLanguage _lang;
private IClassify _classifier;
private IClassifier _classifier;
private ClassifyOptions _options;
@ -23,10 +23,25 @@ namespace BotSharp.NLP.Classify
{
_lang = lang;
_options = options;
_classifier = new IClassify();
featureExtractor = new IFeatureExtractor();
}
public IClassifier GetClassifer(string name)
{
List<Type> types = new List<Type>();
types.AddRange(Assembly.Load(new AssemblyName("BotSharp.Core"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
types.AddRange(Assembly.Load(new AssemblyName("BotSharp.NLP"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
Type type = types.FirstOrDefault(x => x.Name == name);
var instance = (IClassifier)Activator.CreateInstance(type);
return _classifier = instance;
}
public void Train(List<Sentence> sentences)
{
_classifier.Train(sentences, _options);

28
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -22,6 +22,8 @@ using System.IO;
using System.Linq;
using System.Text;
using BotSharp.Algorithm.Features;
using BotSharp.NLP.Featuring;
using BotSharp.NLP.Txt2Vec;
using SVM.BotSharp.MachineLearning;
using Txt2Vec;
@ -32,6 +34,8 @@ namespace BotSharp.NLP.Classify
/// </summary>
public class SVMClassifier : IClassifier
{
private List<string> words;
public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options)
{
Problem predict = new Problem();
@ -50,7 +54,26 @@ namespace BotSharp.NLP.Classify
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
// SVMClassifierTrain(featureSets, options);
var tfidf = new TfIdfFeatureExtractor();
tfidf.Dimension = options.Dimension;
tfidf.Sentences = sentences;
tfidf.CalBasedOnCategory();
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
encoder.Words = tfidf.Keywords();
words = encoder.EncodeAll();
var featureSets = new List<FeaturesWithLabel>();
sentences.ForEach(sent =>
{
var fl = new FeaturesWithLabel();
fl.Label = sent.Label;
fl.Features = sent.Words.Select(x => new Feature(words.IndexOf(x.Lemma).ToString(), words.Contains(x.Lemma) ? "1" : "0")).ToList();
featureSets.Add(fl);
});
SVMClassifierTrain(featureSets, options);
}
public List<Tuple<string, double>> Classify(Sentence sentence, ClassifyOptions options)
@ -93,10 +116,11 @@ namespace BotSharp.NLP.Classify
public List<double> GetLabels(List<FeaturesWithLabel> featureSets)
{
var categories = featureSets.Select(x => x.Label).Distinct().OrderBy(x => x).ToList();
List<double> labels = new List<double>();
foreach (var labelFeatureSet in featureSets)
{
labels.Add(double.Parse(labelFeatureSet.Label));
labels.Add(double.Parse(categories.IndexOf(labelFeatureSet.Label).ToString()));
}
return labels;

31
BotSharp.WebHost/Settings/bot.json
View file

@ -7,28 +7,29 @@
},
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier"
"train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier",
"predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier"
},
"BotSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
},
"BotSharpNBayesClassifier": {
},
"BotSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
},
"BotSharpIntentClassifier": {
"classifer": "NaiveBayesClassifier"
},
"BotSharpSVMClassifier": {
"wordvec": ""
},
"BotSharpTagger": {
"tagger": "NGramTagger"
"tagger": "NGramTagger"
},
"BotSharpCRFNer": {
"template": "|App_Data|CRFLite/template.en"
},
"BotSharpCRFNer": {
"template": "|App_Data|CRFLite/template.en"
},
"CRFsuiteEntityRecognizer": {
"fields": "y w pos chk",