diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs new file mode 100644 index 00000000..61418c4c --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -0,0 +1,57 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP; +using BotSharp.NLP.Classify; +using BotSharp.NLP.Txt2Vec; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpNBayesClassifier : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + meta.Model = "classification-nb.model"; + + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + + var encoder = new OneHotEncoder(); + encoder.Sentences = doc.Sentences.Select(x => new NLP.Sentence + { + Label = x.Intent.Label, + Text = x.Text, + Words = x.Tokens + }).ToList(); + encoder.EncodeAll(); + + var options = new ClassifyOptions + { + TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + classifier.Train(encoder.Sentences); + + Console.WriteLine($"Saved model to {modelFileName}"); + meta.Meta = new JObject(); + meta.Meta["compiled at"] = "Sep 12, 2018"; + + return true; + } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + return true; + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs index 0bba34a9..40d2476a 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs @@ -1,5 +1,6 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; +using BotSharp.Core.Intents; using BotSharp.NLP; using BotSharp.NLP.Tokenize; using Microsoft.Extensions.Configuration; @@ -14,14 +15,12 @@ namespace BotSharp.Core.Engines.BotSharp { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } - private TokenizerFactory _tokenizer; + private TokenizerFactory _tokenizer; public BotSharpTokenizer() { - _tokenizer = new TokenizerFactory(new TokenizationOptions + _tokenizer = new TokenizerFactory(new TokenizationOptions { - Pattern = RegexTokenizer.WORD_PUNC, - SpecialWords = new List { "'s" } }, SupportedLanguage.English); } @@ -48,7 +47,8 @@ namespace BotSharp.Core.Engines.BotSharp doc.Sentences.Add(new NlpDocSentence { Tokens = _tokenizer.Tokenize(say.Text), - Text = say.Text + Text = say.Text, + Intent = new TextClassificationResult { Label = say.Intent } }); }); diff --git a/Settings/bot.json b/Settings/bot.json index 013beac1..34e48d70 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -7,10 +7,13 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpSVMClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier" }, - + + "BotSharpNBayesClassifier": { + }, + "BotSharpSVMClassifier": { "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" },