diff --git a/.gitignore b/.gitignore index 52ba8fc0..4298cb56 100644 --- a/.gitignore +++ b/.gitignore @@ -301,3 +301,5 @@ __pycache__/ /BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/metadata.json /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.parsed.txt /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.corpus.txt +/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318 +/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318 diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index b102c18f..ed56e360 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -71,6 +71,18 @@ namespace BotSharp.Core.Engines Pipeline = new List() { pipeModel } }; + var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); + if (!Directory.Exists(dirTrain)) + { + Directory.CreateDirectory(dirTrain); + } + + var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); + if (!Directory.Exists(dirModel)) + { + Directory.CreateDirectory(dirModel); + } + // pipe process var pipelines = provider.Configuration.GetSection($"Pipe").Value .Split(',') diff --git a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs index 8d082818..673d27da 100644 --- a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs @@ -26,26 +26,17 @@ namespace BotSharp.Core.Engines.CRFsuite var dc = new DefaultDataContextLoader().GetDefaultDc(); var corpus = agent.Corpus; + meta.Model = "ner-crf.model"; + List> tokens = data["Tokens"].ToObject>>(); List> userSays = corpus.UserSays; List> list = new List>(); var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); - if (!Directory.Exists(dirTrain)) - { - Directory.CreateDirectory(dirTrain); - } - var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); - if (!Directory.Exists(dirModel)) - { - Directory.CreateDirectory(dirModel); - } - string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt"); string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt"); - string modelFileName = Path.Join(dirModel, $"ner-crf.model"); - string logFileName = Path.Join(dirTrain, $"ner-crf.log.txt"); + string modelFileName = Path.Join(dirModel, meta.Model); using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) { @@ -80,7 +71,6 @@ namespace BotSharp.Core.Engines.CRFsuite Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); - meta.Meta["model"] = $"ner-crf.model"; meta.Meta["fields"] = fields; meta.Meta["uniFeatures"] = uniFeatures; meta.Meta["biFeatures"] = biFeatures; diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs new file mode 100644 index 00000000..3d442da1 --- /dev/null +++ b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs @@ -0,0 +1,49 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using DotNetToolkit; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.Classifiers +{ + public class FasttextClassifier : INlpPipeline + { + public IConfiguration Configuration { get; set; } + + public Task Predict(Agent agent, JObject data, PipeModel meta) + { + throw new NotImplementedException(); + } + + public async Task Train(Agent agent, JObject data, PipeModel meta) + { + meta.Model = "classification-fasttext.model"; + var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); + var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); + var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); + + string parsedTrainingDataFileName = Path.Join(dirTrain, $"classification-fasttext.parsed.txt"); + string modelFileName = Path.Join(dirModel, meta.Model); + + // assemble corpus + StringBuilder corpus = new StringBuilder(); + agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}")); + + File.WriteAllText(parsedTrainingDataFileName, corpus.ToString()); + + var output = CmdHelper.Run(Path.Join(algorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}"); + + Console.WriteLine($"Saved model to {modelFileName}"); + meta.Meta = new JObject(); + meta.Meta["compiled at"] = "Aug 3, 2018"; + + + return true; + } + } +} diff --git a/BotSharp.Core/Engines/PipeModel.cs b/BotSharp.Core/Engines/PipeModel.cs index 5791b7ac..d971408a 100644 --- a/BotSharp.Core/Engines/PipeModel.cs +++ b/BotSharp.Core/Engines/PipeModel.cs @@ -19,6 +19,8 @@ namespace BotSharp.Core.Engines public DateTime Time { get; set; } + public string Model { get; set; } + /// /// Extra meta data according to pipe /// diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index e5991559..f61d9810 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -22,6 +22,8 @@ namespace BotSharp.Core.Engines.SpaCy var response = client.Execute(request); meta.Meta = JObject.FromObject(response.Data); + meta.Meta["models"] = null; + meta.Model = response.Data.Models; return response.IsSuccessful; } diff --git a/BotSharp.WebHost/Algorithms/fasttext.exe b/BotSharp.WebHost/Algorithms/fasttext.exe new file mode 100644 index 00000000..0ffcdd4c Binary files /dev/null and b/BotSharp.WebHost/Algorithms/fasttext.exe differ diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index ed0fcc38..055b5818 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -72,7 +72,6 @@ - diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index 959f4f61..3e721a5e 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -9,7 +9,7 @@ "SpaCyProvider": { "Url": "http://10.2.21.200:5005" }, - "Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer", + "Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier", "CRFsuiteEntityRecognizer": { "fields": "y w pos chk", "uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs", diff --git a/README.md b/README.md index 4a61e83c..b23b17e1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ### The Open Source AI Chatbot Platform Builder for Enterprise ### -**BotSharp** is an open source AI chatbot platform builder not only a bot builder, it's a complete out of box toolkit for building up a functional chabot platform which utilize aritifical intelligence. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. +**BotSharp** is an open source AI chatbot platform builder's framework. It's not only a bot builder but also a complete out of box toolkit for building up a functional chabot platform which utilize aritifical intelligence. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. Why we do this? because we all know python is not friendly programming language for enterprise developers, it's not only because it's low performance but also it's a type weak language, it will be a disater if you use python to build your bussiness system.