SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Added fasttext classification trainer pipeline.

Browse source
This commit is contained in:
haiping008@gmail.com 2018-08-09 15:22:53 -05:00
parent ec67e768a0
commit 2c35f6177a
10 changed files with 72 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -301,3 +301,5 @@ __pycache__/
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/metadata.json
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.parsed.txt
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.corpus.txt
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318

12
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -71,6 +71,18 @@ namespace BotSharp.Core.Engines
Pipeline = new List<PipeModel>() { pipeModel }
};
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
if (!Directory.Exists(dirTrain))
{
Directory.CreateDirectory(dirTrain);
}
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
if (!Directory.Exists(dirModel))
{
Directory.CreateDirectory(dirModel);
}
// pipe process
var pipelines = provider.Configuration.GetSection($"Pipe").Value
.Split(',')

16
BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs
View file

@ -26,26 +26,17 @@ namespace BotSharp.Core.Engines.CRFsuite
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
meta.Model = "ner-crf.model";
List<List<NlpToken>> tokens = data["Tokens"].ToObject<List<List<NlpToken>>>();
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
List<List<TrainingData>> list = new List<List<TrainingData>>();
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
if (!Directory.Exists(dirTrain))
{
Directory.CreateDirectory(dirTrain);
}
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
if (!Directory.Exists(dirModel))
{
Directory.CreateDirectory(dirModel);
}
string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt");
string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt");
string modelFileName = Path.Join(dirModel, $"ner-crf.model");
string logFileName = Path.Join(dirTrain, $"ner-crf.log.txt");
string modelFileName = Path.Join(dirModel, meta.Model);
using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
{
@ -80,7 +71,6 @@ namespace BotSharp.Core.Engines.CRFsuite
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["model"] = $"ner-crf.model";
meta.Meta["fields"] = fields;
meta.Meta["uniFeatures"] = uniFeatures;
meta.Meta["biFeatures"] = biFeatures;

49
BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs Normal file
View file

@ -0,0 +1,49 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.Classifiers
{
public class FasttextClassifier : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
throw new NotImplementedException();
}
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
meta.Model = "classification-fasttext.model";
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
string parsedTrainingDataFileName = Path.Join(dirTrain, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Join(dirModel, meta.Model);
// assemble corpus
StringBuilder corpus = new StringBuilder();
agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}"));
File.WriteAllText(parsedTrainingDataFileName, corpus.ToString());
var output = CmdHelper.Run(Path.Join(algorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 3, 2018";
return true;
}
}
}

2
BotSharp.Core/Engines/PipeModel.cs
View file

@ -19,6 +19,8 @@ namespace BotSharp.Core.Engines
public DateTime Time { get; set; }
public string Model { get; set; }
/// <summary>
/// Extra meta data according to pipe
/// </summary>

2
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -22,6 +22,8 @@ namespace BotSharp.Core.Engines.SpaCy
var response = client.Execute<Result>(request);
meta.Meta = JObject.FromObject(response.Data);
meta.Meta["models"] = null;
meta.Model = response.Data.Models;
return response.IsSuccessful;
}

BIN
BotSharp.WebHost/Algorithms/fasttext.exe Normal file
View file

Binary file not shown.

1
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -72,7 +72,6 @@
<ItemGroup>
<Folder Include="App_Data\DbInitializer\Agents\Rasa\" />
<Folder Include="Algorithms\" />
</ItemGroup>
<ItemGroup>

2
BotSharp.WebHost/Settings/bot.json
View file

@ -9,7 +9,7 @@
"SpaCyProvider": {
"Url": "http://10.2.21.200:5005"
},
"Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer",
"Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"CRFsuiteEntityRecognizer": {
"fields": "y w pos chk",
"uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs",

2
README.md
View file

@ -2,7 +2,7 @@
### The Open Source AI Chatbot Platform Builder for Enterprise
###
**BotSharp** is an open source AI chatbot platform builder not only a bot builder, it's a complete out of box toolkit for building up a functional chabot platform which utilize aritifical intelligence. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope.
**BotSharp** is an open source AI chatbot platform builder's framework. It's not only a bot builder but also a complete out of box toolkit for building up a functional chabot platform which utilize aritifical intelligence. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope.
Why we do this? because we all know python is not friendly programming language for enterprise developers, it's not only because it's low performance but also it's a type weak language, it will be a disater if you use python to build your bussiness system.