From f42a42d34e69cfab6d288f1bca18d832c140eb2d Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 22 Aug 2018 16:36:39 -0500 Subject: [PATCH] Add BotSharpTokenizer, BotSharpTagger for BotSharpAi pipeline. --- BotSharp.Core.UnitTest/AgentTest.cs | 3 +- BotSharp.Core/Agents/Agent.cs | 7 --- .../Engines/BotSharp/BotSharpProvider.cs | 24 ++++++++++ .../Engines/BotSharp/BotSharpTagger.cs | 46 ++++++++++++++++++ .../Engines/BotSharp/BotSharpTokenizer.cs | 47 +++++++++++++++++++ .../Dialogflow/AgentImporterInDialogflow.cs | 4 -- .../Engines/Sebis/AgentImporterInSebis.cs | 4 -- BotSharp.WebHost/BotSharp.WebHost.csproj | 2 +- BotSharp.WebHost/Program.cs | 5 +- BotSharp.WebHost/Settings/bot.json | 25 +++++----- BotSharp.WebHost/Startup.cs | 5 +- 11 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs create mode 100644 BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs create mode 100644 BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs diff --git a/BotSharp.Core.UnitTest/AgentTest.cs b/BotSharp.Core.UnitTest/AgentTest.cs index 58cce66a..000057c8 100644 --- a/BotSharp.Core.UnitTest/AgentTest.cs +++ b/BotSharp.Core.UnitTest/AgentTest.cs @@ -24,8 +24,7 @@ namespace BotSharp.Core.UnitTest var agent = new Agent { Id = BOT_ID, - Language = "en", - UserId = Guid.NewGuid().ToString() + Language = "en" }; var rasa = new RasaAi(); //int row = dc.DbTran(() => rasa.agent.SaveAgent(dc)); diff --git a/BotSharp.Core/Agents/Agent.cs b/BotSharp.Core/Agents/Agent.cs index 698d6d90..e83027f5 100644 --- a/BotSharp.Core/Agents/Agent.cs +++ b/BotSharp.Core/Agents/Agent.cs @@ -44,13 +44,6 @@ namespace BotSharp.Core.Agents [StringLength(32)] public String DeveloperAccessToken { get; set; } - /// - /// Who created this bot - /// - [Required] - [StringLength(36)] - public String UserId { get; set; } - [ForeignKey("AgentId")] public List Intents { get; set; } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs b/BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs new file mode 100644 index 00000000..17e6d07c --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs @@ -0,0 +1,24 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpProvider : INlpProvider + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Load(Agent agent, PipeModel meta) + { + meta.Meta = JObject.FromObject(new { version = "0.1.0" }); + + return true; + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs new file mode 100644 index 00000000..caf9f2e2 --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs @@ -0,0 +1,46 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP; +using BotSharp.NLP.Corpus; +using BotSharp.NLP.Tag; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpTagger : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + throw new NotImplementedException(); + } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + string dataDir = Path.Combine(Configuration.GetValue("BotSharpTagger:dataDir"), "CoNLL"); + var data = new CoNLLReader().Read(new ReaderOptions + { + DataDir = dataDir, + FileName = "conll2000_chunking_train.txt" + }); + + var tagger = new TaggerFactory(new TagOptions + { + NGram = 1, + Tag = "NN", + Corpus = data + }, SupportedLanguage.English); + + doc.Sentences.ForEach(x => tagger.Tag(new Sentence { Words = x.Tokens })); + + return true; + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs new file mode 100644 index 00000000..b0004797 --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs @@ -0,0 +1,47 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.NLP; +using BotSharp.NLP.Tokenize; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpTokenizer : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + return true; + } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + List> tokens = new List>(); + var corpus = agent.Corpus; + + var tokenizer = new TokenizerFactory(new TokenizationOptions + { + Pattern = RegexTokenizer.WORD_PUNC + }, SupportedLanguage.English); + + doc.Sentences = new List(); + List sentencesList = new List(); + corpus.UserSays.ForEach(say => + { + doc.Sentences.Add(new NlpDocSentence + { + Tokens = tokenizer.Tokenize(say.Text), + Text = say.Text + }); + }); + + return true; + } + } +} diff --git a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs index 029324c4..3b643b4e 100644 --- a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs +++ b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs @@ -38,10 +38,6 @@ namespace BotSharp.Core.Engines var result = agent.ToObject(); result.ClientAccessToken = agentHeader.ClientAccessToken; result.DeveloperAccessToken = agentHeader.DeveloperAccessToken; - if(agentHeader.UserId != null) - { - result.UserId = agentHeader.UserId; - } result.MlConfig = agent.ToObject(); result.MlConfig.MinConfidence = agent.MlMinConfidence; diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index ef53368d..a0c40150 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -39,10 +39,6 @@ namespace BotSharp.Core.Engines var result = agent.ToObject(); result.ClientAccessToken = agentHeader.ClientAccessToken; result.DeveloperAccessToken = agentHeader.DeveloperAccessToken; - if(agentHeader.UserId != null) - { - result.UserId = agentHeader.UserId; - } return result; } diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index 6f5735a7..2f15a870 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -7,7 +7,7 @@ - TRACE;DEBUG;MODE_RASA + TRACE;DEBUG;RASA_UI diff --git a/BotSharp.WebHost/Program.cs b/BotSharp.WebHost/Program.cs index 82d23ba5..b09272a8 100644 --- a/BotSharp.WebHost/Program.cs +++ b/BotSharp.WebHost/Program.cs @@ -17,15 +17,14 @@ namespace BotSharp.WebHost Microsoft.AspNetCore.WebHost.CreateDefaultBuilder(args) .ConfigureAppConfiguration((hostingContext, config) => { - var env = hostingContext.HostingEnvironment; - var settings = Directory.GetFiles($"{env.ContentRootPath}{Path.DirectorySeparatorChar}Settings", "*.json"); + var settings = Directory.GetFiles(Path.Combine(env.ContentRootPath, "Settings"), "*.json"); settings.ToList().ForEach(setting => { config.AddJsonFile(setting, optional: false, reloadOnChange: true); }); }) -#if MODE_RASA +#if RASA_UI .UseUrls("http://0.0.0.0:5000") #else .UseUrls("http://0.0.0.0:3112") diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index 828286ba..c89584d5 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -1,29 +1,26 @@ { - "RasaAi": { - "url": "http://10.2.21.200:5000" - }, "BotSharpAi": { "Lang": "en", - "Provider": "SpaCyProvider", - "SpaCyProvider": { - "url": "http://localhost:5005" - }, - "NltkProvider": { - "url": "http://localhost:5005" - }, - "Pipe": { - "train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier", - "predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer" + "Provider": "BotSharpProvider", + "BotSharpProvider": { }, - "SpaCyTokenizer": { + "Pipe": { + "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier", + "predict": "BotSharpTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer, FasttextClassifier" }, + + "BotSharpTagger": { + "dataDir": "D:\\Projects\\BotSharp.NLP\\Data" + }, + "CRFsuiteEntityRecognizer": { "fields": "y w pos chk", "uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs", "biFeatures": "w pos chk shaped type" }, + "WitAiEntityRecognizer": { "url": "https://api.wit.ai", "resource": "message", diff --git a/BotSharp.WebHost/Startup.cs b/BotSharp.WebHost/Startup.cs index 74d20aa3..e5359f55 100644 --- a/BotSharp.WebHost/Startup.cs +++ b/BotSharp.WebHost/Startup.cs @@ -108,10 +108,7 @@ namespace BotSharp.WebHost var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration"); context.Request.Headers["ClientAccessToken"] = token; - var dc = new DefaultDataContextLoader().GetDefaultDc(); - var userId = dc.Table().FirstOrDefault(x => x.ClientAccessToken == token)?.UserId; - - context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId); + // context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId); } await next.Invoke();