From 343a300680f78fa89ada544536bd98d2afc71689 Mon Sep 17 00:00:00 2001 From: Bo Peng Date: Mon, 13 Aug 2018 16:52:17 -0500 Subject: [PATCH] optimised the SpaCyTokenizer API . Right now it will post a list of sentence and call spacy provider once to get all the tokens and poses. --- BotSharp.Core/Engines/BotTrainer.cs | 4 +- BotSharp.Core/Engines/Nltk/NltkTokenizer.cs | 37 +++++++++------ BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs | 45 ++++++++++++------- BotSharp.WebHost/Settings/bot.json | 10 ++--- 4 files changed, 60 insertions(+), 36 deletions(-) diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 9c6cadb4..94d475e5 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -47,7 +47,7 @@ namespace BotSharp.Core.Engines var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies"); var platform = config.GetSection($"BotPlatform").Value; string providerName = config.GetSection($"{platform}:Provider").Value; - var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpTrain; + var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider; provider.Configuration = config.GetSection(platform); var pipeModel = new PipeModel @@ -58,7 +58,7 @@ namespace BotSharp.Core.Engines Time = DateTime.UtcNow }; - await provider.Train(agent, data, pipeModel); + await provider.Load(agent, pipeModel); var meta = new ModelMetaData { diff --git a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs index 6191ae03..a1cfe329 100644 --- a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs +++ b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs @@ -23,7 +23,7 @@ namespace BotSharp.Core.Engines.SpaCy public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value); - var request = new RestRequest("nltktokenizesentences", Method.GET); + var request = new RestRequest("nltktokenizesentences", Method.POST); List> tokens = new List>(); Boolean res = true; var dc = new DefaultDataContextLoader().GetDefaultDc(); @@ -33,19 +33,24 @@ namespace BotSharp.Core.Engines.SpaCy List sentencesList = new List(); corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text)); - - - request.RequestFormat = DataFormat.Json; - request.AddParameter("application/json", JsonConvert.SerializeObject(new { sentences = sentencesList})); + request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody); var response = client.Execute(request); + tokens = response.Data.TokensList; - - - + for (int i = 0; i < sentencesList.Count; i++) + { + doc.Sentences.Add(new NlpDocSentence + { + Tokens = tokens[i], + Text = sentencesList[i] + }); + } + res = res && response.IsSuccessful; + return res; /* corpus.UserSays.ForEach(usersay => { Console.WriteLine(usersay.Text); @@ -61,12 +66,8 @@ namespace BotSharp.Core.Engines.SpaCy }); res = res && response.IsSuccessful; - }); */ - - - return res; } public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) @@ -91,7 +92,17 @@ namespace BotSharp.Core.Engines.SpaCy private class Result { - public List> Tokens { get; set; } + public List> TokensList { get; set; } + } + + private class Documents + { + public List Sentences { get; set; } + + public Documents(List sentences) + { + this.Sentences = sentences; + } } } } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index 8a834d1c..cdabaa69 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -4,6 +4,7 @@ using BotSharp.Core.Models; using BotSharp.MachineLearning.NLP; using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using RestSharp; using System; @@ -22,31 +23,33 @@ namespace BotSharp.Core.Engines.SpaCy public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); - var request = new RestRequest("tokenizer", Method.GET); + var request = new RestRequest("spacytokenizesentences", Method.POST); List> tokens = new List>(); Boolean res = true; var dc = new DefaultDataContextLoader().GetDefaultDc(); var corpus = agent.Corpus; doc.Sentences = new List(); + List sentencesList = new List(); + corpus.UserSays.ForEach(usersay => sentencesList.Add(usersay.Text)); - corpus.UserSays.ForEach(usersay => { - Console.WriteLine(usersay.Text); - request.AddParameter("text", usersay.Text); - var response = client.Execute(request); - - tokens.Add(response.Data.Tokens); + request.RequestFormat = DataFormat.Json; + request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody); + + var response = client.Execute(request); + + tokens = response.Data.TokensList; + + for (int i = 0; i < sentencesList.Count; i++) + { doc.Sentences.Add(new NlpDocSentence { - Tokens = response.Data.Tokens, - Text = usersay.Text + Tokens = tokens[i], + Text = sentencesList[i] }); - - res = res && response.IsSuccessful; - - }); - + } + res = res && response.IsSuccessful; return res; } @@ -61,7 +64,7 @@ namespace BotSharp.Core.Engines.SpaCy request.AddParameter("text", doc.Sentences[0].Text); var response = client.Execute(request); - tokens.Add(response.Data.Tokens); + tokens = response.Data.TokensList; res = res && response.IsSuccessful; @@ -72,7 +75,17 @@ namespace BotSharp.Core.Engines.SpaCy private class Result { - public List Tokens { get; set; } + public List> TokensList { get; set; } + } + + private class Documents + { + public List Sentences { get; set; } + + public Documents(List sentences) + { + this.Sentences = sentences; + } } } } diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index cea88c28..c027c4a1 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -1,20 +1,20 @@ { "RasaAi": { - "url": "http://localhost:5000" + "url": "http://10.21.2.200:5000" }, "BotSharpAi": { "Lang": "en", "Provider": "SpaCyProvider", "SpaCyProvider": { - "url": "http://localhost:5005" + "url": "http://10.2.21.200:5005" }, "NltkProvider": { - "url": "http://localhost:5005" + "url": "http://10.2.21.200:5005" }, "Pipe": { - "train": "NltkTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier", - "predict": "NltkTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer" + "train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier", + "predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer" }, "SpaCyTokenizer": {