SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

optimised the SpaCyTokenizer API . Right now it will post a list of sentence and call spacy provider once to get all the tokens and poses.

Browse source
This commit is contained in:
Bo Peng 2018-08-13 16:52:17 -05:00
parent 97d644e5fe
commit 343a300680
4 changed files with 60 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -47,7 +47,7 @@ namespace BotSharp.Core.Engines
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var platform = config.GetSection($"BotPlatform").Value;
string providerName = config.GetSection($"{platform}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpTrain;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(platform);
var pipeModel = new PipeModel
@ -58,7 +58,7 @@ namespace BotSharp.Core.Engines
Time = DateTime.UtcNow
};
await provider.Train(agent, data, pipeModel);
await provider.Load(agent, pipeModel);
var meta = new ModelMetaData
{

37
BotSharp.Core/Engines/Nltk/NltkTokenizer.cs
View file

@ -23,7 +23,7 @@ namespace BotSharp.Core.Engines.SpaCy
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
var request = new RestRequest("nltktokenizesentences", Method.GET);
var request = new RestRequest("nltktokenizesentences", Method.POST);
List<List<NlpToken>> tokens = new List<List<NlpToken>>();
Boolean res = true;
var dc = new DefaultDataContextLoader().GetDefaultDc();
@ -33,19 +33,24 @@ namespace BotSharp.Core.Engines.SpaCy
List<string> sentencesList = new List<string>();
corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text));
request.RequestFormat = DataFormat.Json;
request.AddParameter("application/json", JsonConvert.SerializeObject(new { sentences = sentencesList}));
request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody);
var response = client.Execute<Result>(request);
tokens = response.Data.TokensList;
for (int i = 0; i < sentencesList.Count; i++)
{
doc.Sentences.Add(new NlpDocSentence
{
Tokens = tokens[i],
Text = sentencesList[i]
});
}
res = res && response.IsSuccessful;
return res;
/*
corpus.UserSays.ForEach(usersay => {
Console.WriteLine(usersay.Text);
@ -61,12 +66,8 @@ namespace BotSharp.Core.Engines.SpaCy
});
res = res && response.IsSuccessful;
});
*/
return res;
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
@ -91,7 +92,17 @@ namespace BotSharp.Core.Engines.SpaCy
private class Result
{
public List<List<NlpToken>> Tokens { get; set; }
public List<List<NlpToken>> TokensList { get; set; }
}
private class Documents
{
public List<string> Sentences { get; set; }
public Documents(List<string> sentences)
{
this.Sentences = sentences;
}
}
}
}

45
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -4,6 +4,7 @@ using BotSharp.Core.Models;
using BotSharp.MachineLearning.NLP;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
@ -22,31 +23,33 @@ namespace BotSharp.Core.Engines.SpaCy
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tokenizer", Method.GET);
var request = new RestRequest("spacytokenizesentences", Method.POST);
List<List<NlpToken>> tokens = new List<List<NlpToken>>();
Boolean res = true;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
doc.Sentences = new List<NlpDocSentence>();
List<string> sentencesList = new List<string>();
corpus.UserSays.ForEach(usersay => sentencesList.Add(usersay.Text));
corpus.UserSays.ForEach(usersay => {
Console.WriteLine(usersay.Text);
request.AddParameter("text", usersay.Text);
var response = client.Execute<Result>(request);
tokens.Add(response.Data.Tokens);
request.RequestFormat = DataFormat.Json;
request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody);
var response = client.Execute<Result>(request);
tokens = response.Data.TokensList;
for (int i = 0; i < sentencesList.Count; i++)
{
doc.Sentences.Add(new NlpDocSentence
{
Tokens = response.Data.Tokens,
Text = usersay.Text
Tokens = tokens[i],
Text = sentencesList[i]
});
res = res && response.IsSuccessful;
});
}
res = res && response.IsSuccessful;
return res;
}
@ -61,7 +64,7 @@ namespace BotSharp.Core.Engines.SpaCy
request.AddParameter("text", doc.Sentences[0].Text);
var response = client.Execute<Result>(request);
tokens.Add(response.Data.Tokens);
tokens = response.Data.TokensList;
res = res && response.IsSuccessful;
@ -72,7 +75,17 @@ namespace BotSharp.Core.Engines.SpaCy
private class Result
{
public List<NlpToken> Tokens { get; set; }
public List<List<NlpToken>> TokensList { get; set; }
}
private class Documents
{
public List<string> Sentences { get; set; }
public Documents(List<string> sentences)
{
this.Sentences = sentences;
}
}
}
}

10
BotSharp.WebHost/Settings/bot.json
View file

@ -1,20 +1,20 @@
{
"RasaAi": {
"url": "http://localhost:5000"
"url": "http://10.21.2.200:5000"
},
"BotSharpAi": {
"Lang": "en",
"Provider": "SpaCyProvider",
"SpaCyProvider": {
"url": "http://localhost:5005"
"url": "http://10.2.21.200:5005"
},
"NltkProvider": {
"url": "http://localhost:5005"
"url": "http://10.2.21.200:5005"
},
"Pipe": {
"train": "NltkTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "NltkTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer"
"train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer"
},
"SpaCyTokenizer": {