From f70176b1845e7380053c1b1367774112f61368ea Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 29 Aug 2018 17:06:27 -0500 Subject: [PATCH] Fix entity chunk issue. --- BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Engines/BotSharp/BotSharpTokenizer.cs | 7 ++- .../Engines/NERs/CRFsuiteEntityRecognizer.cs | 47 +++++++++++-------- BotSharp.Core/Engines/NlpDoc.cs | 4 +- BotSharp.RestApi/Rasa/ParseController.cs | 6 +++ 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index b83e0ac7..2c5cf3ad 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -50,7 +50,7 @@ - + diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs index 179d6432..0bba34a9 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs @@ -20,12 +20,15 @@ namespace BotSharp.Core.Engines.BotSharp { _tokenizer = new TokenizerFactory(new TokenizationOptions { - Pattern = RegexTokenizer.WORD_PUNC + Pattern = RegexTokenizer.WORD_PUNC, + SpecialWords = new List { "'s" } }, SupportedLanguage.English); } public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { + doc.Tokenizer = this; + // same as train doc.Sentences.ForEach(snt => { @@ -37,7 +40,9 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { + doc.Tokenizer = this; doc.Sentences = new List(); + agent.Corpus.UserSays.ForEach(say => { doc.Sentences.Add(new NlpDocSentence diff --git a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs index 036d70c4..0d3d44ed 100644 --- a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs @@ -56,7 +56,7 @@ namespace BotSharp.Core.Engines.NERs { for (int i = 0; i < doc.Sentences.Count; i++) { - List curLine = Merge(doc.Sentences[i].Tokens, userSays[i].Entities); + List curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities); curLine.ForEach(trainingData => { string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk }; @@ -89,7 +89,7 @@ namespace BotSharp.Core.Engines.NERs return true; } - public List Merge(List tokens, List entities) + public List Merge(NlpDoc doc, List tokens, List entities) { List trainingTuple = new List(); HashSet entityWordBag = new HashSet(); @@ -104,7 +104,10 @@ namespace BotSharp.Core.Engines.NERs entities.ForEach(entity => { if (!entityFinded) { - string[] words = entity.Value.Split(' '); + var vDoc = new NlpDoc { Sentences = new List { new NlpDocSentence { Text = entity.Value } } }; + doc.Tokenizer.Predict(null, vDoc, null); + string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray(); + for (int j = 0; j < words.Length; j++) { if (tokens[i + j].Text == words[j]) @@ -214,7 +217,7 @@ namespace BotSharp.Core.Engines.NERs }); } - List unionedEntities = MergeEntity(entities); + List unionedEntities = MergeEntity(doc.Sentences[0].Text, entities); doc.Sentences[0].Entities = unionedEntities.Where(x => x.Entity != "O").ToList(); @@ -230,29 +233,35 @@ namespace BotSharp.Core.Engines.NERs return true; } - public List MergeEntity (List tokens) + public List MergeEntity (string sentence, List tokens) { List res = new List(); for (int i = 0; i < tokens.Count ; i++) { NlpEntity nlpEntity = new NlpEntity(); - StringBuilder unionValue = new StringBuilder(tokens[i].Value); - StringBuilder unionEntity = new StringBuilder(tokens[i].Entity); - decimal unoinConfidence = tokens[i].Confidence; + var current = tokens[i]; - int j = i + 1; - while (j < tokens.Count && tokens[j].Entity == tokens[i].Entity && tokens[i].Entity != "O") + if (current.Entity != "O") { - unionValue.Append(" " + tokens[j].Value); - j++; + nlpEntity = current.ToObject(); + // greedy search until next entity + int j = 0; + for (j = i + 1; j < tokens.Count; j++) + { + var next = tokens[j]; + if (current.Entity == next.Entity) + { + i = j; + nlpEntity.Value = sentence.Substring(current.Start, next.End - current.Start + 1); + } + else + { + break; + } + } + + res.Add(nlpEntity); } - nlpEntity.Entity = unionEntity.ToString(); - nlpEntity.Start = tokens[i].Start; - nlpEntity.Value = unionValue.ToString(); - nlpEntity.Confidence = unoinConfidence; - nlpEntity.Extrator = tokens[i].Extrator; - res.Add(nlpEntity); - i = j - 1; } return res; } diff --git a/BotSharp.Core/Engines/NlpDoc.cs b/BotSharp.Core/Engines/NlpDoc.cs index 05421ce6..5538601b 100644 --- a/BotSharp.Core/Engines/NlpDoc.cs +++ b/BotSharp.Core/Engines/NlpDoc.cs @@ -1,4 +1,5 @@ -using BotSharp.Models.NLP; +using BotSharp.Core.Abstractions; +using BotSharp.Models.NLP; using BotSharp.NLP.Tokenize; using System; using System.Collections.Generic; @@ -8,6 +9,7 @@ namespace BotSharp.Core.Engines { public class NlpDoc { + public INlpPredict Tokenizer { get; set; } public List Sentences { get; set; } } diff --git a/BotSharp.RestApi/Rasa/ParseController.cs b/BotSharp.RestApi/Rasa/ParseController.cs index 4ddde6c2..0d4d76be 100644 --- a/BotSharp.RestApi/Rasa/ParseController.cs +++ b/BotSharp.RestApi/Rasa/ParseController.cs @@ -54,6 +54,12 @@ namespace BotSharp.RestApi.Rasa // Load agent var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project); + + if (String.IsNullOrEmpty(request.Model)) + { + request.Model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last(); + } + var modelPath = Path.Combine(projectPath, request.Model); var agent = _platform.LoadAgentFromFile(modelPath);