SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fix entity chunk issue.

Browse source
This commit is contained in:
Oceania2018 2018-08-29 17:06:27 -05:00
parent 0f6a101de2
commit f70176b184
5 changed files with 44 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.Core/BotSharp.Core.csproj
View file

@ -50,7 +50,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="BotSharp.NLP" Version="0.2.3" />
<PackageReference Include="BotSharp.NLP" Version="0.2.4" />
<PackageReference Include="DotNetToolkit" Version="1.6.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.9.1" />
<PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="2.1.1" />

7
BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs
View file

@ -20,12 +20,15 @@ namespace BotSharp.Core.Engines.BotSharp
{
_tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
Pattern = RegexTokenizer.WORD_PUNC,
SpecialWords = new List<string> { "'s" }
}, SupportedLanguage.English);
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
doc.Tokenizer = this;
// same as train
doc.Sentences.ForEach(snt =>
{
@ -37,7 +40,9 @@ namespace BotSharp.Core.Engines.BotSharp
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
doc.Tokenizer = this;
doc.Sentences = new List<NlpDocSentence>();
agent.Corpus.UserSays.ForEach(say =>
{
doc.Sentences.Add(new NlpDocSentence

47
BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs
View file

@ -56,7 +56,7 @@ namespace BotSharp.Core.Engines.NERs
{
for (int i = 0; i < doc.Sentences.Count; i++)
{
List<TrainingData> curLine = Merge(doc.Sentences[i].Tokens, userSays[i].Entities);
List<TrainingData> curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities);
curLine.ForEach(trainingData =>
{
string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk };
@ -89,7 +89,7 @@ namespace BotSharp.Core.Engines.NERs
return true;
}
public List<TrainingData> Merge(List<Token> tokens, List<TrainingIntentExpressionPart> entities)
public List<TrainingData> Merge(NlpDoc doc, List<Token> tokens, List<TrainingIntentExpressionPart> entities)
{
List<TrainingData> trainingTuple = new List<TrainingData>();
HashSet<String> entityWordBag = new HashSet<String>();
@ -104,7 +104,10 @@ namespace BotSharp.Core.Engines.NERs
entities.ForEach(entity => {
if (!entityFinded)
{
string[] words = entity.Value.Split(' ');
var vDoc = new NlpDoc { Sentences = new List<NlpDocSentence> { new NlpDocSentence { Text = entity.Value } } };
doc.Tokenizer.Predict(null, vDoc, null);
string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray();
for (int j = 0; j < words.Length; j++)
{
if (tokens[i + j].Text == words[j])
@ -214,7 +217,7 @@ namespace BotSharp.Core.Engines.NERs
});
}
List<NlpEntity> unionedEntities = MergeEntity(entities);
List<NlpEntity> unionedEntities = MergeEntity(doc.Sentences[0].Text, entities);
doc.Sentences[0].Entities = unionedEntities.Where(x => x.Entity != "O").ToList();
@ -230,29 +233,35 @@ namespace BotSharp.Core.Engines.NERs
return true;
}
public List<NlpEntity> MergeEntity (List<NlpEntity> tokens)
public List<NlpEntity> MergeEntity (string sentence, List<NlpEntity> tokens)
{
List<NlpEntity> res = new List<NlpEntity>();
for (int i = 0; i < tokens.Count ; i++)
{
NlpEntity nlpEntity = new NlpEntity();
StringBuilder unionValue = new StringBuilder(tokens[i].Value);
StringBuilder unionEntity = new StringBuilder(tokens[i].Entity);
decimal unoinConfidence = tokens[i].Confidence;
var current = tokens[i];
int j = i + 1;
while (j < tokens.Count && tokens[j].Entity == tokens[i].Entity && tokens[i].Entity != "O")
if (current.Entity != "O")
{
unionValue.Append(" " + tokens[j].Value);
j++;
nlpEntity = current.ToObject<NlpEntity>();
// greedy search until next entity
int j = 0;
for (j = i + 1; j < tokens.Count; j++)
{
var next = tokens[j];
if (current.Entity == next.Entity)
{
i = j;
nlpEntity.Value = sentence.Substring(current.Start, next.End - current.Start + 1);
}
else
{
break;
}
}
res.Add(nlpEntity);
}
nlpEntity.Entity = unionEntity.ToString();
nlpEntity.Start = tokens[i].Start;
nlpEntity.Value = unionValue.ToString();
nlpEntity.Confidence = unoinConfidence;
nlpEntity.Extrator = tokens[i].Extrator;
res.Add(nlpEntity);
i = j - 1;
}
return res;
}

4
BotSharp.Core/Engines/NlpDoc.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Models.NLP;
using BotSharp.Core.Abstractions;
using BotSharp.Models.NLP;
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
@ -8,6 +9,7 @@ namespace BotSharp.Core.Engines
{
public class NlpDoc
{
public INlpPredict Tokenizer { get; set; }
public List<NlpDocSentence> Sentences { get; set; }
}

6
BotSharp.RestApi/Rasa/ParseController.cs
View file

@ -54,6 +54,12 @@ namespace BotSharp.RestApi.Rasa
// Load agent
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project);
if (String.IsNullOrEmpty(request.Model))
{
request.Model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last();
}
var modelPath = Path.Combine(projectPath, request.Model);
var agent = _platform.LoadAgentFromFile(modelPath);