From 2c4872f068736a9ec318c813a0abafbc0c159945 Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Fri, 14 Sep 2018 09:10:35 -0500 Subject: [PATCH] add BotSharpCRFNer to bot process pipeline --- .../Engines/BotSharp/BotSharpCRFNer.cs | 149 ++++++++++++++++++ .../Engines/NERs/CRFsuiteEntityRecognizer.cs | 1 - Settings/bot.json | 10 +- 3 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs new file mode 100644 index 00000000..c1c9b90a --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs @@ -0,0 +1,149 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.Models.CRFLite; +using BotSharp.Models.CRFLite.Encoder; +using BotSharp.NLP.Tokenize; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpCRFNer : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + var corpus = agent.Corpus; + + meta.Model = "ner-crf.model"; + + List> userSays = corpus.UserSays; + List> list = new List>(); + + string rawTrainingDataFileName = System.IO.Path.Combine(Settings.TempDir, "ner-crf.corpus.txt"); + string modelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model); + + using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) + { + using (StreamWriter sw = new StreamWriter(fs)) + { + for (int i = 0; i < doc.Sentences.Count; i++) + { + List curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities); + curLine.ForEach(trainingData => + { + string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk }; + string wordStr = string.Join(" ", wordParams); + sw.Write(wordStr + "\n"); + }); + list.Add(curLine); + sw.Write("\n"); + } + sw.Flush(); + } + } + + string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); + string template = Configuration.GetValue($"BotSharpCRFNer:template"); + template = template.Replace("|App_Data|", contentDir); + + var encoder = new CRFEncoder(); + bool result = encoder.Learn(new EncoderOptions + { + TrainingCorpusFileName = rawTrainingDataFileName, + TemplateFileName = template, + ModelFileName = modelFileName, + }); + + return result; + } + + public List Merge(NlpDoc doc, List tokens, List entities) + { + List trainingTuple = new List(); + HashSet entityWordBag = new HashSet(); + int wordCandidateCount = 0; + + for (int i = 0; i < tokens.Count; i++) + { + TrainingIntentExpressionPart curEntity = null; + if (entities != null) + { + bool entityFinded = false; + entities.ForEach(entity => { + if (!entityFinded) + { + var vDoc = new NlpDoc { Sentences = new List { new NlpDocSentence { Text = entity.Value } } }; + doc.Tokenizer.Predict(null, vDoc, null); + string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray(); + + for (int j = 0; j < words.Length; j++) + { + if (tokens[i + j].Text == words[j]) + { + wordCandidateCount++; + if (j == words.Length - 1) + { + curEntity = entity; + } + } + else + { + wordCandidateCount = 0; + break; + } + } + if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset) + { + String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity; + foreach (string s in words) + { + trainingTuple.Add(new TrainingData(entityName, s, tokens[i].Pos, "I")); + } + entityFinded = true; + } + } + }); + } + if (wordCandidateCount == 0) + { + trainingTuple.Add(new TrainingData("O", tokens[i].Text, tokens[i].Pos, "O")); + } + else + { + i = i + wordCandidateCount - 1; + } + } + + return trainingTuple; + } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + throw new NotImplementedException(); + } + + public class TrainingData + { + public String Token { get; set; } + public String Entity { get; set; } + public String Pos { get; set; } + public String Chunk { get; set; } + + public TrainingData(string entity, string token, string pos, string chunk) + { + Token = token; + Entity = entity; + Pos = pos; + Chunk = chunk; + } + } + } +} diff --git a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs index b1d85122..d67c3739 100644 --- a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs @@ -38,7 +38,6 @@ namespace BotSharp.Core.Engines.NERs public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { - var dc = new DefaultDataContextLoader().GetDefaultDc(); var corpus = agent.Corpus; meta.Model = "ner-crf.model"; diff --git a/Settings/bot.json b/Settings/bot.json index 34e48d70..8849a7e8 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -7,19 +7,23 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier" }, "BotSharpNBayesClassifier": { }, "BotSharpSVMClassifier": { - "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" + "wordvec": "" }, "BotSharpTagger": { }, + + "BotSharpCRFNer": { + "template": "|App_Data|CRFLite\template.en" + }, "CRFsuiteEntityRecognizer": { "fields": "y w pos chk",