From 0c41b47057247c8e62dde0e1f8f43a38ca3bab0b Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Fri, 14 Sep 2018 11:31:35 -0500 Subject: [PATCH] Fix NER offset issue. --- .../BotSharp/BotSharpCBOWClassifier.cs | 3 - .../Engines/BotSharp/BotSharpCRFNer.cs | 188 ++++++++++--- .../BotSharp/BotSharpNBayesClassifier.cs | 2 - BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs | 261 ++---------------- BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs | 44 --- BotSharp.NLP/Models/CRFLite/CRFEncoder.cs | 14 +- .../Models/CRFLite/Decoder/DecoderOptions.cs | 2 +- .../Models/CRFLite/Encoder/EncoderOptions.cs | 4 +- .../Models/CRFLite/Encoder/ModelWriter.cs | 16 +- BotSharp.RestApi/BotSharp.RestApi.xml | 2 +- BotSharp.RestApi/Rasa/TrainController.cs | 4 +- Settings/bot.json | 2 +- 12 files changed, 204 insertions(+), 338 deletions(-) diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs index fa276a49..b3db17a7 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs @@ -58,9 +58,6 @@ namespace BotSharp.Core.Engines.BotSharp var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false); Console.WriteLine($"Saved model to {modelFileName}"); - meta.Meta = new JObject(); - meta.Meta["compiled at"] = "Aug 3, 2018"; - return true; } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs index c1c9b90a..42b6160b 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs @@ -1,8 +1,11 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.Models.CRFLite; +using BotSharp.Models.CRFLite.Decoder; using BotSharp.Models.CRFLite.Encoder; +using BotSharp.Models.NLP; using BotSharp.NLP.Tokenize; +using DotNetToolkit; using Microsoft.Extensions.Configuration; using System; using System.Collections.Generic; @@ -39,12 +42,12 @@ namespace BotSharp.Core.Engines.BotSharp List curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities); curLine.ForEach(trainingData => { - string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk }; - string wordStr = string.Join(" ", wordParams); - sw.Write(wordStr + "\n"); + string[] wordParams = { trainingData.Token, trainingData.Pos, trainingData.Entity }; + string wordStr = string.Join("\t", wordParams); + sw.WriteLine(wordStr); }); list.Add(curLine); - sw.Write("\n"); + sw.WriteLine(); } sw.Flush(); } @@ -52,7 +55,7 @@ namespace BotSharp.Core.Engines.BotSharp string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); string template = Configuration.GetValue($"BotSharpCRFNer:template"); - template = template.Replace("|App_Data|", contentDir); + template = template.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar); var encoder = new CRFEncoder(); bool result = encoder.Learn(new EncoderOptions @@ -65,7 +68,7 @@ namespace BotSharp.Core.Engines.BotSharp return result; } - public List Merge(NlpDoc doc, List tokens, List entities) + private List Merge(NlpDoc doc, List tokens, List entities) { List trainingTuple = new List(); HashSet entityWordBag = new HashSet(); @@ -74,47 +77,75 @@ namespace BotSharp.Core.Engines.BotSharp for (int i = 0; i < tokens.Count; i++) { TrainingIntentExpressionPart curEntity = null; - if (entities != null) - { - bool entityFinded = false; - entities.ForEach(entity => { - if (!entityFinded) - { - var vDoc = new NlpDoc { Sentences = new List { new NlpDocSentence { Text = entity.Value } } }; - doc.Tokenizer.Predict(null, vDoc, null); - string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray(); + if (entities == null) continue; - for (int j = 0; j < words.Length; j++) + bool entityFinded = false; + for (int entityIndex = 0; entityIndex < entities.Count; entityIndex++) + { + var entity = entities[entityIndex]; + + if (!entityFinded) + { + var vDoc = new NlpDoc { Sentences = new List { new NlpDocSentence { Text = entity.Value } } }; + doc.Tokenizer.Predict(null, vDoc, null); + string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray(); + + for (int j = 0; j < words.Length; j++) + { + if (tokens[i + j].Text == words[j]) { - if (tokens[i + j].Text == words[j]) + wordCandidateCount++; + if (j == words.Length - 1) { - wordCandidateCount++; - if (j == words.Length - 1) + curEntity = entity; + } + } + else + { + wordCandidateCount = 0; + break; + } + } + if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset) + { + String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity; + + for(int wordIndex = 0; wordIndex < words.Length; wordIndex++) + { + var tag = entityName; + + if (wordIndex == 0) + { + if (words.Length == 1) { - curEntity = entity; + tag = "S_" + entityName; } + else + { + tag = "B_" + entityName; + } + } + else if (wordIndex == words.Length - 1) + { + tag = "E_" + entityName; } else { - wordCandidateCount = 0; - break; + tag = "M_" + entityName; } + + var word = words[wordIndex]; + trainingTuple.Add(new TrainingData(tag, word, tokens[i].Pos)); } - if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset) - { - String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity; - foreach (string s in words) - { - trainingTuple.Add(new TrainingData(entityName, s, tokens[i].Pos, "I")); - } - entityFinded = true; - } + + entityFinded = true; } - }); + } } + if (wordCandidateCount == 0) { - trainingTuple.Add(new TrainingData("O", tokens[i].Text, tokens[i].Pos, "O")); + trainingTuple.Add(new TrainingData("S", tokens[i].Text, tokens[i].Pos)); } else { @@ -127,7 +158,92 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { - throw new NotImplementedException(); + var decoder = new CRFDecoder(); + var options = new DecoderOptions + { + ModelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model) + }; + + //Load encoded model from file + decoder.LoadModel(options.ModelFileName); + + //Create decoder tagger instance. + var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); + tagger.set_vlevel(options.ProbLevel); + + //Initialize result + var crf_out = new CRFSegOut[options.NBest]; + for (var i = 0; i < options.NBest; i++) + { + crf_out[i] = new CRFSegOut(options.MaxWord); + } + + doc.Sentences.ForEach(sent => + { + List> dataset = new List>(); + dataset.AddRange(sent.Tokens.Select(token => new List { token.Text, token.Pos }).ToList()); + //predict given string's tags + decoder.Segment(crf_out, tagger, dataset); + + var entities = new List(); + + for (int i = 0; i < sent.Tokens.Count; i++) + { + var entity = crf_out[0].result_; + entities.Add(new NlpEntity + { + Entity = entity[i], + Start = doc.Sentences[0].Tokens[i].Start, + Value = doc.Sentences[0].Tokens[i].Text, + Confidence = 0, + Extrator = "BotSharpCRFNer" + }); + } + + sent.Entities = MergeEntity(doc.Sentences[0].Text, entities); + }); + + return true; + } + + private List MergeEntity(string sentence, List tokens) + { + List res = new List(); + + for(int i = 0; i < tokens.Count; i++) + { + var entity = tokens[i]; + + if (entity.Entity.StartsWith("S_")) + { + entity.Entity = entity.Entity.Split('_')[1]; + res.Add(entity); + } + else if (entity.Entity.StartsWith("B_")) + { + entity.Entity = entity.Entity.Split('_')[1]; + + for(int j = i; j < tokens.Count; j++) + { + var token = tokens[j]; + if (token.Entity.StartsWith("E_")) + { + res.Add(new NlpEntity + { + Value = sentence.Substring(entity.Start, token.End - entity.Start + 1), + Entity = entity.Entity, + Extrator = entity.Extrator, + Start = entity.Start, + Confidence = entity.Confidence + }); + } + + i++; + } + } + } + + return res; } public class TrainingData @@ -135,14 +251,12 @@ namespace BotSharp.Core.Engines.BotSharp public String Token { get; set; } public String Entity { get; set; } public String Pos { get; set; } - public String Chunk { get; set; } - public TrainingData(string entity, string token, string pos, string chunk) + public TrainingData(string entity, string token, string pos) { Token = token; Entity = entity; Pos = pos; - Chunk = chunk; } } } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs index bc1380e5..11845549 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -40,8 +40,6 @@ namespace BotSharp.Core.Engines.BotSharp classifier.Train(sentences); Console.WriteLine($"Saved model to {modelFileName}"); - meta.Meta = new JObject(); - meta.Meta["compiled at"] = "Sep 12, 2018"; return true; } diff --git a/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs index d782c569..2642c4b8 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs @@ -13,257 +13,48 @@ namespace BotSharp.NLP.UnitTest.CRFLite [TestClass] public class DecoderTest { + object rdLocker = new object(); + [TestMethod] public void TestDecode() { - var encoder = new CRFDecoder(); - bool result = Decode(new DecoderOptions + var decoder = new CRFDecoder(); + var options = new DecoderOptions { - InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\test.txt", - ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng", - }); - } - - object rdLocker = new object(); - - bool Decode(DecoderOptions options) - { - var parallelOption = new ParallelOptions(); - var watch = Stopwatch.StartNew(); - - var sr = new StreamReader(options.InputFileName); - StreamWriter sw = null, swSeg = null; - - - //Create CRFSharp wrapper instance. It's a global instance - var crfWrapper = new CRFDecoder(); + ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" + }; //Load encoded model from file - //Logger.WriteLine("Loading model from {0}", options.strModelFileName); - crfWrapper.LoadModel(options.ModelFileName); + decoder.LoadModel(options.ModelFileName); - var queueRecords = new ConcurrentQueue>>(); - var queueSegRecords = new ConcurrentQueue>>(); + //Create decoder tagger instance. + var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); + tagger.set_vlevel(options.ProbLevel); - parallelOption.MaxDegreeOfParallelism = options.Thread; - Parallel.For(0, options.Thread, parallelOption, t => + //Initialize result + var crf_out = new CRFSegOut[options.NBest]; + for (var i = 0; i < options.NBest; i++) { - - //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance - var tagger = crfWrapper.CreateTagger(options.NBest, options.MaxWord); - tagger.set_vlevel(options.ProbLevel); - - //Initialize result - var crf_out = new CRFSegOut[options.NBest]; - for (var i = 0; i < options.NBest; i++) - { - crf_out[i] = new CRFSegOut(tagger.crf_max_word_num); - } - - var inbuf = new List>(); - while (true) - { - lock (rdLocker) - { - if (ReadRecord(inbuf, sr) == false) - { - break; - } - - queueRecords.Enqueue(inbuf); - queueSegRecords.Enqueue(inbuf); - } - - //Call CRFSharp wrapper to predict given string's tags - if (swSeg != null) - { - crfWrapper.Segment(crf_out, tagger, inbuf); - } - else - { - crfWrapper.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf); - } - - List> peek = null; - //Save segmented tagged result into file - if (swSeg != null) - { - var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out); - while (peek != inbuf) - { - queueSegRecords.TryPeek(out peek); - } - for (int index = 0; index < rstList.Count; index++) - { - var item = rstList[index]; - swSeg.WriteLine(item); - } - queueSegRecords.TryDequeue(out peek); - peek = null; - } - - //Save raw tagged result (with probability) into file - if (sw != null) - { - while (peek != inbuf) - { - queueRecords.TryPeek(out peek); - } - OutputRawResultToFile(inbuf, crf_out, tagger, sw); - queueRecords.TryDequeue(out peek); - - } - } - }); - - - sr.Close(); - - if (sw != null) - { - sw.Close(); + crf_out[i] = new CRFSegOut(options.MaxWord); } - if (swSeg != null) - { - swSeg.Close(); - } - watch.Stop(); - //Logger.WriteLine("Elapsed: {0} ms", watch.ElapsedMilliseconds); - return true; + var dataset = GetTestData(); + + //predict given string's tags + decoder.Segment(crf_out, tagger, dataset); } - private bool ReadRecord(List> inbuf, StreamReader sr) + private List> GetTestData() { - inbuf.Clear(); + var dataset = new List>(); - while (true) - { - var strLine = sr.ReadLine(); - if (strLine == null) - { - //At the end of current file - if (inbuf.Count == 0) - { - return false; - } - else - { - return true; - } - } - strLine = strLine.Trim(); - if (strLine.Length == 0) - { - return true; - } + dataset.Add(new List { "'", "PUN" }); + dataset.Add(new List { "'", "POS" }); + dataset.Add(new List { "Duchy", "NNP" }); + dataset.Add(new List { "of", "IN" }); + dataset.Add(new List { "Lithuania", "NNP" }); - //Read feature set for each record - var items = strLine.Split(new char[] { '\t' }); - inbuf.Add(new List()); - for (int index = 0; index < items.Length; index++) - { - var item = items[index]; - inbuf[inbuf.Count - 1].Add(item); - } - } - } - - //Output raw result with probability - private void OutputRawResultToFile(List> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw) - { - for (var k = 0; k < crf_out.Length; k++) - { - if (crf_out[k] == null) - { - //No more result - break; - } - - var sb = new StringBuilder(); - - var crf_seg_out = crf_out[k]; - //Show the entire sequence probability - //For each token - for (var i = 0; i < inbuf.Count; i++) - { - //Show all features - for (var j = 0; j < inbuf[i].Count; j++) - { - sb.Append(inbuf[i][j]); - sb.Append("\t"); - } - - //Show the best result and its probability - sb.Append(crf_seg_out.result_[i]); - - if (tagger.vlevel_ > 1) - { - sb.Append("\t"); - sb.Append(crf_seg_out.weight_[i]); - - //Show the probability of all tags - sb.Append("\t"); - for (var j = 0; j < tagger.ysize_; j++) - { - sb.Append(tagger.yname(j)); - sb.Append("/"); - sb.Append(tagger.prob(i, j)); - - if (j < tagger.ysize_ - 1) - { - sb.Append("\t"); - } - } - } - sb.AppendLine(); - } - if (tagger.vlevel_ > 0) - { - sw.WriteLine("#{0}", crf_seg_out.prob); - } - sw.WriteLine(sb.ToString().Trim()); - sw.WriteLine(); - } - } - - //Convert CRFSharp output format to string list - private List ConvertCRFTermOutToStringList(List> inbuf, CRFSegOut[] crf_out) - { - var sb = new StringBuilder(); - for (var i = 0; i < inbuf.Count; i++) - { - sb.Append(inbuf[i][0]); - } - - var strText = sb.ToString(); - var rstList = new List(); - for (var i = 0; i < crf_out.Length; i++) - { - if (crf_out[i] == null) - { - //No more result - break; - } - - sb.Clear(); - var crf_term_out = crf_out[i]; - for (var j = 0; j < crf_term_out.Count; j++) - { - var str = strText.Substring(crf_term_out.tokenList[j].Offset, crf_term_out.tokenList[j].Length); - var strNE = crf_term_out.tokenList[j].Tag; - - sb.Append(str); - if (strNE.Length > 0) - { - sb.Append("[" + strNE + "]"); - } - sb.Append(" "); - } - rstList.Add(sb.ToString().Trim()); - } - - return rstList; + return dataset; } } } diff --git a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs index fd403378..5f4d82ce 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs @@ -43,49 +43,5 @@ namespace BotSharp.NLP.UnitTest.CRFLite Assert.IsTrue(result); } - - object rdLocker = new object(); - - [TestMethod] - public void TestDecode() - { - var decoder = new CRFDecoder(); - var options = new DecoderOptions - { - ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" - }; - - //Load encoded model from file - decoder.LoadModel(options.ModelFileName); - - //Create decoder tagger instance. - var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); - tagger.set_vlevel(options.ProbLevel); - - //Initialize result - var crf_out = new CRFSegOut[options.NBest]; - for (var i = 0; i < options.NBest; i++) - { - crf_out[i] = new CRFSegOut(options.MaxWord); - } - - var dataset = GetTestData(); - - //predict given string's tags - decoder.Segment(crf_out, tagger, dataset); - } - - private List> GetTestData() - { - var dataset = new List>(); - - dataset.Add(new List { "'", "PUN" }); - dataset.Add(new List { "'", "POS" }); - dataset.Add(new List { "Duchy", "NNP" }); - dataset.Add(new List { "of", "IN" }); - dataset.Add(new List { "Lithuania", "NNP" }); - - return dataset; - } } } diff --git a/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs b/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs index cc85b626..886e6f71 100644 --- a/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs +++ b/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs @@ -61,24 +61,17 @@ namespace BotSharp.Models.CRFLite var xList = modelWriter.ReadAllRecords(); - modelWriter.Shrink(xList, args.MinFeatureFreq); if (!modelWriter.SaveModelMetaData(args.ModelFileName)) { return false; } - else - { - } if (!modelWriter.BuildFeatureSetIntoIndex(args.ModelFileName, args.SlotUsageRateThreshold, args.DebugLevel)) { return false; } - else - { - } if (xList.Length == 0) { @@ -101,6 +94,8 @@ namespace BotSharp.Models.CRFLite bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args) { + Console.WriteLine("Running encoding process..."); + var old_obj = double.MaxValue; var converge = 0; var lbfgs = new LBFGS(args.ThreadsNum); @@ -165,6 +160,9 @@ namespace BotSharp.Models.CRFLite lbfgs.err += processList[i].err; lbfgs.zeroone += processList[i].zeroone; + Console.WriteLine($"Thread: {i}, Iterating {itr} / {args.MaxIteration}"); + Console.WriteLine($"{lbfgs.obj} {lbfgs.err} {lbfgs.zeroone}"); + //Calculate error for (var j = 0; j < modelWriter.y_.Count; j++) { @@ -263,6 +261,8 @@ namespace BotSharp.Models.CRFLite } } + Console.WriteLine("Completed encoding process."); + return true; } diff --git a/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs b/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs index 2a988eb1..b7b19d65 100644 --- a/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs +++ b/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs @@ -40,7 +40,7 @@ namespace BotSharp.Models.CRFLite.Decoder public DecoderOptions() { Thread = 1; - NBest = 1; + NBest = 2; ProbLevel = 0; MaxWord = 128; } diff --git a/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs b/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs index 410862cb..7ea76303 100644 --- a/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs +++ b/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs @@ -14,7 +14,7 @@ namespace BotSharp.Models.CRFLite.Encoder /// /// Minimum feature frequency, if one feature's frequency is less than this value, the feature will be dropped. /// - public int MinFeatureFreq = 2; + public int MinFeatureFreq = 1; /// /// Minimum diff value, when diff less than the value consecutive 3 times, the process will be ended. @@ -79,7 +79,7 @@ namespace BotSharp.Models.CRFLite.Encoder public EncoderOptions() { MaxIteration = 100; - MinFeatureFreq = 2; + MinFeatureFreq = 1; MinDifference = 0.0001; SlotUsageRateThreshold = 0.95; ThreadsNum = 1; diff --git a/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs b/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs index b440bd8b..c711de3f 100644 --- a/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs +++ b/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs @@ -43,6 +43,8 @@ namespace BotSharp.Models.CRFLite.Encoder //Regenerate feature id and shrink features with lower frequency public void Shrink(EncoderTagger[] xList, int freq) { + Console.WriteLine($"Shrink features lower than {freq} frequency"); + var old2new = new CRFLite.Utils.BTreeDictionary(); featureLexicalDict.Shrink(freq); maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count); @@ -86,7 +88,7 @@ namespace BotSharp.Models.CRFLite.Encoder var oldValue = Interlocked.Increment(ref arrayEncoderTaggerSize) - 1; arrayEncoderTagger[oldValue] = _x; - if (oldValue % 10000 == 0) + if (oldValue % 100 == 0) { //Show current progress on console Console.Write("{0}...", oldValue); @@ -94,10 +96,11 @@ namespace BotSharp.Models.CRFLite.Encoder } }); + Console.WriteLine($"Read {trainCorpusList.Count} records"); + trainCorpusList.Clear(); trainCorpusList = null; - - Console.WriteLine(); + return arrayEncoderTagger; } @@ -136,6 +139,8 @@ namespace BotSharp.Models.CRFLite.Encoder //Save indexed feature set into file da.save(filename_featureset); + Console.WriteLine($"Saved featureset to {filename_featureset}"); + if (string.IsNullOrWhiteSpace(modelFileName)) { //Clean up all data @@ -220,6 +225,7 @@ namespace BotSharp.Models.CRFLite.Encoder tofs.Close(); + Console.WriteLine($"Saved meta data to {filename}"); return true; } @@ -277,6 +283,8 @@ namespace BotSharp.Models.CRFLite.Encoder bool OpenTemplateFile(string filename) { + Console.WriteLine($"Open template: {filename}"); + var ifs = new StreamReader(filename); unigram_templs_ = new List(); bigram_templs_ = new List(); @@ -305,6 +313,8 @@ namespace BotSharp.Models.CRFLite.Encoder bool OpenTrainCorpusFile(string strTrainingCorpusFileName) { + Console.WriteLine($"Open corpus: {strTrainingCorpusFileName}"); + var ifs = new StreamReader(strTrainingCorpusFileName); y_ = new List(); trainCorpusList = new List>>(); diff --git a/BotSharp.RestApi/BotSharp.RestApi.xml b/BotSharp.RestApi/BotSharp.RestApi.xml index 9ec6d8f0..2231b601 100644 --- a/BotSharp.RestApi/BotSharp.RestApi.xml +++ b/BotSharp.RestApi/BotSharp.RestApi.xml @@ -120,7 +120,7 @@ Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. Model name - + Agent name or agent id diff --git a/BotSharp.RestApi/Rasa/TrainController.cs b/BotSharp.RestApi/Rasa/TrainController.cs index ae2eb3cb..a0d2785e 100644 --- a/BotSharp.RestApi/Rasa/TrainController.cs +++ b/BotSharp.RestApi/Rasa/TrainController.cs @@ -38,10 +38,10 @@ namespace BotSharp.RestApi.Rasa /// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. /// /// Model name - /// + /// Agent name or agent id /// [HttpPost] - public async Task> Train([FromQuery] string model, [FromQuery] string project) + public async Task> Train([FromQuery] string project, [FromQuery] string model) { string body = ""; using (var reader = new StreamReader(Request.Body)) diff --git a/Settings/bot.json b/Settings/bot.json index 8849a7e8..b41b7b92 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -22,7 +22,7 @@ }, "BotSharpCRFNer": { - "template": "|App_Data|CRFLite\template.en" + "template": "|App_Data|CRFLite/template.en" }, "CRFsuiteEntityRecognizer": {