From f1e6132392d468b12156218a5037d3854ace0f9d Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Fri, 14 Sep 2018 08:09:52 -0500 Subject: [PATCH] simplify CRFLite decode unit test. Remove parallal test. --- BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs | 10 +- BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs | 107 ++++-------------- BotSharp.NLP/Models/CRFLite/CRFDecoder.cs | 2 +- BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs | 12 +- .../Models/CRFLite/SegDecoderTagger.cs | 18 +-- 5 files changed, 41 insertions(+), 108 deletions(-) diff --git a/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs index b03a5dc8..d782c569 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs @@ -54,10 +54,10 @@ namespace BotSharp.NLP.UnitTest.CRFLite tagger.set_vlevel(options.ProbLevel); //Initialize result - var crf_out = new crf_seg_out[options.NBest]; + var crf_out = new CRFSegOut[options.NBest]; for (var i = 0; i < options.NBest; i++) { - crf_out[i] = new crf_seg_out(tagger.crf_max_word_num); + crf_out[i] = new CRFSegOut(tagger.crf_max_word_num); } var inbuf = new List>(); @@ -228,7 +228,7 @@ namespace BotSharp.NLP.UnitTest.CRFLite } //Convert CRFSharp output format to string list - private List ConvertCRFTermOutToStringList(List> inbuf, crf_seg_out[] crf_out) + private List ConvertCRFTermOutToStringList(List> inbuf, CRFSegOut[] crf_out) { var sb = new StringBuilder(); for (var i = 0; i < inbuf.Count; i++) @@ -250,8 +250,8 @@ namespace BotSharp.NLP.UnitTest.CRFLite var crf_term_out = crf_out[i]; for (var j = 0; j < crf_term_out.Count; j++) { - var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length); - var strNE = crf_term_out.tokenList[j].strTag; + var str = strText.Substring(crf_term_out.tokenList[j].Offset, crf_term_out.tokenList[j].Length); + var strNE = crf_term_out.tokenList[j].Tag; sb.Append(str); if (strNE.Length > 0) diff --git a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs index f17c16fd..fd403378 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs @@ -52,107 +52,40 @@ namespace BotSharp.NLP.UnitTest.CRFLite var decoder = new CRFDecoder(); var options = new DecoderOptions { - /* - * input data format - * - In IN - its PRP$ - sixth JJ - edition NN - , PUN - the DT - Beijing NNP - . PUN - */ - InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\test.txt", ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" }; - var sr = new StreamReader(options.InputFileName); - //Load encoded model from file decoder.LoadModel(options.ModelFileName); - var parallelOption = new ParallelOptions(); - parallelOption.MaxDegreeOfParallelism = options.Thread; - Parallel.For(0, options.Thread, parallelOption, t => + //Create decoder tagger instance. + var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); + tagger.set_vlevel(options.ProbLevel); + + //Initialize result + var crf_out = new CRFSegOut[options.NBest]; + for (var i = 0; i < options.NBest; i++) { + crf_out[i] = new CRFSegOut(options.MaxWord); + } - //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance - var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); - tagger.set_vlevel(options.ProbLevel); + var dataset = GetTestData(); - //Initialize result - var crf_out = new crf_seg_out[options.NBest]; - for (var i = 0; i < options.NBest; i++) - { - crf_out[i] = new crf_seg_out(tagger.crf_max_word_num); - } - - var inbuf = new List>(); - - inbuf.Add(new List - { - "' PUN", - "' POS", - "Duchy NNP", - "of IN", - "Lithuania NNP" - }); - - while (true) - { - lock (rdLocker) - { - if (ReadRecord(inbuf, sr) == false) - { - break; - } - } - - //Call CRFSharp wrapper to predict given string's tags - decoder.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf); - } - }); - - - sr.Close(); + //predict given string's tags + decoder.Segment(crf_out, tagger, dataset); } - private bool ReadRecord(List> inbuf, StreamReader sr) + private List> GetTestData() { - inbuf.Clear(); + var dataset = new List>(); - while (true) - { - var strLine = sr.ReadLine(); - if (strLine == null) - { - //At the end of current file - if (inbuf.Count == 0) - { - return false; - } - else - { - return true; - } - } - strLine = strLine.Trim(); - if (strLine.Length == 0) - { - return true; - } + dataset.Add(new List { "'", "PUN" }); + dataset.Add(new List { "'", "POS" }); + dataset.Add(new List { "Duchy", "NNP" }); + dataset.Add(new List { "of", "IN" }); + dataset.Add(new List { "Lithuania", "NNP" }); - //Read feature set for each record - var items = strLine.Split(new char[] { '\t' }); - inbuf.Add(new List()); - for (int index = 0; index < items.Length; index++) - { - var item = items[index]; - inbuf[inbuf.Count - 1].Add(item); - } - } + return dataset; } } } diff --git a/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs b/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs index 38f5fb9e..2686a4c3 100644 --- a/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs +++ b/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs @@ -74,7 +74,7 @@ namespace BotSharp.Models.CRFLite } //Segment given text - public int Segment(crf_seg_out[] pout, //segment result + public int Segment(CRFSegOut[] pout, //segment result SegDecoderTagger tagger, //Tagger per thread List> inbuf //feature set for segment ) diff --git a/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs b/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs index ed8e234c..c5949ccd 100644 --- a/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs +++ b/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs @@ -7,13 +7,13 @@ namespace BotSharp.Models.CRFLite { public class SegToken { - public int offset; - public int length; - public string strTag; //CRF对应于term组合后的Tag字符串 - public double fWeight; //对应属性id的概率值，或者得分 + public int Offset; + public int Length; + public string Tag; + public double Weight; }; - public class crf_seg_out : CRFTermOut + public class CRFSegOut : CRFTermOut { //Segmented token by merging raw CRF model output public int termTotalLength; // the total term length in character @@ -30,7 +30,7 @@ namespace BotSharp.Models.CRFLite tokenList.Clear(); } - public crf_seg_out(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM): + public CRFSegOut(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM): base(max_word_num) { termTotalLength = 0; diff --git a/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs b/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs index 6904b6c0..8a1a9a98 100644 --- a/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs +++ b/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs @@ -13,7 +13,7 @@ namespace BotSharp.Models.CRFLite crf_max_word_num = this_crf_max_word_num; } - int seg_termbuf_build(crf_seg_out term_buf) + int seg_termbuf_build(CRFSegOut term_buf) { term_buf.Clear(); @@ -42,24 +42,24 @@ namespace BotSharp.Models.CRFLite i == x_.Count - 1) { var tkn = new SegToken(); - tkn.length = term_len; - tkn.offset = term_buf.termTotalLength; + tkn.Length = term_len; + tkn.Offset = term_buf.termTotalLength; var spos = strTag.IndexOf('_'); if (spos < 0) { if (strTag == "NOR") { - tkn.strTag = ""; + tkn.Tag = ""; } else { - tkn.strTag = strTag; + tkn.Tag = strTag; } } else { - tkn.strTag = strTag.Substring(spos + 1); + tkn.Tag = strTag.Substring(spos + 1); } term_buf.termTotalLength += term_len; @@ -67,10 +67,10 @@ namespace BotSharp.Models.CRFLite switch (vlevel_) { case 0: - tkn.fWeight = 0.0; + tkn.Weight = 0.0; break; case 2: - tkn.fWeight = weight / num; + tkn.Weight = weight / num; weight = 0.0; num = 0; break; @@ -86,7 +86,7 @@ namespace BotSharp.Models.CRFLite } - public int output(crf_seg_out[] pout) + public int output(CRFSegOut[] pout) { var n = 0; var ret = 0;