SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

simplify CRFLite decode unit test.

Browse source 
Remove parallal test.
This commit is contained in:
botsharp2018 2018-09-14 08:09:52 -05:00
parent 7bcec7260d
commit f1e6132392
5 changed files with 41 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs
View file

@ -54,10 +54,10 @@ namespace BotSharp.NLP.UnitTest.CRFLite
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new crf_seg_out[options.NBest];
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
crf_out[i] = new CRFSegOut(tagger.crf_max_word_num);
}
var inbuf = new List<List<string>>();
@ -228,7 +228,7 @@ namespace BotSharp.NLP.UnitTest.CRFLite
}
//Convert CRFSharp output format to string list
private List<string> ConvertCRFTermOutToStringList(List<List<string>> inbuf, crf_seg_out[] crf_out)
private List<string> ConvertCRFTermOutToStringList(List<List<string>> inbuf, CRFSegOut[] crf_out)
{
var sb = new StringBuilder();
for (var i = 0; i < inbuf.Count; i++)
@ -250,8 +250,8 @@ namespace BotSharp.NLP.UnitTest.CRFLite
var crf_term_out = crf_out[i];
for (var j = 0; j < crf_term_out.Count; j++)
{
var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length);
var strNE = crf_term_out.tokenList[j].strTag;
var str = strText.Substring(crf_term_out.tokenList[j].Offset, crf_term_out.tokenList[j].Length);
var strNE = crf_term_out.tokenList[j].Tag;
sb.Append(str);
if (strNE.Length > 0)

107
BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs
View file

@ -52,107 +52,40 @@ namespace BotSharp.NLP.UnitTest.CRFLite
var decoder = new CRFDecoder();
var options = new DecoderOptions
{
/*
* input data format
*
In IN
its PRP$
sixth JJ
edition NN
, PUN
the DT
Beijing NNP
. PUN
*/
InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\test.txt",
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model"
};
var sr = new StreamReader(options.InputFileName);
//Load encoded model from file
decoder.LoadModel(options.ModelFileName);
var parallelOption = new ParallelOptions();
parallelOption.MaxDegreeOfParallelism = options.Thread;
Parallel.For(0, options.Thread, parallelOption, t =>
//Create decoder tagger instance.
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new CRFSegOut(options.MaxWord);
}
//Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
var dataset = GetTestData();
//Initialize result
var crf_out = new crf_seg_out[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
}
var inbuf = new List<List<string>>();
inbuf.Add(new List<string>
{
"' PUN",
"' POS",
"Duchy NNP",
"of IN",
"Lithuania NNP"
});
while (true)
{
lock (rdLocker)
{
if (ReadRecord(inbuf, sr) == false)
{
break;
}
}
//Call CRFSharp wrapper to predict given string's tags
decoder.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf);
}
});
sr.Close();
//predict given string's tags
decoder.Segment(crf_out, tagger, dataset);
}
private bool ReadRecord(List<List<string>> inbuf, StreamReader sr)
private List<List<string>> GetTestData()
{
inbuf.Clear();
var dataset = new List<List<string>>();
while (true)
{
var strLine = sr.ReadLine();
if (strLine == null)
{
//At the end of current file
if (inbuf.Count == 0)
{
return false;
}
else
{
return true;
}
}
strLine = strLine.Trim();
if (strLine.Length == 0)
{
return true;
}
dataset.Add(new List<string> { "'", "PUN" });
dataset.Add(new List<string> { "'", "POS" });
dataset.Add(new List<string> { "Duchy", "NNP" });
dataset.Add(new List<string> { "of", "IN" });
dataset.Add(new List<string> { "Lithuania", "NNP" });
//Read feature set for each record
var items = strLine.Split(new char[] { '\t' });
inbuf.Add(new List<string>());
for (int index = 0; index < items.Length; index++)
{
var item = items[index];
inbuf[inbuf.Count - 1].Add(item);
}
}
return dataset;
}
}
}

2
BotSharp.NLP/Models/CRFLite/CRFDecoder.cs
View file

@ -74,7 +74,7 @@ namespace BotSharp.Models.CRFLite
}
//Segment given text
public int Segment(crf_seg_out[] pout, //segment result
public int Segment(CRFSegOut[] pout, //segment result
SegDecoderTagger tagger, //Tagger per thread
List<List<string>> inbuf //feature set for segment
)

12
BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs
View file

@ -7,13 +7,13 @@ namespace BotSharp.Models.CRFLite
{
public class SegToken
{
public int offset;
public int length;
public string strTag; //CRF对应于term组合后的Tag字符串
public double fWeight; //对应属性id的概率值，或者得分
public int Offset;
public int Length;
public string Tag;
public double Weight;
};
public class crf_seg_out : CRFTermOut
public class CRFSegOut : CRFTermOut
{
//Segmented token by merging raw CRF model output
public int termTotalLength; // the total term length in character
@ -30,7 +30,7 @@ namespace BotSharp.Models.CRFLite
tokenList.Clear();
}
public crf_seg_out(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM):
public CRFSegOut(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM):
base(max_word_num)
{
termTotalLength = 0;

18
BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs
View file

@ -13,7 +13,7 @@ namespace BotSharp.Models.CRFLite
crf_max_word_num = this_crf_max_word_num;
}
int seg_termbuf_build(crf_seg_out term_buf)
int seg_termbuf_build(CRFSegOut term_buf)
{
term_buf.Clear();
@ -42,24 +42,24 @@ namespace BotSharp.Models.CRFLite
i == x_.Count - 1)
{
var tkn = new SegToken();
tkn.length = term_len;
tkn.offset = term_buf.termTotalLength;
tkn.Length = term_len;
tkn.Offset = term_buf.termTotalLength;
var spos = strTag.IndexOf('_');
if (spos < 0)
{
if (strTag == "NOR")
{
tkn.strTag = "";
tkn.Tag = "";
}
else
{
tkn.strTag = strTag;
tkn.Tag = strTag;
}
}
else
{
tkn.strTag = strTag.Substring(spos + 1);
tkn.Tag = strTag.Substring(spos + 1);
}
term_buf.termTotalLength += term_len;
@ -67,10 +67,10 @@ namespace BotSharp.Models.CRFLite
switch (vlevel_)
{
case 0:
tkn.fWeight = 0.0;
tkn.Weight = 0.0;
break;
case 2:
tkn.fWeight = weight / num;
tkn.Weight = weight / num;
weight = 0.0;
num = 0;
break;
@ -86,7 +86,7 @@ namespace BotSharp.Models.CRFLite
}
public int output(crf_seg_out[] pout)
public int output(CRFSegOut[] pout)
{
var n = 0;
var ret = 0;