diff --git a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs index 36bd1dd5..84d9594e 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs @@ -1,6 +1,12 @@ using BotSharp.Models.CRFLite; +using BotSharp.Models.CRFLite.Decoder; using BotSharp.Models.CRFLite.Encoder; using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; namespace BotSharp.NLP.UnitTest.CRFLite { @@ -13,10 +19,260 @@ namespace BotSharp.NLP.UnitTest.CRFLite var encoder = new CRFEncoder(); bool result = encoder.Learn(new EncoderOptions { - TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\corpus\eng.1K.training", - TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\template.NE", - ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng" + TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\eng.1k.training", + TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\template.en", + ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" }); + + Assert.IsTrue(result); + } + + object rdLocker = new object(); + + [TestMethod] + public void TestDecode() + { + var decoder = new CRFDecoder(); + var options = new DecoderOptions + { + InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\test.txt", + OutputSegFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\test.seg.txt", + OutputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\test.output.txt", + ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" + }; + + var sr = new StreamReader(options.InputFileName); + StreamWriter sw = null, swSeg = null; + + if (options.OutputFileName != null && options.OutputFileName.Length > 0) + { + sw = new StreamWriter(options.OutputFileName); + } + if (options.OutputSegFileName != null && options.OutputSegFileName.Length > 0) + { + swSeg = new StreamWriter(options.OutputSegFileName); + } + + //Load encoded model from file + decoder.LoadModel(options.ModelFileName); + + var queueRecords = new ConcurrentQueue>>(); + var queueSegRecords = new ConcurrentQueue>>(); + + var parallelOption = new ParallelOptions(); + parallelOption.MaxDegreeOfParallelism = options.Thread; + Parallel.For(0, options.Thread, parallelOption, t => + { + + //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance + var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); + tagger.set_vlevel(options.ProbLevel); + + //Initialize result + var crf_out = new crf_seg_out[options.NBest]; + for (var i = 0; i < options.NBest; i++) + { + crf_out[i] = new crf_seg_out(tagger.crf_max_word_num); + } + + var inbuf = new List>(); + while (true) + { + lock (rdLocker) + { + if (ReadRecord(inbuf, sr) == false) + { + break; + } + + queueRecords.Enqueue(inbuf); + queueSegRecords.Enqueue(inbuf); + } + + //Call CRFSharp wrapper to predict given string's tags + if (swSeg != null) + { + decoder.Segment(crf_out, tagger, inbuf); + } + else + { + decoder.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf); + } + + List> peek = null; + //Save segmented tagged result into file + if (swSeg != null) + { + var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out); + while (peek != inbuf) + { + queueSegRecords.TryPeek(out peek); + } + for (int index = 0; index < rstList.Count; index++) + { + var item = rstList[index]; + swSeg.WriteLine(item); + } + queueSegRecords.TryDequeue(out peek); + peek = null; + } + + //Save raw tagged result (with probability) into file + if (sw != null) + { + while (peek != inbuf) + { + queueRecords.TryPeek(out peek); + } + OutputRawResultToFile(inbuf, crf_out, tagger, sw); + queueRecords.TryDequeue(out peek); + + } + } + }); + + + sr.Close(); + + if (sw != null) + { + sw.Close(); + } + if (swSeg != null) + { + swSeg.Close(); + } + } + + private bool ReadRecord(List> inbuf, StreamReader sr) + { + inbuf.Clear(); + + while (true) + { + var strLine = sr.ReadLine(); + if (strLine == null) + { + //At the end of current file + if (inbuf.Count == 0) + { + return false; + } + else + { + return true; + } + } + strLine = strLine.Trim(); + if (strLine.Length == 0) + { + return true; + } + + //Read feature set for each record + var items = strLine.Split(new char[] { '\t' }); + inbuf.Add(new List()); + for (int index = 0; index < items.Length; index++) + { + var item = items[index]; + inbuf[inbuf.Count - 1].Add(item); + } + } + } + + private void OutputRawResultToFile(List> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw) + { + for (var k = 0; k < crf_out.Length; k++) + { + if (crf_out[k] == null) + { + //No more result + break; + } + + var sb = new StringBuilder(); + + var crf_seg_out = crf_out[k]; + //Show the entire sequence probability + //For each token + for (var i = 0; i < inbuf.Count; i++) + { + //Show all features + for (var j = 0; j < inbuf[i].Count; j++) + { + sb.Append(inbuf[i][j]); + sb.Append("\t"); + } + + //Show the best result and its probability + sb.Append(crf_seg_out.result_[i]); + + if (tagger.vlevel_ > 1) + { + sb.Append("\t"); + sb.Append(crf_seg_out.weight_[i]); + + //Show the probability of all tags + sb.Append("\t"); + for (var j = 0; j < tagger.ysize_; j++) + { + sb.Append(tagger.yname(j)); + sb.Append("/"); + sb.Append(tagger.prob(i, j)); + + if (j < tagger.ysize_ - 1) + { + sb.Append("\t"); + } + } + } + sb.AppendLine(); + } + if (tagger.vlevel_ > 0) + { + sw.WriteLine("#{0}", crf_seg_out.prob); + } + sw.WriteLine(sb.ToString().Trim()); + sw.WriteLine(); + } + } + + private List ConvertCRFTermOutToStringList(List> inbuf, crf_seg_out[] crf_out) + { + var sb = new StringBuilder(); + for (var i = 0; i < inbuf.Count; i++) + { + sb.Append(inbuf[i][0]); + } + + var strText = sb.ToString(); + var rstList = new List(); + for (var i = 0; i < crf_out.Length; i++) + { + if (crf_out[i] == null) + { + //No more result + break; + } + + sb.Clear(); + var crf_term_out = crf_out[i]; + for (var j = 0; j < crf_term_out.Count; j++) + { + var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length); + var strNE = crf_term_out.tokenList[j].strTag; + + sb.Append(str); + if (strNE.Length > 0) + { + sb.Append("[" + strNE + "]"); + } + sb.Append(" "); + } + rstList.Add(sb.ToString().Trim()); + } + + return rstList; } } } diff --git a/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs b/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs index f7834fd0..35a32435 100644 --- a/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs +++ b/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs @@ -52,7 +52,7 @@ namespace BotSharp.Models.CRFLite.Decoder Thread = 1; NBest = 1; ProbLevel = 0; - MaxWord = 100; + MaxWord = 10; } } } diff --git a/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs b/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs index 880a06a4..60f8dc63 100644 --- a/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs +++ b/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs @@ -68,7 +68,7 @@ namespace BotSharp.Models.CRFLite.Decoder LoadFeatureWeights(); } - //获取key对应的特征id + //get key feature id public virtual int get_id(string str) { return da.SearchByPerfectMatch(str); @@ -143,26 +143,20 @@ namespace BotSharp.Models.CRFLite.Decoder var sr = new StreamReader(metadataStream); string strLine; - //读入版本号 strLine = sr.ReadLine(); version = uint.Parse(strLine.Split(':')[1].Trim()); - //读入cost_factor strLine = sr.ReadLine(); cost_factor_ = double.Parse(strLine.Split(':')[1].Trim()); - //读入maxid strLine = sr.ReadLine(); maxid_ = long.Parse(strLine.Split(':')[1].Trim()); - //读入xsize strLine = sr.ReadLine(); xsize_ = uint.Parse(strLine.Split(':')[1].Trim()); - //读入空行 strLine = sr.ReadLine(); - //读入待标注的标签 y_ = new List(); while (true) { @@ -174,7 +168,7 @@ namespace BotSharp.Models.CRFLite.Decoder y_.Add(strLine); } - //读入unigram和bigram模板 + // load unigram and bigram template unigram_templs_ = new List(); bigram_templs_ = new List(); while (sr.EndOfStream == false) diff --git a/BotSharp.WebHost/Algorithms/crfsuite b/BotSharp.WebHost/Algorithms/crfsuite deleted file mode 100755 index 3fab0c91..00000000 Binary files a/BotSharp.WebHost/Algorithms/crfsuite and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/crfsuite.exe b/BotSharp.WebHost/Algorithms/crfsuite.exe deleted file mode 100644 index 74c7ad44..00000000 Binary files a/BotSharp.WebHost/Algorithms/crfsuite.exe and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/cyggcc_s-seh-1.dll b/BotSharp.WebHost/Algorithms/cyggcc_s-seh-1.dll deleted file mode 100644 index e5d3d8c8..00000000 Binary files a/BotSharp.WebHost/Algorithms/cyggcc_s-seh-1.dll and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/cygstdc++-6.dll b/BotSharp.WebHost/Algorithms/cygstdc++-6.dll deleted file mode 100644 index 03606607..00000000 Binary files a/BotSharp.WebHost/Algorithms/cygstdc++-6.dll and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/cygwin1.dll b/BotSharp.WebHost/Algorithms/cygwin1.dll deleted file mode 100644 index e2a5919a..00000000 Binary files a/BotSharp.WebHost/Algorithms/cygwin1.dll and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/fasttext.exe b/BotSharp.WebHost/Algorithms/fasttext.exe deleted file mode 100644 index 0ffcdd4c..00000000 Binary files a/BotSharp.WebHost/Algorithms/fasttext.exe and /dev/null differ