diff --git a/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj b/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj
new file mode 100644
index 00000000..57109843
--- /dev/null
+++ b/BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj
@@ -0,0 +1,19 @@
+
+
+
+ netcoreapp2.1
+
+ false
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/BotSharp.MachineLearning.UnitTest/DecoderTest.cs b/BotSharp.MachineLearning.UnitTest/DecoderTest.cs
new file mode 100644
index 00000000..225bbea0
--- /dev/null
+++ b/BotSharp.MachineLearning.UnitTest/DecoderTest.cs
@@ -0,0 +1,287 @@
+using BotSharp.MachineLearning.CRFLite;
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.UnitTest
+{
+ [TestClass]
+ public class DecoderTest
+ {
+ [TestMethod]
+ public void TestDecode()
+ {
+ var encoder = new CRFDecoder();
+ bool bRet = Decode(new DecoderOptions
+ {
+
+ });
+ }
+
+ object rdLocker = new object();
+
+ bool Decode(DecoderOptions options)
+ {
+ var parallelOption = new ParallelOptions();
+ var watch = Stopwatch.StartNew();
+ if (File.Exists(options.strInputFileName) == false)
+ {
+ //Logger.WriteLine("FAILED: Open {0} file failed.", options.strInputFileName);
+ return false;
+ }
+
+ if (File.Exists(options.strModelFileName) == false)
+ {
+ //Logger.WriteLine("FAILED: Open {0} file failed.", options.strModelFileName);
+ return false;
+ }
+
+ var sr = new StreamReader(options.strInputFileName);
+ StreamWriter sw = null, swSeg = null;
+
+ if (options.strOutputFileName != null && options.strOutputFileName.Length > 0)
+ {
+ sw = new StreamWriter(options.strOutputFileName);
+ }
+ if (options.strOutputSegFileName != null && options.strOutputSegFileName.Length > 0)
+ {
+ swSeg = new StreamWriter(options.strOutputSegFileName);
+ }
+
+ //Create CRFSharp wrapper instance. It's a global instance
+ var crfWrapper = new CRFDecoder();
+
+ //Load encoded model from file
+ //Logger.WriteLine("Loading model from {0}", options.strModelFileName);
+ crfWrapper.LoadModel(options.strModelFileName);
+
+ var queueRecords = new ConcurrentQueue>>();
+ var queueSegRecords = new ConcurrentQueue>>();
+
+ parallelOption.MaxDegreeOfParallelism = options.thread;
+ Parallel.For(0, options.thread, parallelOption, t =>
+ {
+
+ //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
+ var tagger = crfWrapper.CreateTagger(options.nBest, options.maxword);
+ tagger.set_vlevel(options.probLevel);
+
+ //Initialize result
+ var crf_out = new crf_seg_out[options.nBest];
+ for (var i = 0; i < options.nBest; i++)
+ {
+ crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
+ }
+
+ var inbuf = new List>();
+ while (true)
+ {
+ lock (rdLocker)
+ {
+ if (ReadRecord(inbuf, sr) == false)
+ {
+ break;
+ }
+
+ queueRecords.Enqueue(inbuf);
+ queueSegRecords.Enqueue(inbuf);
+ }
+
+ //Call CRFSharp wrapper to predict given string's tags
+ if (swSeg != null)
+ {
+ crfWrapper.Segment(crf_out, tagger, inbuf);
+ }
+ else
+ {
+ crfWrapper.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf);
+ }
+
+ List> peek = null;
+ //Save segmented tagged result into file
+ if (swSeg != null)
+ {
+ var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out);
+ while (peek != inbuf)
+ {
+ queueSegRecords.TryPeek(out peek);
+ }
+ for (int index = 0; index < rstList.Count; index++)
+ {
+ var item = rstList[index];
+ swSeg.WriteLine(item);
+ }
+ queueSegRecords.TryDequeue(out peek);
+ peek = null;
+ }
+
+ //Save raw tagged result (with probability) into file
+ if (sw != null)
+ {
+ while (peek != inbuf)
+ {
+ queueRecords.TryPeek(out peek);
+ }
+ OutputRawResultToFile(inbuf, crf_out, tagger, sw);
+ queueRecords.TryDequeue(out peek);
+
+ }
+ }
+ });
+
+
+ sr.Close();
+
+ if (sw != null)
+ {
+ sw.Close();
+ }
+ if (swSeg != null)
+ {
+ swSeg.Close();
+ }
+ watch.Stop();
+ //Logger.WriteLine("Elapsed: {0} ms", watch.ElapsedMilliseconds);
+
+ return true;
+ }
+
+ private bool ReadRecord(List> inbuf, StreamReader sr)
+ {
+ inbuf.Clear();
+
+ while (true)
+ {
+ var strLine = sr.ReadLine();
+ if (strLine == null)
+ {
+ //At the end of current file
+ if (inbuf.Count == 0)
+ {
+ return false;
+ }
+ else
+ {
+ return true;
+ }
+ }
+ strLine = strLine.Trim();
+ if (strLine.Length == 0)
+ {
+ return true;
+ }
+
+ //Read feature set for each record
+ var items = strLine.Split(new char[] { '\t' });
+ inbuf.Add(new List());
+ for (int index = 0; index < items.Length; index++)
+ {
+ var item = items[index];
+ inbuf[inbuf.Count - 1].Add(item);
+ }
+ }
+ }
+
+ //Output raw result with probability
+ private void OutputRawResultToFile(List> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw)
+ {
+ for (var k = 0; k < crf_out.Length; k++)
+ {
+ if (crf_out[k] == null)
+ {
+ //No more result
+ break;
+ }
+
+ var sb = new StringBuilder();
+
+ var crf_seg_out = crf_out[k];
+ //Show the entire sequence probability
+ //For each token
+ for (var i = 0; i < inbuf.Count; i++)
+ {
+ //Show all features
+ for (var j = 0; j < inbuf[i].Count; j++)
+ {
+ sb.Append(inbuf[i][j]);
+ sb.Append("\t");
+ }
+
+ //Show the best result and its probability
+ sb.Append(crf_seg_out.result_[i]);
+
+ if (tagger.vlevel_ > 1)
+ {
+ sb.Append("\t");
+ sb.Append(crf_seg_out.weight_[i]);
+
+ //Show the probability of all tags
+ sb.Append("\t");
+ for (var j = 0; j < tagger.ysize_; j++)
+ {
+ sb.Append(tagger.yname(j));
+ sb.Append("/");
+ sb.Append(tagger.prob(i, j));
+
+ if (j < tagger.ysize_ - 1)
+ {
+ sb.Append("\t");
+ }
+ }
+ }
+ sb.AppendLine();
+ }
+ if (tagger.vlevel_ > 0)
+ {
+ sw.WriteLine("#{0}", crf_seg_out.prob);
+ }
+ sw.WriteLine(sb.ToString().Trim());
+ sw.WriteLine();
+ }
+ }
+
+ //Convert CRFSharp output format to string list
+ private List ConvertCRFTermOutToStringList(List> inbuf, crf_seg_out[] crf_out)
+ {
+ var sb = new StringBuilder();
+ for (var i = 0; i < inbuf.Count; i++)
+ {
+ sb.Append(inbuf[i][0]);
+ }
+
+ var strText = sb.ToString();
+ var rstList = new List();
+ for (var i = 0; i < crf_out.Length; i++)
+ {
+ if (crf_out[i] == null)
+ {
+ //No more result
+ break;
+ }
+
+ sb.Clear();
+ var crf_term_out = crf_out[i];
+ for (var j = 0; j < crf_term_out.Count; j++)
+ {
+ var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length);
+ var strNE = crf_term_out.tokenList[j].strTag;
+
+ sb.Append(str);
+ if (strNE.Length > 0)
+ {
+ sb.Append("[" + strNE + "]");
+ }
+ sb.Append(" ");
+ }
+ rstList.Add(sb.ToString().Trim());
+ }
+
+ return rstList;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning.UnitTest/EncoderTest.cs b/BotSharp.MachineLearning.UnitTest/EncoderTest.cs
new file mode 100644
index 00000000..1958e760
--- /dev/null
+++ b/BotSharp.MachineLearning.UnitTest/EncoderTest.cs
@@ -0,0 +1,22 @@
+using BotSharp.MachineLearning.CRFLite;
+using BotSharp.MachineLearning.CRFLite.Encoder;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace BotSharp.MachineLearning.UnitTest
+{
+ [TestClass]
+ public class EncoderTest
+ {
+ [TestMethod]
+ public void TestEncode()
+ {
+ var encoder = new CRFEncoder();
+ bool bRet = encoder.Learn(new EncoderOptions
+ {
+ TrainingCorpusFileName = "",
+ TemplateFileName = "",
+ ModelFileName = ""
+ });
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj b/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj
index d838d0ee..a5c52f3f 100644
--- a/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj
+++ b/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj
@@ -9,4 +9,10 @@
+
+
+ ..\..\..\..\..\..\Program Files\dotnet\sdk\NuGetFallbackFolder\microsoft.netcore.app\2.1.0\ref\netcoreapp2.1\System.ComponentModel.Annotations.dll
+
+
+
diff --git a/BotSharp.MachineLearning/CRFLite/BaseModel.cs b/BotSharp.MachineLearning/CRFLite/BaseModel.cs
new file mode 100644
index 00000000..e20925f3
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/BaseModel.cs
@@ -0,0 +1,154 @@
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class BaseModel
+ {
+ public long maxid_;
+ public double cost_factor_;
+
+ public List unigram_templs_;
+ public List bigram_templs_;
+
+ //Labeling tag list
+ public List y_;
+ public uint ysize() { return (uint)y_.Count; }
+
+ //The dimension training corpus
+ public uint xsize_;
+
+ //Feature set value array
+ public double[] alpha_;
+
+ public BaseModel()
+ {
+ cost_factor_ = 1.0;
+ }
+
+ //获取类别i的字符表示
+ public string y(int i) { return y_[i]; }
+
+ public long feature_size() { return maxid_; }
+
+ public StringBuilder apply_rule(string p, int pos, StringBuilder resultContainer, Tagger tagger)
+ {
+ resultContainer.Clear();
+ for (var i = 0; i < p.Length; i++)
+ {
+ if (p[i] == '%')
+ {
+ i++;
+ if (p[i] == 'x')
+ {
+ i++;
+ var res = get_index(p, pos, i, tagger);
+ i = res.idx;
+ if (res.value == null)
+ {
+ return null;
+ }
+ resultContainer.Append(res.value);
+ }
+ else
+ {
+ return null;
+ }
+ }
+ else
+ {
+ resultContainer.Append(p[i]);
+ }
+ }
+ return resultContainer;
+ }
+
+ Index get_index(string p, int pos, int i, Tagger tagger)
+ {
+ if (p[i] != '[')
+ {
+ return new Index(null, i);
+ }
+ i++;
+ var isInRow = true;
+ var col = 0;
+ var row = 0;
+ var neg = 1;
+
+ if (p[i] == '-')
+ {
+ neg = -1;
+ i++;
+ }
+
+ for (; i < p.Length; i++)
+ {
+ var c = p[i];
+ if (isInRow)
+ {
+ if (c >= '0' && c <= '9')
+ {
+ row = 10 * row + (c - '0');
+ }
+ else if (c == ',')
+ {
+ isInRow = false;
+ }
+ else
+ {
+ return new Index(null, i);
+ }
+ }
+ else
+ {
+ if (c >= '0' && c <= '9')
+ {
+ col = 10 * col + (c - '0');
+ }
+ else if (c == ']')
+ {
+ break;
+ }
+ else
+ {
+ return new Index(null, i);
+ }
+ }
+ }
+
+ row *= neg;
+
+ if (col < 0 || col >= xsize_)
+ {
+ return new Index(null, i);
+ }
+ var idx = pos + row;
+ if (idx < 0)
+ {
+ return new Index("_B-" + (-idx).ToString(), i); ;
+ }
+ if (idx >= tagger.word_num)
+ {
+ return new Index("_B+" + (idx - tagger.word_num + 1).ToString(), i);
+ }
+
+ return new Index(tagger.x_[idx][col], i);
+
+ }
+
+ private struct Index
+ {
+ public int idx;
+ public string value;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ public Index(string value, int idx)
+ {
+ this.idx = idx;
+ this.value = value;
+ }
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs b/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs
new file mode 100644
index 00000000..931ba99e
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/CRFDecoder.cs
@@ -0,0 +1,144 @@
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class CRFDecoder
+ {
+ ModelReader _modelReader;
+
+ ///
+ /// Load encoded model from file
+ ///
+ ///
+ /// The model path.
+ ///
+ ///
+ public void LoadModel(string modelFilename)
+ {
+ _modelReader = new ModelReader(modelFilename);
+ _modelReader.LoadModel();
+ }
+
+ ///
+ /// Loads an encoded model using the specified delegate.
+ /// Using this overload you can read the model e.g.
+ /// from network, zipped archives or other locations, as you wish.
+ ///
+ ///
+ /// Allows reading the model from arbitrary formats and sources.
+ ///
+ ///
+ /// The model file name, as used by the given
+ /// for file resolution.
+ ///
+ ///
+ public void LoadModel(Func modelLoader, string modelFilename)
+ {
+ this._modelReader = new ModelReader(modelLoader, modelFilename);
+ _modelReader.LoadModel();
+ }
+
+ public SegDecoderTagger CreateTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+ {
+ if (_modelReader == null)
+ {
+ return null;
+ }
+
+ var tagger = new SegDecoderTagger(nbest, this_crf_max_word_num);
+ tagger.init_by_model(_modelReader);
+
+ return tagger;
+ }
+
+ //Segment given text
+ public int Segment(crf_seg_out[] pout, //segment result
+ SegDecoderTagger tagger, //Tagger per thread
+ List> inbuf //feature set for segment
+ )
+ {
+ var ret = 0;
+ if (inbuf.Count == 0)
+ {
+ //Empty input string
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ ret = tagger.reset();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ ret = tagger.add(inbuf);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ //parse
+ ret = tagger.parse();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ //wrap result
+ ret = tagger.output(pout);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+
+
+ //Segment given text
+ public int Segment(CRFTermOut[] pout, //segment result
+ DecoderTagger tagger, //Tagger per thread
+ List> inbuf //feature set for segment
+ )
+ {
+ var ret = 0;
+ if (inbuf.Count == 0)
+ {
+ //Empty input string
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ ret = tagger.reset();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ ret = tagger.add(inbuf);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ //parse
+ ret = tagger.parse();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ //wrap result
+ ret = tagger.output(pout);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs b/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs
new file mode 100644
index 00000000..fb228bb8
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/CRFEncoder.cs
@@ -0,0 +1,317 @@
+using BotSharp.MachineLearning.CRFLite.Encoder;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class CRFEncoder
+ {
+ public enum REG_TYPE { L1, L2 };
+
+ //encoding CRF model from training corpus
+ public bool Learn(EncoderOptions args)
+ {
+ if (args.MinDifference <= 0.0)
+ {
+ return false;
+ }
+
+ if (args.CostFactor < 0.0)
+ {
+ return false;
+ }
+
+ if (args.ThreadsNum <= 0)
+ {
+ return false;
+ }
+
+ if (args.HugeLexMemLoad > 0)
+ {
+ }
+
+ var modelWriter = new ModelWriter(args.ThreadsNum, args.CostFactor,
+ args.HugeLexMemLoad, args.RetrainModelFileName);
+
+ if (modelWriter.Open(args.TemplateFileName, args.TrainingCorpusFileName) == false)
+ {
+ return false;
+ }
+
+ var xList = modelWriter.ReadAllRecords();
+
+
+ modelWriter.Shrink(xList, args.MinFeatureFreq);
+
+ if (!modelWriter.SaveModelMetaData(args.ModelFileName))
+ {
+ return false;
+ }
+ else
+ {
+ }
+
+ if (!modelWriter.BuildFeatureSetIntoIndex(args.ModelFileName, args.SlotUsageRateThreshold, args.DebugLevel))
+ {
+ return false;
+ }
+ else
+ {
+ }
+
+ if (xList.Length == 0)
+ {
+ return false;
+ }
+
+ var orthant = false;
+ if (args.RegType == REG_TYPE.L1)
+ {
+ orthant = true;
+ }
+ if (runCRF(xList, modelWriter, orthant, args) == false)
+ {
+ }
+
+ modelWriter.SaveFeatureWeight(args.ModelFileName, args.BVQ);
+
+ return true;
+ }
+
+ bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args)
+ {
+ var old_obj = double.MaxValue;
+ var converge = 0;
+ var lbfgs = new LBFGS(args.ThreadsNum);
+ lbfgs.expected = new double[modelWriter.feature_size() + 1];
+
+ var processList = new List();
+ var parallelOption = new ParallelOptions();
+ parallelOption.MaxDegreeOfParallelism = args.ThreadsNum;
+
+ //Initialize encoding threads
+ for (var i = 0; i < args.ThreadsNum; i++)
+ {
+ var thread = new CRFEncoderThread();
+ thread.start_i = i;
+ thread.thread_num = args.ThreadsNum;
+ thread.x = x;
+ thread.lbfgs = lbfgs;
+ thread.Init();
+ processList.Add(thread);
+ }
+
+ //Statistic term and result tags frequency
+ var termNum = 0;
+ int[] yfreq;
+ yfreq = new int[modelWriter.y_.Count];
+ for (int index = 0; index < x.Length; index++)
+ {
+ var tagger = x[index];
+ termNum += tagger.word_num;
+ for (var j = 0; j < tagger.word_num; j++)
+ {
+ yfreq[tagger.answer_[j]]++;
+ }
+ }
+
+ //Iterative training
+ var startDT = DateTime.Now;
+ var dMinErrRecord = 1.0;
+ for (var itr = 0; itr < args.MaxIteration; ++itr)
+ {
+ //Clear result container
+ lbfgs.obj = 0.0f;
+ lbfgs.err = 0;
+ lbfgs.zeroone = 0;
+
+ Array.Clear(lbfgs.expected, 0, lbfgs.expected.Length);
+
+ var threadList = new List();
+ for (var i = 0; i < args.ThreadsNum; i++)
+ {
+ var thread = new Thread(processList[i].Run);
+ thread.Start();
+ threadList.Add(thread);
+ }
+
+ int[,] merr;
+ merr = new int[modelWriter.y_.Count, modelWriter.y_.Count];
+ for (var i = 0; i < args.ThreadsNum; ++i)
+ {
+ threadList[i].Join();
+ lbfgs.obj += processList[i].obj;
+ lbfgs.err += processList[i].err;
+ lbfgs.zeroone += processList[i].zeroone;
+
+ //Calculate error
+ for (var j = 0; j < modelWriter.y_.Count; j++)
+ {
+ for (var k = 0; k < modelWriter.y_.Count; k++)
+ {
+ merr[j, k] += processList[i].merr[j, k];
+ }
+ }
+ }
+
+ long num_nonzero = 0;
+ var fsize = modelWriter.feature_size();
+ var alpha = modelWriter.alpha_;
+ if (orthant == true)
+ {
+ //L1 regularization
+ Parallel.For(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) =>
+ {
+ subtotal += Math.Abs(alpha[k] / modelWriter.cost_factor_);
+ if (alpha[k] != 0.0)
+ {
+ Interlocked.Increment(ref num_nonzero);
+ }
+ return subtotal;
+ },
+ (subtotal) => // lock free accumulator
+ {
+ double initialValue;
+ double newValue;
+ do
+ {
+ initialValue = lbfgs.obj; // read current value
+ newValue = initialValue + subtotal; //calculate new value
+ }
+ while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue));
+ });
+ }
+ else
+ {
+ //L2 regularization
+ num_nonzero = fsize;
+ Parallel.For(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) =>
+ {
+ subtotal += (alpha[k] * alpha[k] / (2.0 * modelWriter.cost_factor_));
+ lbfgs.expected[k] += (alpha[k] / modelWriter.cost_factor_);
+ return subtotal;
+ },
+ (subtotal) => // lock free accumulator
+ {
+ double initialValue;
+ double newValue;
+ do
+ {
+ initialValue = lbfgs.obj; // read current value
+ newValue = initialValue + subtotal; //calculate new value
+ }
+ while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue));
+ });
+ }
+
+ //Show each iteration result
+ var diff = (itr == 0 ? 1.0f : Math.Abs(old_obj - lbfgs.obj) / old_obj);
+ old_obj = lbfgs.obj;
+
+ ShowEvaluation(x.Length, modelWriter, lbfgs, termNum, itr, merr, yfreq, diff, startDT, num_nonzero, args);
+ if (diff < args.MinDifference)
+ {
+ converge++;
+ }
+ else
+ {
+ converge = 0;
+ }
+ if (itr > args.MaxIteration || converge == 3)
+ {
+ break; // 3 is ad-hoc
+ }
+
+ if (args.DebugLevel > 0 && (double)lbfgs.zeroone / (double)x.Length < dMinErrRecord)
+ {
+ var cc = Console.ForegroundColor;
+ Console.ForegroundColor = ConsoleColor.Red;
+ Console.Write("[Debug Mode] ");
+ Console.ForegroundColor = cc;
+
+ //Save current best feature weight into file
+ dMinErrRecord = (double)lbfgs.zeroone / (double)x.Length;
+ modelWriter.SaveFeatureWeight("feature_weight_tmp", false);
+ }
+
+ int iret;
+ iret = lbfgs.optimize(alpha, modelWriter.cost_factor_, orthant);
+ if (iret <= 0)
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private static void ShowEvaluation(int recordNum, ModelWriter feature_index, LBFGS lbfgs, int termNum, int itr, int[,] merr, int[] yfreq, double diff, DateTime startDT, long nonzero_feature_num, EncoderOptions args)
+ {
+ var ts = DateTime.Now - startDT;
+
+ if (args.DebugLevel > 1)
+ {
+ for (var i = 0; i < feature_index.y_.Count; i++)
+ {
+ var total_merr = 0;
+ var sdict = new SortedDictionary>();
+ for (var j = 0; j < feature_index.y_.Count; j++)
+ {
+ total_merr += merr[i, j];
+ var v = (double)merr[i, j] / (double)yfreq[i];
+ if (v > 0.0001)
+ {
+ if (sdict.ContainsKey(v) == false)
+ {
+ sdict.Add(v, new List());
+ }
+ sdict[v].Add(feature_index.y_[j]);
+ }
+ }
+ var vet = (double)total_merr / (double)yfreq[i];
+ vet = vet * 100.0F;
+
+ Console.ForegroundColor = ConsoleColor.Green;
+ Console.Write("{0} ", feature_index.y_[i]);
+ Console.ResetColor();
+ Console.Write("[FR={0}, TE=", yfreq[i]);
+ Console.ForegroundColor = ConsoleColor.Yellow;
+ Console.Write("{0:0.00}%", vet);
+ Console.ResetColor();
+ Console.WriteLine("]");
+
+ var n = 0;
+ foreach (var pair in sdict.Reverse())
+ {
+ for (int index = 0; index < pair.Value.Count; index++)
+ {
+ var item = pair.Value[index];
+ n += item.Length + 1 + 7;
+ if (n > 80)
+ {
+ //only show data in one line, more data in tail will not be show.
+ break;
+ }
+ Console.Write("{0}:", item);
+ Console.ForegroundColor = ConsoleColor.Red;
+ Console.Write("{0:0.00}% ", pair.Key * 100);
+ Console.ResetColor();
+ }
+ if (n > 80)
+ {
+ break;
+ }
+ }
+ Console.WriteLine();
+ }
+ }
+
+ var act_feature_rate = (double)(nonzero_feature_num) / (double)(feature_index.feature_size()) * 100.0;
+ //Logger.WriteLine("iter={0} terr={1:0.00000} serr={2:0.00000} diff={3:0.000000} fsize={4}({5:0.00}% act)", itr, 1.0 * lbfgs.err / termNum, 1.0 * lbfgs.zeroone / recordNum, diff, feature_index.feature_size(), act_feature_rate);
+ //Logger.WriteLine("Time span: {0}, Aver. time span per iter: {1}", ts, new TimeSpan(0, 0, (int)(ts.TotalSeconds / (itr + 1))));
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/CRFSharpHelper.cs b/BotSharp.MachineLearning/CRFLite/CRFSharpHelper.cs
new file mode 100644
index 00000000..674133a3
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/CRFSharpHelper.cs
@@ -0,0 +1,41 @@
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class SegToken
+ {
+ public int offset;
+ public int length;
+ public string strTag; //CRF对应于term组合后的Tag字符串
+ public double fWeight; //对应属性id的概率值，或者得分
+ };
+
+ public class crf_seg_out : CRFTermOut
+ {
+ //Segmented token by merging raw CRF model output
+ public int termTotalLength; // the total term length in character
+ public List tokenList;
+
+ public int Count
+ {
+ get { return tokenList.Count; }
+ }
+
+ public void Clear()
+ {
+ termTotalLength = 0;
+ tokenList.Clear();
+ }
+
+ public crf_seg_out(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM):
+ base(max_word_num)
+ {
+ termTotalLength = 0;
+ tokenList = new List();
+ }
+ };
+
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Decoder/CRFTermOut.cs b/BotSharp.MachineLearning/CRFLite/Decoder/CRFTermOut.cs
new file mode 100644
index 00000000..69468a43
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Decoder/CRFTermOut.cs
@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+ public class CRFTermOut
+ {
+ //Sequence label probability
+ public double prob;
+
+ //Raw CRF model output
+ public string[] result_;
+ public double[] weight_;
+
+ public CRFTermOut(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+ {
+ prob = 0;
+ result_ = new string[max_word_num];
+ weight_ = new double[max_word_num];
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Decoder/DecoderOptions.cs b/BotSharp.MachineLearning/CRFLite/Decoder/DecoderOptions.cs
new file mode 100644
index 00000000..4bf694e9
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Decoder/DecoderOptions.cs
@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+ public class DecoderOptions
+ {
+ [Required]
+ public string strModelFileName;
+ [Required]
+ public string strInputFileName;
+ public string strOutputFileName;
+ public string strOutputSegFileName;
+ public int nBest;
+ public int thread;
+ public int probLevel;
+ public int maxword;
+
+ public DecoderOptions()
+ {
+ thread = 1;
+ nBest = 1;
+ probLevel = 0;
+ maxword = 100;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs b/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs
new file mode 100644
index 00000000..581fa235
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs
@@ -0,0 +1,471 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+ public class DecoderTagger : Tagger
+ {
+ private readonly Pool _buildersPool =
+ new Pool(p => new StringBuilder(100), b => b.Clear());
+
+ public int forward_backward_stat; //前向后向过程运行状态，0为未运行，1为已经运行
+
+ //概率计算函数
+ double toprob(Node n, double Z)
+ {
+ return Math.Exp(n.alpha + n.beta - n.cost - Z);
+ }
+
+ //To get the fastest decoded result, please set vlevel=0 and nbest=1, since it only outputs 1-best result without probability (forward-backward and A* aren't performed, only run viterbi)
+ public int vlevel_; //Need to calculate probability 0 - no need to calculate, 1 - calculate sequence label probability, 2 - calculate both sequence label and individual entity probability
+ protected int nbest_; //output top N-best result
+ //CrfModel model;
+ ModelReader featureIndex;
+
+ Node node(int i, int j)
+ {
+ return node_[i, j];
+ }
+
+ Heap heap_queue; //Using min-heap to get next result, it's only used when nbest > 1
+ public int crf_max_word_num;
+
+ public DecoderTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+ {
+ crf_max_word_num = this_crf_max_word_num;
+ vlevel_ = 0;
+ nbest_ = nbest;
+ cost_ = 0.0;
+ Z_ = 0;
+
+ ysize_ = 0;
+ word_num = 0;
+ heap_queue = null;
+ node_ = null;
+ x_ = null;
+ result_ = null;
+ }
+
+ public void InitializeFeatureCache()
+ {
+ feature_cache_ = new List();
+ var feature_cache_every_row_size = 0;
+ if (featureIndex.unigram_templs_.Count > featureIndex.bigram_templs_.Count)
+ {
+ feature_cache_every_row_size = featureIndex.unigram_templs_.Count + 1;
+ }
+ else
+ {
+ feature_cache_every_row_size = featureIndex.bigram_templs_.Count + 1;
+ }
+ for (var i = 0; i < crf_max_word_num * 2; i++)
+ {
+ var features = new long[feature_cache_every_row_size];
+ for (var j = 0; j < feature_cache_every_row_size; j++)
+ {
+ features[j] = -1;
+ }
+ feature_cache_.Add(features);
+ }
+ }
+
+ //获取序列的词数
+ public short get_word_num()
+ {
+ return word_num;
+ }
+
+ public double prob(int i, int j)
+ {
+ return toprob(node_[i, j], Z_);
+ }
+
+ //Get the probability of the i-th word's best result
+ public double prob(int i)
+ {
+ return toprob(node_[i, result_[i]], Z_);
+ }
+
+ //Get entire sequence probability
+ public double prob()
+ {
+ return Math.Exp(-cost_ - Z_);
+ }
+
+ //Get the string of i-th tag
+ public string yname(int i) { return featureIndex.y(i); }
+
+ //设置vlevel
+ public void set_vlevel(int vlevel_value)
+ {
+ vlevel_ = vlevel_value;
+ }
+
+ //使用模型初始化tag，必须先使用该函数初始化才能使用add和parse
+ //正常返回为0， 错误返回<0
+ public int init_by_model(ModelReader model_p)
+ {
+ featureIndex = model_p;
+ ysize_ = (short)model_p.ysize();
+
+ if (nbest_ > 1)
+ {
+ //Only allocate heap when nbest is more than 1
+ heap_queue = BaseUtils.heap_init((int)(crf_max_word_num * ysize_ * ysize_));
+ }
+
+ //Initialize feature set cache according unigram and bigram templates
+ InitializeFeatureCache();
+
+ node_ = new Node[crf_max_word_num, ysize_];
+ result_ = new short[crf_max_word_num];
+
+ //Create node and path cache
+ for (short cur = 0; cur < crf_max_word_num; cur++)
+ {
+ for (short i = 0; i < ysize_; i++)
+ {
+ var n = new Node();
+ node_[cur, i] = n;
+
+ n.lpathList = new List();
+ n.rpathList = new List();
+ n.x = cur;
+ n.y = i;
+ }
+ }
+
+ for (var cur = 1; cur < crf_max_word_num; cur++)
+ {
+ for (var j = 0; j < ysize_; ++j)
+ {
+ for (var i = 0; i < ysize_; ++i)
+ {
+ var p = new Path();
+ p.add(node_[cur - 1, j], node_[cur, i]);
+ }
+ }
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ public int initNbest()
+ {
+ var k = (int)word_num - 1;
+ for (var i = 0; i < ysize_; ++i)
+ {
+ var eos = BaseUtils.allc_from_heap(heap_queue);
+ eos.node = node_[k, i];
+ eos.fx = -node_[k, i].bestCost;
+ eos.gx = -node_[k, i].cost;
+ eos.next = null;
+ if (BaseUtils.heap_insert(eos, heap_queue) < 0)
+ {
+ return BaseUtils.ERROR_INSERT_HEAP_FAILED;
+ }
+ }
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ public int next()
+ {
+ while (!BaseUtils.is_heap_empty(heap_queue))
+ {
+ var top = BaseUtils.heap_delete_min(heap_queue);
+ var rnode = top.node;
+
+ if (rnode.x == 0)
+ {
+ for (var n = top; n != null; n = n.next)
+ {
+ result_[n.node.x] = n.node.y;
+ }
+ cost_ = top.gx;
+ return 0;
+ }
+
+ for (int index = 0; index < rnode.lpathList.Count; index++)
+ {
+ var p = rnode.lpathList[index];
+ var n = BaseUtils.allc_from_heap(heap_queue);
+ var x_num = (rnode.x) - 1;
+ n.node = p.lnode;
+ n.gx = -p.lnode.cost - p.cost + top.gx;
+ n.fx = -p.lnode.bestCost - p.cost + top.gx;
+ // | h(x) | | g(x) |
+ n.next = top;
+ if (BaseUtils.heap_insert(n, heap_queue) < 0)
+ {
+ return BaseUtils.ERROR_INSERT_HEAP_FAILED;
+ }
+ }
+ }
+ return 0;
+ }
+
+ public int reset()
+ {
+ word_num = 0;
+ Z_ = cost_ = 0.0;
+
+ BaseUtils.heap_reset(heap_queue);
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ int buildLattice()
+ {
+ //Generate feature ids for all nodes and paths
+ RebuildFeatures();
+
+ for (int i = 0; i < word_num; ++i)
+ {
+ for (int j = 0; j < ysize_; ++j)
+ {
+ var currentNode = node_[i, j];
+ calcCost(currentNode);
+ for (int index = 0; index < currentNode.lpathList.Count; ++index)
+ {
+ var p = currentNode.lpathList[index];
+ calcCost(p);
+ }
+ }
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ public int add(List> row_p)
+ {
+ x_ = row_p;
+ word_num = (short)x_.Count;
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+
+ public int termbuf_build(CRFTermOut term_buf)
+ {
+ if (vlevel_ > 0)
+ {
+ //Calcuate the sequence label probability
+ term_buf.prob = prob();
+ }
+
+ var this_word_num = get_word_num();
+
+ for (var i = 0; i < this_word_num; ++i)
+ {
+ term_buf.result_[i] = yname(result_[i]);
+ switch (vlevel_)
+ {
+ case 0:
+ term_buf.weight_[i] = 0.0;
+ break;
+ case 2:
+ term_buf.weight_[i] = prob(i);
+ break;
+ }
+ }
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ //Label input string. The result is saved as result []
+ //If nbest > 1, get nbest result by "next"
+ //Returen value: Successed - 0, Failed < 0
+ public int parse()
+ {
+ var ret = 0;
+ //no word need to be labeled
+ if (word_num == 0)
+ {
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ //building feature set
+ ret = buildFeatures();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+
+ ret = buildLattice();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ //4.forward-backward when we need to calcuate probability
+ if (vlevel_ > 0)
+ {
+ forwardbackward();
+ }
+
+
+ //5.using viterbi to search best result path
+ ret = viterbi();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ //6.initNbest
+ // 求nbest(n>1)时的数据结构初始化，此后可以调用next()来获取nbest结果
+ if (nbest_ > 1)
+ {
+ //如果只求1-best，不需要使用initNbest()和next()获取结果
+ ret = initNbest();
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+
+ public int buildFeatures()
+ {
+ if (word_num <= 0)
+ {
+ return BaseUtils.ERROR_INVALIDATED_PARAMETER;
+ }
+ using (var v = _buildersPool.GetOrCreate())
+ {
+ var builder = v.Item;
+ var id = 0;
+ var feature_cache_row_size = 0;
+ var feature_cache_size = 0;
+ for (var cur = 0; cur < word_num; cur++)
+ {
+ feature_cache_row_size = 0;
+ for (int index = 0; index < featureIndex.unigram_templs_.Count; index++)
+ {
+ var templ = featureIndex.unigram_templs_[index];
+ var res = featureIndex.apply_rule(templ, cur, builder, this);
+ if (res == null)
+ {
+ return BaseUtils.ERROR_EMPTY_FEATURE;
+ }
+ id = featureIndex.get_id(res.ToString());
+ if (id != -1)
+ {
+ feature_cache_[feature_cache_size][feature_cache_row_size] = id;
+ feature_cache_row_size++;
+ }
+ }
+ feature_cache_[feature_cache_size][feature_cache_row_size] = -1;
+ feature_cache_size++;
+ }
+
+ for (var cur = 0; cur < word_num; cur++)
+ {
+ feature_cache_row_size = 0;
+ for (int index = 0; index < featureIndex.bigram_templs_.Count; index++)
+ {
+ var templ = featureIndex.bigram_templs_[index];
+ var strFeature = featureIndex.apply_rule(templ, cur, builder, this);
+ if (strFeature == null)
+ {
+ return BaseUtils.ERROR_EMPTY_FEATURE;
+ }
+
+ id = featureIndex.get_id(strFeature.ToString());
+ if (id != -1)
+ {
+ feature_cache_[feature_cache_size][feature_cache_row_size] = id;
+ feature_cache_row_size++;
+ }
+ }
+ feature_cache_[feature_cache_size][feature_cache_row_size] = -1;
+ feature_cache_size++;
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+ }
+
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public void calcCost(Node n)
+ {
+ double c = 0;
+ var f = feature_cache_[n.fid];
+
+ for (int i = 0; i < f.Length; ++i)
+ {
+ int fCurrent = (int)f[i];
+ if (fCurrent == -1)
+ break;
+ c += featureIndex.GetAlpha(fCurrent + n.y);
+ }
+
+ n.cost = featureIndex.cost_factor_ * c;
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public void calcCost(Path p)
+ {
+ double c = 0;
+ long[] f = feature_cache_[p.fid];
+ for (int i = 0; i < f.Length; ++i)
+ {
+ int fCurrent = (int)f[i];
+ if (fCurrent == -1)
+ break;
+ c += featureIndex.GetAlpha((fCurrent + p.lnode.y * ysize_ + p.rnode.y));
+ }
+
+ p.cost = featureIndex.cost_factor_ * c;
+ }
+
+
+ public int output(CRFTermOut[] pout)
+ {
+ var n = 0;
+ var ret = 0;
+
+ if (nbest_ == 1)
+ {
+ //If only best result and no need probability, "next" is not to be used
+ ret = termbuf_build(pout[0]);
+ if (ret < 0)
+ {
+ return ret;
+ }
+ }
+ else
+ {
+ //Fill the n best result
+ var iNBest = nbest_;
+ if (pout.Length < iNBest)
+ {
+ iNBest = pout.Length;
+ }
+
+ for (n = 0; n < iNBest; ++n)
+ {
+ ret = next();
+ if (ret < 0)
+ {
+ break;
+ }
+
+ ret = termbuf_build(pout[n]);
+ if (ret < 0)
+ {
+ return ret;
+ }
+ }
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Decoder/ModelReader.cs b/BotSharp.MachineLearning/CRFLite/Decoder/ModelReader.cs
new file mode 100644
index 00000000..d20a44be
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Decoder/ModelReader.cs
@@ -0,0 +1,251 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using BotSharp.MachineLearning.CRFLite.Utils;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+ public class ModelReader : BaseModel
+ {
+ private readonly Func modelLoader = null;
+
+ public uint version; //模型版本号,读取模型时读入
+ private CRFLite.Utils.DoubleArrayTrieSearch da; //特征集合
+
+ ///
+ /// Returns the model path.
+ ///
+ public string ModelPath { get; private set; }
+
+ ///
+ /// Creates a new
+ /// that will load the model from the file system,
+ /// using the given .
+ ///
+ ///
+ /// Path to the model.
+ ///
+ public ModelReader(string modelPath) :
+ this(GetStreamFromFileSystem, modelPath)
+ {
+
+ }
+
+ ///
+ /// Creates a new
+ /// that will load the model from the file system,
+ /// using the given .
+ ///
+ ///
+ /// A delegate capable of resolving
+ /// the given
+ /// into a stream with the model file.
+ ///
+ ///
+ /// Path to the model.
+ ///
+ public ModelReader(Func modelLoader,
+ string modelPath)
+ {
+ this.modelLoader = modelLoader;
+ this.ModelPath = modelPath;
+ }
+
+ ///
+ /// Loads the model into memory.
+ ///
+ public void LoadModel()
+ {
+ //Load model meta data
+ LoadMetadata();
+
+ //Load all feature set data
+ LoadFeatureSet();
+
+ //Load all features alpha data
+ LoadFeatureWeights();
+ }
+
+ //获取key对应的特征id
+ public virtual int get_id(string str)
+ {
+ return da.SearchByPerfectMatch(str);
+ }
+
+ public virtual double GetAlpha(long index)
+ {
+ return alpha_[index];
+ }
+
+ ///
+ /// The default model loading strategy -
+ /// load files from the file system.
+ ///
+ ///
+ /// Model file path.
+ ///
+ /// A stream containing the requested file.
+ ///
+ private static Stream GetStreamFromFileSystem(string path)
+ {
+ path.ThrowIfNotExists();
+ return File.OpenRead(path);
+ }
+
+ ///
+ /// Provides access to the metadata stream.
+ ///
+ ///
+ /// A instance
+ /// that points to the model metadata file.
+ ///
+ private Stream GetMetadataStream()
+ {
+ string path = ModelPath.ToMetadataModelName();
+
+ return modelLoader(path);
+ }
+
+ ///
+ /// Provides access to the feature set stream.
+ ///
+ ///
+ /// A instance
+ /// that allows accessing the model feature set file.
+ ///
+ private Stream GetFeatureSetStream()
+ {
+ string path = ModelPath.ToFeatureSetFileName();
+
+ return modelLoader(path);
+ }
+
+ ///
+ /// Provides access to the feature set stream.
+ ///
+ ///
+ /// A instance
+ /// that allows accessing the model feature weight file.
+ ///
+ private Stream GetFeatureWeightStream()
+ {
+ string path = ModelPath.ToFeatureWeightFileName();
+
+ return modelLoader(path);
+ }
+
+ private void LoadMetadata()
+ {
+ using (Stream metadataStream = GetMetadataStream())
+ {
+ var sr = new StreamReader(metadataStream);
+ string strLine;
+
+ //读入版本号
+ strLine = sr.ReadLine();
+ version = uint.Parse(strLine.Split(':')[1].Trim());
+
+ //读入cost_factor
+ strLine = sr.ReadLine();
+ cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());
+
+ //读入maxid
+ strLine = sr.ReadLine();
+ maxid_ = long.Parse(strLine.Split(':')[1].Trim());
+
+ //读入xsize
+ strLine = sr.ReadLine();
+ xsize_ = uint.Parse(strLine.Split(':')[1].Trim());
+
+ //读入空行
+ strLine = sr.ReadLine();
+
+ //读入待标注的标签
+ y_ = new List();
+ while (true)
+ {
+ strLine = sr.ReadLine();
+ if (strLine.Length == 0)
+ {
+ break;
+ }
+ y_.Add(strLine);
+ }
+
+ //读入unigram和bigram模板
+ unigram_templs_ = new List();
+ bigram_templs_ = new List();
+ while (sr.EndOfStream == false)
+ {
+ strLine = sr.ReadLine();
+ if (strLine.Length == 0)
+ {
+ break;
+ }
+ if (strLine[0] == 'U')
+ {
+ unigram_templs_.Add(strLine);
+ }
+ if (strLine[0] == 'B')
+ {
+ bigram_templs_.Add(strLine);
+ }
+ }
+ sr.Close();
+ }
+ }
+
+ private void LoadFeatureSet()
+ {
+ Stream featureSetStream = GetFeatureSetStream();
+ da = new DoubleArrayTrieSearch();
+ da.Load(featureSetStream);
+ }
+
+ private void LoadFeatureWeights()
+ {
+ //feature weight array
+ alpha_ = new double[maxid_ + 1];
+
+ using (Stream featureWeightStream = GetFeatureWeightStream())
+ {
+ //Load all features alpha data
+ var sr_alpha = new StreamReader(featureWeightStream);
+ var br_alpha = new BinaryReader(sr_alpha.BaseStream);
+
+ //Get VQ Size
+ int vqSize = br_alpha.ReadInt32();
+
+ if (vqSize > 0)
+ {
+ //This is a VQ model, we need to get code book at first
+ List vqCodeBook = new List();
+ for (int i = 0; i < vqSize; i++)
+ {
+ vqCodeBook.Add(br_alpha.ReadDouble());
+ }
+
+ //Load weights
+ for (long i = 0; i < maxid_; i++)
+ {
+ int vqIdx = br_alpha.ReadByte();
+ alpha_[i] = vqCodeBook[vqIdx];
+ }
+ }
+ else
+ {
+ //This is a normal model
+ for (long i = 0; i < maxid_; i++)
+ {
+ alpha_[i] = br_alpha.ReadSingle();
+ }
+ }
+
+ br_alpha.Close();
+ }
+ }
+
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Decoder/ModelReaderExtensions.cs b/BotSharp.MachineLearning/CRFLite/Decoder/ModelReaderExtensions.cs
new file mode 100644
index 00000000..fa4602e8
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Decoder/ModelReaderExtensions.cs
@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+ internal static class ModelReaderExtensions
+ {
+ private static readonly string featureFileNameExtension = ".feature";
+ private static readonly string weightFileNameExtension = ".alpha";
+
+ internal static string ToMetadataModelName(this string modelName)
+ {
+ return modelName;
+ }
+
+ internal static string ToFeatureSetFileName(this string modelName)
+ {
+ return String.Concat(modelName, featureFileNameExtension);
+ }
+
+ internal static string ToFeatureWeightFileName(this string modelName)
+ {
+ return String.Concat(modelName, weightFileNameExtension);
+ }
+
+ internal static void ThrowIfNotExists(this string fileName)
+ {
+ if (String.IsNullOrWhiteSpace(fileName))
+ throw new ArgumentNullException("fileName",
+ "Please specify a valid model path");
+
+ if (!File.Exists(fileName))
+ throw new FileNotFoundException("fileName",
+ "Please specify a valid model path");
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/CRFEncoderThread.cs b/BotSharp.MachineLearning/CRFLite/Encoder/CRFEncoderThread.cs
new file mode 100644
index 00000000..dfd04c3d
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/CRFEncoderThread.cs
@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public class CRFEncoderThread
+ {
+ public EncoderTagger[] x;
+ public int start_i;
+ public int thread_num;
+ public int zeroone;
+ public int err;
+ public double obj;
+ public Node[,] node_;
+ short[] result_;
+ public short max_xsize_;
+ public LBFGS lbfgs;
+ public int[,] merr;
+
+ public void Init()
+ {
+ if (x.Length == 0)
+ {
+ return;
+ }
+
+ var ysize_ = x[0].ysize_;
+ max_xsize_ = 0;
+ for (var i = start_i; i < x.Length; i += thread_num)
+ {
+ if (max_xsize_ < x[i].word_num)
+ {
+ max_xsize_ = x[i].word_num;
+ }
+ }
+
+ result_ = new short[max_xsize_];
+ node_ = new Node[max_xsize_, ysize_];
+ for (var i = 0; i < max_xsize_; i++)
+ {
+ for (var j = 0; j < ysize_; j++)
+ {
+ node_[i, j] = new Node();
+ node_[i, j].x = (short)i;
+ node_[i, j].y = (short)j;
+ node_[i, j].lpathList = new List(ysize_);
+ node_[i, j].rpathList = new List(ysize_);
+ }
+ }
+
+ for (short cur = 1; cur < max_xsize_; ++cur)
+ {
+ for (short j = 0; j < ysize_; ++j)
+ {
+ for (short i = 0; i < ysize_; ++i)
+ {
+ var path = new Path();
+ path.fid = -1;
+ path.cost = 0.0;
+ path.add(node_[cur - 1, j], node_[cur, i]);
+ }
+ }
+ }
+
+ merr = new int[ysize_, ysize_];
+ }
+
+ public void Run()
+ {
+ //Initialize thread self data structure
+ obj = 0.0f;
+ err = zeroone = 0;
+ //expected.Clear();
+ Array.Clear(merr, 0, merr.Length);
+ for (var i = start_i; i < x.Length; i += thread_num)
+ {
+ x[i].Init(result_, node_);
+ obj += x[i].gradient(lbfgs.expected);
+ var error_num = x[i].eval(merr);
+ err += error_num;
+ if (error_num > 0)
+ {
+ ++zeroone;
+ }
+ }
+ }
+
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs b/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs
new file mode 100644
index 00000000..797d27da
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs
@@ -0,0 +1,125 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public class DefaultFeatureLexicalDict : IFeatureLexicalDict
+ {
+ CRFLite.Utils.BTreeDictionary featureset_dict_;
+ long maxid_;
+ Object thisLock = new object();
+ ParallelOptions parallelOption;
+
+ public DefaultFeatureLexicalDict(int thread_num)
+ {
+ featureset_dict_ = new CRFLite.Utils.BTreeDictionary(StringComparer.Ordinal, 128);
+ maxid_ = 0;
+ parallelOption = new ParallelOptions();
+ parallelOption.MaxDegreeOfParallelism = thread_num;
+ }
+
+ public void Clear()
+ {
+ featureset_dict_.Clear();
+ featureset_dict_ = null;
+ }
+
+ public long Size
+ {
+ get
+ {
+ return featureset_dict_.Count;
+ }
+ }
+
+ public void Shrink(int freq)
+ {
+ var i = 0;
+ while (i < featureset_dict_.Count)
+ {
+ if (featureset_dict_.ValueList[i].Value < freq)
+ {
+ //If the feature's frequency is less than specific frequency, drop the feature.
+ featureset_dict_.RemoveAt(i);
+ }
+ else
+ {
+ i++;
+ }
+ }
+ }
+
+ public void GenerateLexicalIdList(out IList keyList, out IList valList)
+ {
+ keyList = featureset_dict_.KeyList;
+ var fixArrayValue = new int[Size];
+ valList = fixArrayValue;
+
+ Parallel.For(0, featureset_dict_.ValueList.Count, parallelOption, i =>
+ {
+ fixArrayValue[i] = (int)featureset_dict_.ValueList[i].Key;
+ });
+
+ }
+
+ public long RegenerateFeatureId(CRFLite.Utils.BTreeDictionary old2new, long ysize)
+ {
+ long new_maxid = 0;
+ //Regenerate new feature id and create feature ids mapping
+ foreach (var it in featureset_dict_)
+ {
+ var strFeature = it.Key;
+ //Regenerate new feature id
+ old2new.Add(it.Value.Key, new_maxid);
+ it.Value.Key = new_maxid;
+
+ var addValue = (strFeature[0] == 'U' ? ysize : ysize * ysize);
+ new_maxid += addValue;
+ }
+
+ return new_maxid;
+ }
+
+ //Get feature id from feature set by feature string
+ //If feature string is not existed in the set, generate a new id and return it
+ private long GetId(string key)
+ {
+ FeatureIdPair pair;
+ if (featureset_dict_.TryGetValue(key, out pair) == true)
+ {
+ return pair.Key;
+ }
+
+ return BaseUtils.ERROR_INVALIDATED_FEATURE;
+ }
+
+ public long GetOrAddId(string key)
+ {
+ FeatureIdPair pair;
+ if (featureset_dict_.TryGetValue(key, out pair) == true && pair != null)
+ {
+ //Find its feature id
+ System.Threading.Interlocked.Increment(ref pair.Value);
+ }
+ else
+ {
+ lock (thisLock)
+ {
+ if (featureset_dict_.TryGetValue(key, out pair) == true)
+ {
+ System.Threading.Interlocked.Increment(ref pair.Value);
+ }
+ else
+ {
+ var oldValue = Interlocked.Increment(ref maxid_) - 1;
+ pair = new FeatureIdPair(oldValue, 1);
+ featureset_dict_.Add(key, pair);
+ }
+ }
+ }
+ return pair.Key;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/EncoderOptions.cs b/BotSharp.MachineLearning/CRFLite/Encoder/EncoderOptions.cs
new file mode 100644
index 00000000..9fa61c70
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/EncoderOptions.cs
@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public class EncoderOptions
+ {
+ ///
+ /// Maximum iteration
+ ///
+ public int MaxIteration { get; set; }
+
+ ///
+ /// Minimum feature frequency, if one feature's frequency is less than this value, the feature will be dropped.
+ ///
+ public int MinFeatureFreq = 2;
+
+ ///
+ /// Minimum diff value, when diff less than the value consecutive 3 times, the process will be ended.
+ ///
+ public double MinDifference;
+
+ ///
+ /// The maximum slot usage rate threshold when building feature set.
+ ///
+ public double SlotUsageRateThreshold { get; set; }
+
+ ///
+ /// The amount of threads used to train model.
+ ///
+ public int ThreadsNum { get; set; }
+
+ ///
+ /// Regularization type
+ ///
+ public CRFEncoder.REG_TYPE RegType { get; set; }
+
+ ///
+ /// Template file name
+ ///
+ [Required]
+ public string TemplateFileName { get; set; }
+
+ ///
+ /// Training corpus file name
+ ///
+ [Required]
+ public string TrainingCorpusFileName { get; set; }
+
+ ///
+ /// Encoded model file name
+ ///
+ [Required]
+ public string ModelFileName { get; set; }
+
+ ///
+ /// The model file name for re-training
+ ///
+ public string RetrainModelFileName { get; set; }
+
+ ///
+ /// Debug level
+ ///
+ public int DebugLevel { get; set; }
+
+ ///
+ ///
+ ///
+ public uint HugeLexMemLoad = 0;
+
+ ///
+ /// cost factor, too big or small value may lead encoded model over tune or under tune
+ ///
+ public double CostFactor { get; set; }
+
+ ///
+ /// If we build vector quantization model for feature weights
+ ///
+ public bool BVQ { get; set; }
+
+ public EncoderOptions()
+ {
+ MaxIteration = 100;
+ MinFeatureFreq = 2;
+ MinDifference = 0.0001;
+ SlotUsageRateThreshold = 0.95;
+ ThreadsNum = 1;
+ RegType = CRFEncoder.REG_TYPE.L2;
+ CostFactor = 1.0;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/EncoderTagger.cs b/BotSharp.MachineLearning/CRFLite/Encoder/EncoderTagger.cs
new file mode 100644
index 00000000..b43f74af
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/EncoderTagger.cs
@@ -0,0 +1,198 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Security.Cryptography;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public class EncoderTagger : Tagger
+ {
+ public ModelWriter feature_index_;
+ public short[] answer_;
+
+ public int eval(int[,] merr)
+ {
+ var err = 0;
+ for (var i = 0; i < word_num; ++i)
+ {
+ if (answer_[i] != result_[i])
+ {
+ ++err;
+ merr[answer_[i], result_[i]]++;
+ }
+ }
+ return err;
+ }
+
+ public EncoderTagger(ModelWriter modelWriter)
+ {
+ feature_index_ = modelWriter;
+ ysize_ = (short)feature_index_.ysize();
+ }
+
+ public bool GenerateFeature(List> recordList)
+ {
+ word_num = (short)recordList.Count;
+ if (word_num == 0)
+ {
+ return false;
+ }
+
+ //Try to find each record's answer tag
+ var x_num = 0;
+ var xsize = (int)feature_index_.xsize_;
+ answer_ = new short[word_num];
+ for (int index = 0; index < recordList.Count; index++)
+ {
+ var record = recordList[index];
+//get result tag's index and fill answer
+ for (short k = 0; k < ysize_; ++k)
+ {
+ if (feature_index_.y(k) == record[xsize])
+ {
+ answer_[x_num] = k;
+ break;
+ }
+ }
+ x_num++;
+ }
+
+ //Build record feature set
+ x_ = recordList;
+ Z_ = 0.0;
+ feature_cache_ = new List();
+ feature_index_.BuildFeatures(this);
+ x_ = null;
+
+ return true;
+ }
+
+ private void LockFreeAdd(double[] expected, long exp_offset, double addValue)
+ {
+ double initialValue;
+ double newValue;
+ do
+ {
+ initialValue = expected[exp_offset]; // read current value
+ newValue = initialValue + addValue; //calculate new value
+ }
+ while (initialValue != Interlocked.CompareExchange(ref expected[exp_offset], newValue, initialValue));
+ }
+
+ private void calcExpectation(int x, int y, double[] expected)
+ {
+ var n = node_[x, y];
+ var c = Math.Exp(n.alpha + n.beta - n.cost - Z_);
+ var offset = y + 1; //since expected array is based on 1
+ for (int index = 0; index < feature_cache_[n.fid].Length; index++)
+ {
+ var item = feature_cache_[n.fid][index];
+ LockFreeAdd(expected, item + offset, c);
+ }
+
+ for (int index = 0; index < n.lpathList.Count; index++)
+ {
+ var p = n.lpathList[index];
+ c = Math.Exp(p.lnode.alpha + p.cost + p.rnode.beta - Z_);
+ offset = p.lnode.y * ysize_ + p.rnode.y + 1; //since expected array is based on 1
+ for (int i = 0; i < feature_cache_[p.fid].Length; i++)
+ {
+ var item = feature_cache_[p.fid][i];
+ LockFreeAdd(expected, item + offset, c);
+ }
+ }
+ }
+
+ public double gradient(double[] expected)
+ {
+ buildLattice();
+ forwardbackward();
+ var s = 0.0;
+
+ for (var i = 0; i < word_num; ++i)
+ {
+ for (var j = 0; j < ysize_; ++j)
+ {
+ calcExpectation(i, j, expected);
+ }
+ }
+
+ for (var i = 0; i < word_num; ++i)
+ {
+ var answer_val = answer_[i];
+ var answer_Node = node_[i, answer_val];
+ var offset = answer_val + 1; //since expected array is based on 1
+ for (int index = 0; index < feature_cache_[answer_Node.fid].Length; index++)
+ {
+ var fid = feature_cache_[answer_Node.fid][index];
+ LockFreeAdd(expected, fid + offset, -1.0f);
+ }
+ s += answer_Node.cost; // UNIGRAM cost
+
+
+ for (int index = 0; index < answer_Node.lpathList.Count; index++)
+ {
+ var lpath = answer_Node.lpathList[index];
+ if (lpath.lnode.y == answer_[lpath.lnode.x])
+ {
+ offset = lpath.lnode.y * ysize_ + lpath.rnode.y + 1;
+ for (int index1 = 0; index1 < feature_cache_[lpath.fid].Length; index1++)
+ {
+ var fid = feature_cache_[lpath.fid][index1];
+ LockFreeAdd(expected, fid + offset, -1.0f);
+ }
+
+ s += lpath.cost; // BIGRAM COST
+ break;
+ }
+ }
+ }
+
+ viterbi(); // call for eval()
+ return Z_ - s;
+ }
+
+ public void Init(short[] result, Node[,] node)
+ {
+ result_ = result;
+ node_ = node;
+ }
+
+
+
+
+ public void buildLattice()
+ {
+ RebuildFeatures();
+ for (var i = 0; i < word_num; ++i)
+ {
+ for (var j = 0; j < ysize_; ++j)
+ {
+ var node_i_j = node_[i, j];
+ node_i_j.cost = calcCost(node_i_j.fid, j);
+ for (int index = 0; index < node_i_j.lpathList.Count; index++)
+ {
+ var p = node_i_j.lpathList[index];
+ var offset = p.lnode.y * ysize_ + p.rnode.y;
+ p.cost = calcCost(p.fid, offset);
+ }
+ }
+ }
+ }
+
+ public double calcCost(int featureListIdx, int offset)
+ {
+ double c = 0.0f;
+ offset++; //since alpha_ array is based on 1
+ for (int index = 0; index < feature_cache_[featureListIdx].Length; index++)
+ {
+ var fid = feature_cache_[featureListIdx][index];
+ c += feature_index_.alpha_[fid + offset];
+ }
+ return feature_index_.cost_factor_ * c;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/FeatureIdPair.cs b/BotSharp.MachineLearning/CRFLite/Encoder/FeatureIdPair.cs
new file mode 100644
index 00000000..b266f098
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/FeatureIdPair.cs
@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public sealed class FeatureIdPair
+ {
+ public long Key;
+ public int Value;
+
+ public FeatureIdPair(long key, int value)
+ {
+ Key = key;
+ Value = value;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/FeatureItem.cs b/BotSharp.MachineLearning/CRFLite/Encoder/FeatureItem.cs
new file mode 100644
index 00000000..6f665d13
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/FeatureItem.cs
@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public sealed class FeatureItem : IComparable
+ {
+ public string strFeature;
+ public FeatureIdPair feaIdPair;
+
+ public FeatureItem(string s, FeatureIdPair item)
+ {
+ strFeature = s;
+ feaIdPair = item;
+ }
+
+ public int CompareTo(FeatureItem fi)
+ {
+ return StringComparer.Ordinal.Compare(strFeature, fi.strFeature);
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/HugeFeatureLexicalDict.cs b/BotSharp.MachineLearning/CRFLite/Encoder/HugeFeatureLexicalDict.cs
new file mode 100644
index 00000000..700f44ec
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/HugeFeatureLexicalDict.cs
@@ -0,0 +1,305 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
+ public class MEMORYSTATUSEX
+ {
+ public uint dwLength;
+ public uint dwMemoryLoad;
+ public ulong ullTotalPhys;
+ public ulong ullAvailPhys;
+ public ulong ullTotalPageFile;
+ public ulong ullAvailPageFile;
+ public ulong ullTotalVirtual;
+ public ulong ullAvailVirtual;
+ public ulong ullAvailExtendedVirtual;
+ public MEMORYSTATUSEX()
+ {
+ this.dwLength = (uint)Marshal.SizeOf(typeof(MEMORYSTATUSEX));
+ }
+ }
+
+ public sealed class FeatureFreq : IComparable
+ {
+ public string strFeature;
+ public long value;
+
+ public int CompareTo(FeatureFreq fi)
+ {
+ return StringComparer.Ordinal.Compare(strFeature, fi.strFeature);
+ }
+ }
+
+ public sealed class HugeFeatureLexicalDict : IFeatureLexicalDict
+ {
+ CRFLite.Utils.VarBigArray arrayFeatureFreq;
+ long arrayFeatureFreqSize;
+ uint SHRINK_AVALI_MEM_LOAD;
+ CRFLite.Utils.MD5 md5;
+ ParallelOptions parallelOption;
+
+ [return: MarshalAs(UnmanagedType.Bool)]
+ [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
+ static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer);
+
+ public HugeFeatureLexicalDict(int thread_num, uint shrinkMemLoad)
+ {
+ SHRINK_AVALI_MEM_LOAD = shrinkMemLoad;
+ arrayFeatureFreq = new CRFLite.Utils.VarBigArray(1024 * 1024);
+ arrayFeatureFreqSize = 0;
+ md5 = new CRFLite.Utils.MD5();
+ parallelOption = new ParallelOptions();
+ parallelOption.MaxDegreeOfParallelism = thread_num;
+ }
+
+ public void Clear()
+ {
+ arrayFeatureFreq.Clear();
+ arrayFeatureFreq = null;
+ }
+
+
+ public CRFLite.Utils.VarBigArray featureFreq
+ {
+ get
+ {
+ return arrayFeatureFreq;
+ }
+ }
+
+ public long Size
+ {
+ get
+ {
+ return arrayFeatureFreqSize;
+ }
+ }
+
+ private long ParallelMerge(long startIndex, long endIndex, int freq)
+ {
+ var sizePerThread = (endIndex - startIndex + 1) / parallelOption.MaxDegreeOfParallelism;
+ //Fistly, merge items in each block by parallel
+ Parallel.For(0, parallelOption.MaxDegreeOfParallelism, parallelOption, i =>
+ {
+ Merge(startIndex + i * sizePerThread, startIndex + (i + 1) * sizePerThread - 1, 0);
+ });
+
+ //Secondly, merge all items
+ return Merge(startIndex, endIndex, freq);
+ }
+
+ private void ForceCollectMemory()
+ {
+ GC.Collect();
+ GC.WaitForPendingFinalizers();
+ GC.Collect();
+ }
+
+ //Merge same items in sorted list
+ private long Merge(long startIndex, long endIndex, int freq)
+ {
+ var newEndIndex = startIndex;
+
+ //Try to find first not null item
+ while ((arrayFeatureFreq[startIndex] == null) &&
+ startIndex <= endIndex)
+ {
+ startIndex++;
+ }
+ arrayFeatureFreq[newEndIndex] = arrayFeatureFreq[startIndex];
+ for (var i = startIndex + 1; i <= endIndex; i++)
+ {
+ if (arrayFeatureFreq[i] == null)
+ {
+ continue;
+ }
+
+ if (arrayFeatureFreq[newEndIndex].strFeature == arrayFeatureFreq[i].strFeature)
+ {
+ //two same items, sum their value up
+ arrayFeatureFreq[newEndIndex].value += arrayFeatureFreq[i].value;
+ arrayFeatureFreq[i] = null;
+ }
+ else
+ {
+ //two different items
+ if (arrayFeatureFreq[newEndIndex].value >= freq)
+ {
+ newEndIndex++;
+ }
+
+ arrayFeatureFreq[newEndIndex] = arrayFeatureFreq[i];
+ if (newEndIndex < i)
+ {
+ arrayFeatureFreq[i] = null;
+ }
+ }
+ }
+
+ return newEndIndex;
+ }
+
+ //Generate feature string and its id list
+ public void GenerateLexicalIdList(out IList keyList, out IList valList)
+ {
+ var fixArrayKey = new CRFLite.Utils.FixedBigArray(Size, 0);
+ keyList = fixArrayKey;
+
+ var fixArrayValue = new CRFLite.Utils.FixedBigArray(Size, 0);
+ valList = fixArrayValue;
+ Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
+ {
+ fixArrayKey[i] = arrayFeatureFreq[i].strFeature;
+ fixArrayValue[i] = (int)(arrayFeatureFreq[i].value);
+ });
+ }
+
+ Object thisLock = new object();
+ //Generate feature id by NGram rules
+ public long RegenerateFeatureId(CRFLite.Utils.BTreeDictionary old2new, long ysize)
+ {
+ long maxid_ = 0;
+ Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
+ {
+ //Generate new feature id
+ var addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
+ var oldValue = maxid_;
+ while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
+ {
+ oldValue = maxid_;
+ }
+
+ //Create existed and new feature ids mapping
+ lock (thisLock)
+ {
+ old2new.Add(
+ GetId(arrayFeatureFreq[i].strFeature),
+ oldValue);
+ }
+
+ arrayFeatureFreq[i].value = oldValue;
+ });
+ return maxid_;
+ }
+
+ //Shrink entire list
+ public void Shrink(int freq)
+ {
+ var newEndIndex = Shrink(0, arrayFeatureFreqSize - 1, freq);
+ arrayFeatureFreqSize = newEndIndex + 1;
+ }
+
+ //Shrink item list
+ private long Shrink(long startIndex, long endIndex, int freq)
+ {
+ Console.Write("Sorting...");
+ arrayFeatureFreq.Sort(startIndex, endIndex - startIndex + 1, parallelOption.MaxDegreeOfParallelism);
+ Console.Write("Merging...");
+
+ var newEndIndex = ParallelMerge(startIndex, endIndex, freq);
+ sortedEndIndex = newEndIndex;
+
+ Console.WriteLine("Done!");
+ ForceCollectMemory();
+
+ return newEndIndex;
+ }
+
+ //Get feature string id
+ private long GetId(string strFeature)
+ {
+ var rawbytes = Encoding.UTF8.GetBytes(strFeature);
+
+ lock (thisLock)
+ {
+ return md5.Compute64BitHash(rawbytes);
+ }
+ }
+
+ private long sortedEndIndex = 0;
+ private int ShrinkingLock = 0;
+ private int AddLock = 0;
+ //Add the feature string into list and get feature string id
+ public long GetOrAddId(string strFeature)
+ {
+ while (ShrinkingLock == 1) { Thread.Sleep(5000); }
+
+ //add item-adding lock
+ Interlocked.Increment(ref AddLock);
+
+ var newFFItem = new FeatureFreq();
+ newFFItem.strFeature = strFeature;
+ newFFItem.value = 1;
+ if (sortedEndIndex > 0)
+ {
+ var ff = arrayFeatureFreq.BinarySearch(0, sortedEndIndex, newFFItem);
+ if (ff != null)
+ {
+ Interlocked.Increment(ref ff.value);
+ //free item-adding lock
+ Interlocked.Decrement(ref AddLock);
+ return GetId(strFeature);
+ }
+ }
+
+ var oldValue = Interlocked.Increment(ref arrayFeatureFreqSize) - 1;
+ arrayFeatureFreq[oldValue] = newFFItem;
+
+ //free item-adding lock
+ Interlocked.Decrement(ref AddLock);
+
+ //Check whether shrink process should be started
+ uint memoryLoad = 0;
+ if (oldValue % 10000000 == 0)
+ {
+ var msex = new MEMORYSTATUSEX();
+ GlobalMemoryStatusEx(msex);
+ memoryLoad = msex.dwMemoryLoad;
+ }
+
+ if (memoryLoad >= SHRINK_AVALI_MEM_LOAD)
+ {
+ if (Interlocked.CompareExchange(ref ShrinkingLock, 1, 0) == 0)
+ {
+ //Double check whether shrink should be started
+ var msex = new MEMORYSTATUSEX();
+ GlobalMemoryStatusEx(msex);
+ if (msex.dwMemoryLoad >= SHRINK_AVALI_MEM_LOAD)
+ {
+ while (AddLock != 0) { Thread.Sleep(1000); }
+
+ var startDT = DateTime.Now;
+ Console.WriteLine("Begin to shrink [Feature Size: {0}]...", arrayFeatureFreqSize);
+ var newArrayFeatureFreqSize = Shrink(0, arrayFeatureFreqSize - 1, 0) + 1;
+
+ GlobalMemoryStatusEx(msex);
+ if (msex.dwMemoryLoad >= SHRINK_AVALI_MEM_LOAD - 1)
+ {
+ //Still have enough available memory, raise shrink threshold
+ SHRINK_AVALI_MEM_LOAD = msex.dwMemoryLoad + 1;
+ if (SHRINK_AVALI_MEM_LOAD >= 100)
+ {
+ //if use more than 100% memory, the performance will extremely reduce
+ SHRINK_AVALI_MEM_LOAD = 100;
+ }
+ }
+
+ arrayFeatureFreqSize = newArrayFeatureFreqSize;
+ var ts = DateTime.Now - startDT;
+ Console.WriteLine("Shrink has been done!");
+ Console.WriteLine("[Feature Size:{0}, TimeSpan:{1}, Next Shrink Rate:{2}%]", arrayFeatureFreqSize, ts, SHRINK_AVALI_MEM_LOAD);
+ }
+
+ Interlocked.Decrement(ref ShrinkingLock);
+ }
+ }
+ return GetId(strFeature);
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/IFeatureLexicalDict.cs b/BotSharp.MachineLearning/CRFLite/Encoder/IFeatureLexicalDict.cs
new file mode 100644
index 00000000..10efa8a3
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/IFeatureLexicalDict.cs
@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public interface IFeatureLexicalDict
+ {
+ void Shrink(int freq);
+ long GetOrAddId(string strFeature);
+ long RegenerateFeatureId(CRFLite.Utils.BTreeDictionary old2new, long ysize);
+ void GenerateLexicalIdList(out IList fea, out IList val);
+ void Clear();
+
+ long Size
+ {
+ get;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/LBFGS.cs b/BotSharp.MachineLearning/CRFLite/Encoder/LBFGS.cs
new file mode 100644
index 00000000..0c6f8995
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/LBFGS.cs
@@ -0,0 +1,349 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public class LBFGS
+ {
+ double [] diag;
+ CRFLite.Utils.FixedBigArray w;
+ Mcsrch mcsrch_;
+ long nfev, point, npt, iter, info, ispt, iypt;
+ int iflag_;
+ double stp;
+ public int zeroone;
+ public int err;
+ public double obj;
+
+ public double[] expected;
+ public double[] v;
+ public double[] xi;
+
+ private ParallelOptions parallelOption;
+
+ public LBFGS(int thread_num)
+ {
+ iflag_ = 0; nfev = 0;
+ point = 0; npt = 0; iter = 0; info = 0;
+ ispt = 0; iypt = 0;
+ stp = 0.0;
+ mcsrch_ = new Mcsrch(thread_num);
+
+ parallelOption = new ParallelOptions();
+ parallelOption.MaxDegreeOfParallelism = thread_num;
+ }
+
+ private double ddot_(long size, CRFLite.Utils.FixedBigArray dx, long dx_idx, CRFLite.Utils.FixedBigArray dy, long dy_idx)
+ {
+ double ret = 0.0f;
+ Parallel.For(0, size, parallelOption, () => 0, (i, loop, subtotal) =>
+ {
+ subtotal += dx[i + dx_idx] * dy[i + dy_idx];
+ return subtotal;
+ },
+ (subtotal) => // lock free accumulator
+ {
+ double initialValue;
+ double newValue;
+ do
+ {
+ initialValue = ret; // read current value
+ newValue = initialValue + subtotal; //calculate new value
+ }
+ while (initialValue != Interlocked.CompareExchange(ref ret, newValue, initialValue));
+ });
+ return ret;
+ }
+
+
+ private double ddot_(long size, double[] dx, long dx_idx, double[] dy, long dy_idx)
+ {
+ double ret = 0.0f;
+ Parallel.For(0, size, parallelOption, () => 0, (i, loop, subtotal) =>
+ {
+ subtotal += dx[i + dx_idx] * dy[i + dy_idx];
+ return subtotal;
+ },
+ (subtotal) => // lock free accumulator
+ {
+ double initialValue;
+ double newValue;
+ do
+ {
+ initialValue = ret; // read current value
+ newValue = initialValue + subtotal; //calculate new value
+ }
+ while (initialValue != Interlocked.CompareExchange(ref ret, newValue, initialValue));
+ });
+ return ret;
+ }
+ void pseudo_gradient(double[] x, double C)
+ {
+ var size = expected.LongLength - 1;
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ if (x[i] == 0)
+ {
+ if (expected[i] + C < 0)
+ {
+ v[i] = (expected[i] + C);
+ }
+ else if (expected[i] - C > 0)
+ {
+ v[i] = (expected[i] - C);
+ }
+ else
+ {
+ v[i] = 0;
+ }
+ }
+ else
+ {
+ v[i] = (expected[i] + C * sigma(x[i]));
+ }
+ });
+ }
+ double sigma(double x)
+ {
+ if (x > 0) return 1.0;
+ else if (x < 0) return -1.0;
+ return 0.0;
+ }
+
+ public int optimize(double[] x, double C, bool orthant)
+ {
+ const long msize = 5;
+ var size = x.LongLength - 1;
+ if (w == null || w.LongLength == 0)
+ {
+ iflag_ = 0;
+ w = new CRFLite.Utils.FixedBigArray(size * (2 * msize + 1) + 2 * msize, 1);
+ diag = new double[size + 1];
+ if (orthant == true)
+ {
+ xi = new double[size + 1];
+ v = new double[size + 1];
+ }
+ }
+
+ if (orthant == true)
+ {
+ pseudo_gradient(x, C);
+ }
+ else
+ {
+ v = expected;
+ }
+
+ lbfgs_optimize(msize, x, orthant, C);
+ if (iflag_ < 0)
+ {
+ Console.WriteLine("routine stops with unexpected error");
+ return -1;
+ }
+
+ return iflag_;
+ }
+
+ void lbfgs_optimize(long msize, double[] x, bool orthant, double C)
+ {
+ var size = x.LongLength - 1;
+ var yy = 0.0;
+ var ys = 0.0;
+ long bound = 0;
+ long cp = 0;
+ var bExit = false;
+
+ // initialization
+ if (iflag_ == 0)
+ {
+ point = 0;
+ ispt = size + (msize << 1);
+ iypt = ispt + size * msize;
+
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ diag[i] = 1.0f;
+ w[ispt + i] = -v[i];
+ w[i] = expected[i];
+ });
+
+ if (orthant == true)
+ {
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ xi[i] = (x[i] != 0 ? sigma(x[i]) : sigma(-v[i]));
+ });
+ }
+
+ //第一次试探步长
+ stp = 1.0f / Math.Sqrt(ddot_(size, v, 1, v, 1));
+
+ ++iter;
+ info = 0;
+ nfev = 0;
+ }
+
+ // MAIN ITERATION LOOP
+ bExit = LineSearchAndUpdateStepGradient(msize, x, orthant);
+ while (bExit == false)
+ {
+ ++iter;
+ info = 0;
+
+ if (orthant == true)
+ {
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ xi[i] = (x[i] != 0 ? sigma(x[i]) : sigma(-v[i]));
+ });
+ }
+
+ if (iter > size)
+ {
+ bound = size;
+ }
+
+ // COMPUTE -H*G USING THE FORMULA GIVEN IN: Nocedal, J. 1980,
+ // "Updating quasi-Newton matrices with limited storage",
+ // Mathematics of Computation, Vol.24, No.151, pp. 773-782.
+ ys = ddot_(size, w, iypt + npt + 1, w, ispt + npt + 1);
+ yy = ddot_(size, w, iypt + npt + 1, w, iypt + npt + 1);
+
+ var r_ys_yy = ys / yy;
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ diag[i] = r_ys_yy;
+ w[i] = -v[i];
+ });
+
+ cp = point;
+ if (point == 0)
+ {
+ cp = msize;
+ }
+
+ w[size + cp] = (1.0 / ys);
+ //回退次数
+ bound = Math.Min(iter - 1, msize);
+ cp = point;
+ for (var i = 1; i <= bound; ++i)
+ {
+ --cp;
+ if (cp == -1) cp = msize - 1;
+ var sq = ddot_(size, w, ispt + cp * size + 1, w, 1);
+ var inmc = size + msize + cp + 1;
+ var iycn = iypt + cp * size;
+ w[inmc] = (w[size + cp + 1] * sq);
+ var d = -w[inmc];
+
+ Parallel.For(1, size + 1, parallelOption, j =>
+ {
+ w[j] = (w[j] + d * w[iycn + j]);
+ });
+ }
+
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ w[i] = (diag[i] * w[i]);
+ });
+
+ for (var i = 1; i <= bound; ++i)
+ {
+ var yr = ddot_(size, w, iypt + cp * size + 1, w, 1);
+ var beta = w[size + cp + 1] * yr;
+ var inmc = size + msize + cp + 1;
+ beta = w[inmc] - beta;
+ var iscn = ispt + cp * size;
+
+ Parallel.For(1, size + 1, parallelOption, j =>
+ {
+ w[j] = (w[j] + beta * w[iscn + j]);
+ });
+
+ ++cp;
+ if (cp == msize)
+ {
+ cp = 0;
+ }
+ }
+
+ if (orthant == true)
+ {
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ w[i] = (sigma(w[i]) == sigma(-v[i]) ? w[i] : 0);
+ });
+ }
+
+
+ // STORE THE NEW SEARCH DIRECTION
+ var offset = ispt + point * size;
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ w[offset + i] = w[i];
+ w[i] = expected[i];
+ });
+
+ stp = 1.0f;
+ nfev = 0;
+ bExit = LineSearchAndUpdateStepGradient(msize, x, orthant);
+ }
+ }
+
+ private bool LineSearchAndUpdateStepGradient(long msize, double[] x, bool orthant)
+ {
+ var size = x.LongLength - 1;
+ var bExit = false;
+ mcsrch_.mcsrch(x, obj, v, w, ispt + point * size,
+ ref stp, ref info, ref nfev, diag);
+ if (info == -1)
+ {
+ if (orthant == true)
+ {
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ x[i] = (sigma(x[i]) == sigma(xi[i]) ? x[i] : 0);
+ });
+ }
+
+
+ iflag_ = 1; // next value
+ bExit = true;
+ }
+ else if (info != 1)
+ {
+ //MCSRCH error, please see error code in info
+ iflag_ = -1;
+ bExit = true;
+ }
+ else
+ {
+ // COMPUTE THE NEW STEP AND GRADIENT CHANGE
+ npt = point * size;
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ w[ispt + npt + i] = (stp * w[ispt + npt + i]);
+ w[iypt + npt + i] = expected[i] - w[i];
+ });
+
+ ++point;
+ if (point == msize)
+ {
+ point = 0;
+ }
+
+ var gnorm = Math.Sqrt(ddot_(size, v, 1, v, 1));
+ var xnorm = Math.Max(1.0, Math.Sqrt(ddot_(size, x, 1, x, 1)));
+ if (gnorm / xnorm <= BaseUtils.eps)
+ {
+ iflag_ = 0; // OK terminated
+ bExit = true;
+ }
+ }
+
+ return bExit;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/Mcsrch.cs b/BotSharp.MachineLearning/CRFLite/Encoder/Mcsrch.cs
new file mode 100644
index 00000000..886f3c16
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/Mcsrch.cs
@@ -0,0 +1,461 @@
+using BotSharp.MachineLearning.CRFLite.Utils;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ class Mcsrch
+ {
+ private int infoc;
+ private bool stage1, brackt;
+ private double dginit;
+ private double width, width1;
+ private double fx, dgx, fy, dgy;
+ private double finit;
+ private double dgtest;
+ private double stx, sty;
+ private double stmin, stmax;
+
+ private ParallelOptions parallelOption;
+
+ public Mcsrch(int thread_num)
+ {
+ infoc = 0;
+ stage1 = false;
+ brackt = false;
+ finit = 0.0;
+ dginit = 0.0;
+ dgtest = 0.0;
+ width = 0.0;
+ width1 = 0.0;
+ stx = 0.0;
+ fx = 0.0;
+ dgx = 0.0;
+ sty = 0.0;
+ fy = 0.0;
+ dgy = 0.0;
+ stmin = 0.0;
+ stmax = 0.0;
+
+ parallelOption = new ParallelOptions();
+ parallelOption.MaxDegreeOfParallelism = thread_num;
+ }
+
+
+
+ void mcstep(ref double stx, ref double fx, ref double dx,
+ ref double sty, ref double fy, ref double dy,
+ ref double stp, double fp, double dp,
+ ref bool brackt,
+ double stpmin, double stpmax,
+ ref int info)
+ {
+ var bound = true;
+ double p, q, d3, r, stpq, stpc, stpf;
+ double gamma;
+ double s;
+ double d1, d2;
+ double theta;
+ info = 0;
+
+ if (brackt == true && ((stp <= Math.Min(stx, sty) || stp >= Math.Max(stx, sty)) ||
+ dx * (stp - stx) >= 0.0 || stpmax < stpmin))
+ {
+ return;
+ }
+
+ var sgnd = dp * (dx / Math.Abs(dx));
+ if (fp > fx)
+ {
+ info = 1;
+ bound = true;
+ theta = (fx - fp) * 3 / (stp - stx) + dx + dp;
+ d1 = Math.Abs(theta);
+ d2 = Math.Abs(dx);
+ d1 = Math.Max(d1, d2);
+ d2 = Math.Abs(dp);
+ s = Math.Max(d1, d2);
+ d1 = theta / s;
+ gamma = s * Math.Sqrt(d1 * d1 - dx / s * (dp / s));
+ if (stp < stx)
+ {
+ gamma = -gamma;
+ }
+ p = gamma - dx + theta;
+ q = gamma - dx + gamma + dp;
+ r = p / q;
+ stpc = stx + r * (stp - stx);
+ stpq = stx + dx / ((fx - fp) /
+ (stp - stx) + dx) / 2 * (stp - stx);
+ d1 = stpc - stx;
+ d2 = stpq - stx;
+ if (Math.Abs(d1) < Math.Abs(d2))
+ {
+ stpf = stpc;
+ }
+ else
+ {
+ stpf = stpc + (stpq - stpc) / 2;
+ }
+ brackt = true;
+ }
+ else if (sgnd < 0.0)
+ {
+ info = 2;
+ bound = false;
+ theta = (fx - fp) * 3 / (stp - stx) + dx + dp;
+ d1 = Math.Abs(theta);
+ d2 = Math.Abs(dx);
+ d1 = Math.Max(d1, d2);
+ d2 = Math.Abs(dp);
+ s = Math.Max(d1, d2);
+ d1 = theta / s;
+ gamma = s * Math.Sqrt(d1 * d1 - dx / s * (dp / s));
+ if (stp > stx)
+ {
+ gamma = -gamma;
+ }
+ p = gamma - dp + theta;
+ q = gamma - dp + gamma + dx;
+ r = p / q;
+ stpc = stp + r * (stx - stp);
+ stpq = stp + dp / (dp - dx) * (stx - stp);
+
+ d1 = stpc - stp;
+ d2 = stpq - stp;
+ if (Math.Abs(d1) > Math.Abs(d2))
+ {
+ stpf = stpc;
+ }
+ else
+ {
+ stpf = stpq;
+ }
+ brackt = true;
+ }
+ else if (Math.Abs(dp) < Math.Abs(dx))
+ {
+ info = 3;
+ bound = true;
+ theta = (fx - fp) * 3 / (stp - stx) + dx + dp;
+ d1 = Math.Abs(theta);
+ d2 = Math.Abs(dx);
+ d1 = Math.Max(d1, d2);
+ d2 = Math.Abs(dp);
+ s = Math.Max(d1, d2);
+ d3 = theta / s;
+ d1 = 0.0f;
+ d2 = d3 * d3 - dx / s * (dp / s);
+ gamma = s * Math.Sqrt((Math.Max(d1, d2)));
+ if (stp > stx)
+ {
+ gamma = -gamma;
+ }
+ p = gamma - dp + theta;
+ q = gamma + (dx - dp) + gamma;
+ r = p / q;
+ if (r < 0.0 && gamma != 0.0)
+ {
+ stpc = stp + r * (stx - stp);
+ }
+ else if (stp > stx)
+ {
+ stpc = stpmax;
+ }
+ else
+ {
+ stpc = stpmin;
+ }
+ stpq = stp + dp / (dp - dx) * (stx - stp);
+ if (brackt == true)
+ {
+ d1 = stp - stpc;
+ d2 = stp - stpq;
+ if (Math.Abs(d1) < Math.Abs(d2))
+ {
+ stpf = stpc;
+ }
+ else
+ {
+ stpf = stpq;
+ }
+ }
+ else
+ {
+ d1 = stp - stpc;
+ d2 = stp - stpq;
+ if (Math.Abs(d1) > Math.Abs(d2))
+ {
+ stpf = stpc;
+ }
+ else
+ {
+ stpf = stpq;
+ }
+ }
+ }
+ else
+ {
+ info = 4;
+ bound = false;
+ if (brackt == true)
+ {
+ theta = (fp - fy) * 3 / (sty - stp) + dy + dp;
+ d1 = Math.Abs(theta);
+ d2 = Math.Abs(dy);
+ d1 = Math.Max(d1, d2);
+ d2 = Math.Abs(dp);
+ s = Math.Max(d1, d2);
+ d1 = theta / s;
+ gamma = s * Math.Sqrt(d1 * d1 - dy / s * (dp / s));
+ if (stp > sty)
+ {
+ gamma = -gamma;
+ }
+ p = gamma - dp + theta;
+ q = gamma - dp + gamma + dy;
+ r = p / q;
+ stpc = stp + r * (sty - stp);
+ stpf = stpc;
+ }
+ else if (stp > stx)
+ {
+ stpf = stpmax;
+ }
+ else
+ {
+ stpf = stpmin;
+ }
+ }
+
+ if (fp > fx)
+ {
+ sty = stp;
+ fy = fp;
+ dy = dp;
+ }
+ else
+ {
+ if (sgnd < 0.0)
+ {
+ sty = stx;
+ fy = fx;
+ dy = dx;
+ }
+ stx = stp;
+ fx = fp;
+ dx = dp;
+ }
+
+ stpf = Math.Min(stpmax, stpf);
+ stpf = Math.Max(stpmin, stpf);
+ stp = stpf;
+ if (brackt == true && bound)
+ {
+ if (sty > stx)
+ {
+ d1 = stx + (sty - stx) * 0.66;
+ stp = Math.Min(d1, stp);
+ }
+ else
+ {
+ d1 = stx + (sty - stx) * 0.66;
+ stp = Math.Max(d1, stp);
+ }
+ }
+
+ return;
+ }
+
+
+
+ const double lb3_1_gtol = 0.9;
+ const double xtol = 1e-16;
+ const double lb3_1_stpmin = 1e-20;
+ const double lb3_1_stpmax = 1e20;
+ const double ftol = 1e-4;
+ const double p5 = 0.5;
+ const double p66 = 0.66;
+ const double xtrapf = 4.0;
+ const int maxfev = 20;
+
+ private double ddot_(long size, double[] dx, long dx_idx, FixedBigArray dy, long dy_idx)
+ {
+ double ret = 0.0f;
+ Parallel.For(0, size, parallelOption, () => 0, (i, loop, subtotal) =>
+ {
+ subtotal += dx[i + dx_idx] * dy[i + dy_idx];
+ return subtotal;
+ },
+ (subtotal) => // lock free accumulator
+ {
+ double initialValue;
+ double newValue;
+ do
+ {
+ initialValue = ret; // read current value
+ newValue = initialValue + subtotal; //calculate new value
+ }
+ while (initialValue != Interlocked.CompareExchange(ref ret, newValue, initialValue));
+ });
+ return ret;
+ }
+
+ public void mcsrch(double[] x, double f, double[] g, FixedBigArray s, long s_idx,
+ ref double stp, ref long info, ref long nfev, double[] wa)
+ {
+ var size = x.LongLength - 1;
+ /* Parameter adjustments */
+ if (info == -1)
+ {
+ info = 0;
+ nfev++;
+
+ var dg = ddot_(size, g, 1, s, s_idx + 1);
+ var ftest1 = finit + stp * dgtest;
+
+ if (brackt && ((stp <= stmin || stp >= stmax) || infoc == 0))
+ {
+ info = 6;
+ Console.WriteLine("MCSRCH warning: Rounding errors prevent further progress.There may not be a step which satisfies the sufficient decrease and curvature conditions. Tolerances may be too small.");
+ Console.WriteLine("bracket: {0}, stp:{1}, stmin:{2}, stmax:{3}, infoc:{4}", brackt, stp, stmin, stmax, infoc);
+ }
+ if (stp == lb3_1_stpmax && f <= ftest1 && dg <= dgtest)
+ {
+ info = 5;
+ Console.WriteLine("MCSRCH warning: The step is too large.");
+ }
+ if (stp == lb3_1_stpmin && (f > ftest1 || dg >= dgtest))
+ {
+ info = 4;
+ Console.WriteLine("MCSRCH warning: The step is too small.");
+ Console.WriteLine("stp:{0}, lb3_1_stpmin:{1}, f:{2}, ftest1:{3}, dg:{4}, dgtest:{5}", stp, lb3_1_stpmin, f, ftest1, dg, dgtest);
+ }
+ if (nfev >= maxfev)
+ {
+ info = 3;
+ Console.WriteLine("MCSRCH warning: More than {0} function evaluations were required at the present iteration.", maxfev);
+ }
+ if (brackt && stmax - stmin <= xtol * stmax)
+ {
+ info = 2;
+ Console.WriteLine("MCSRCH warning: Relative width of the interval of uncertainty is at most xtol.");
+ }
+ if (f <= ftest1 && Math.Abs(dg) <= lb3_1_gtol * (-dginit))
+ {
+ info = 1;
+ }
+
+ if (info != 0)
+ {
+ return;
+ }
+
+ if (stage1 && f <= ftest1 && dg >= Math.Min(ftol, lb3_1_gtol) * dginit)
+ {
+ stage1 = false;
+ }
+
+ if (stage1 && f <= fx && f > ftest1)
+ {
+ var fm = f - stp * dgtest;
+ var fxm = fx - stx * dgtest;
+ var fym = fy - sty * dgtest;
+ var dgm = dg - dgtest;
+ var dgxm = dgx - dgtest;
+ var dgym = dgy - dgtest;
+ mcstep(ref stx, ref fxm, ref dgxm, ref sty, ref fym, ref dgym, ref stp, fm, dgm, ref brackt,
+ stmin, stmax, ref infoc);
+ fx = fxm + stx * dgtest;
+ fy = fym + sty * dgtest;
+ dgx = dgxm + dgtest;
+ dgy = dgym + dgtest;
+ }
+ else
+ {
+ mcstep(ref stx, ref fx, ref dgx, ref sty, ref fy, ref dgy, ref stp, f, dg, ref brackt,
+ stmin, stmax, ref infoc);
+ }
+
+ if (brackt)
+ {
+ var d1 = 0.0;
+ d1 = sty - stx;
+ if (Math.Abs(d1) >= p66 * width1)
+ {
+ stp = stx + p5 * (sty - stx);
+ }
+ width1 = width;
+ d1 = sty - stx;
+ width = Math.Abs(d1);
+ }
+ }
+ else
+ {
+ infoc = 1;
+ if (size <= 0 || stp <= 0.0)
+ {
+ return;
+ }
+
+ dginit = ddot_(size, g, 1, s, s_idx + 1);
+ if (dginit >= 0.0)
+ {
+ return;
+ }
+
+ brackt = false;
+ stage1 = true;
+ nfev = 0;
+ finit = f;
+ dgtest = ftol * dginit;
+ width = lb3_1_stpmax - lb3_1_stpmin;
+ width1 = width / p5;
+
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ wa[i] = x[i];
+ }
+ );
+
+ stx = 0.0;
+ fx = finit;
+ dgx = dginit;
+ sty = 0.0;
+ fy = finit;
+ dgy = dginit;
+ }
+
+ if (brackt)
+ {
+ stmin = Math.Min(stx, sty);
+ stmax = Math.Max(stx, sty);
+ }
+ else
+ {
+ stmin = stx;
+ stmax = stp + xtrapf * (stp - stx);
+ }
+
+ stp = Math.Max(stp, lb3_1_stpmin);
+ stp = Math.Min(stp, lb3_1_stpmax);
+
+ if ((brackt && ((stp <= stmin || stp >= stmax) ||
+ nfev >= maxfev - 1 || infoc == 0)) ||
+ (brackt && (stmax - stmin <= xtol * stmax)))
+ {
+ stp = stx;
+ }
+
+ var stp_t = stp;
+ Parallel.For(1, size + 1, parallelOption, i =>
+ {
+ x[i] = (wa[i] + stp_t * s[s_idx + i]);
+ });
+
+ info = -1;
+ }
+
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Encoder/ModelWriter.cs b/BotSharp.MachineLearning/CRFLite/Encoder/ModelWriter.cs
new file mode 100644
index 00000000..15479d0d
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Encoder/ModelWriter.cs
@@ -0,0 +1,405 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.MachineLearning.CRFLite.Decoder;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+ public class ModelWriter : BaseModel
+ {
+ private readonly string modelFileName;
+
+ private readonly Pool _buildersPool =
+ new Pool(p => new StringBuilder(100), b => b.Clear());
+
+
+ int thread_num_;
+ public IFeatureLexicalDict featureLexicalDict;
+ List>> trainCorpusList;
+ ParallelOptions parallelOption = new ParallelOptions();
+
+ public ModelWriter(int thread_num, double cost_factor,
+ uint hugeLexShrinkMemLoad, string modelFileName)
+ {
+ cost_factor_ = cost_factor;
+ maxid_ = 0;
+ thread_num_ = thread_num;
+ this.modelFileName = modelFileName;
+ parallelOption.MaxDegreeOfParallelism = thread_num;
+
+ if (hugeLexShrinkMemLoad > 0)
+ {
+ featureLexicalDict = new HugeFeatureLexicalDict(thread_num_, hugeLexShrinkMemLoad);
+ }
+ else
+ {
+ featureLexicalDict = new DefaultFeatureLexicalDict(thread_num_);
+ }
+ }
+
+ //Regenerate feature id and shrink features with lower frequency
+ public void Shrink(EncoderTagger[] xList, int freq)
+ {
+ var old2new = new CRFLite.Utils.BTreeDictionary();
+ featureLexicalDict.Shrink(freq);
+ maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
+ var feature_count = xList.Length;
+
+ //Update feature ids
+ Parallel.For(0, feature_count, parallelOption, i =>
+ {
+ for (var j = 0; j < xList[i].feature_cache_.Count; j++)
+ {
+ var newfs = new List();
+ long rstValue = 0;
+ for (int index = 0; index < xList[i].feature_cache_[j].Length; index++)
+ {
+ var v = xList[i].feature_cache_[j][index];
+ if (old2new.TryGetValue(v, out rstValue) == true)
+ {
+ newfs.Add(rstValue);
+ }
+ }
+ xList[i].feature_cache_[j] = newfs.ToArray();
+ }
+ });
+ }
+
+ // Load all records and generate features
+ public EncoderTagger[] ReadAllRecords()
+ {
+ var arrayEncoderTagger = new EncoderTagger[trainCorpusList.Count];
+ var arrayEncoderTaggerSize = 0;
+
+ //Generate each record features
+ Parallel.For(0, trainCorpusList.Count, parallelOption, i =>
+ {
+ var _x = new EncoderTagger(this);
+ if (_x.GenerateFeature(trainCorpusList[i]) == false)
+ {
+ }
+ else
+ {
+ var oldValue = Interlocked.Increment(ref arrayEncoderTaggerSize) - 1;
+ arrayEncoderTagger[oldValue] = _x;
+
+ if (oldValue % 10000 == 0)
+ {
+ //Show current progress on console
+ Console.Write("{0}...", oldValue);
+ }
+ }
+ });
+
+ trainCorpusList.Clear();
+ trainCorpusList = null;
+
+ Console.WriteLine();
+ return arrayEncoderTagger;
+ }
+
+ //Open and check training and template file
+ public bool Open(string strTemplateFileName, string strTrainCorpusFileName)
+ {
+ return OpenTemplateFile(strTemplateFileName) && OpenTrainCorpusFile(strTrainCorpusFileName);
+ }
+
+ //Build feature set into indexed data
+ public bool BuildFeatureSetIntoIndex(string filename, double max_slot_usage_rate_threshold, int debugLevel)
+ {
+ IList keyList;
+ IList valList;
+ featureLexicalDict.GenerateLexicalIdList(out keyList, out valList);
+
+ if (debugLevel > 0)
+ {
+ var filename_featureset_raw_format = filename + ".feature.raw_text";
+ var sw = new StreamWriter(filename_featureset_raw_format);
+ // save feature and its id into lists in raw format
+ for (var i = 0; i < keyList.Count; i++)
+ {
+ sw.WriteLine("{0}\t{1}", keyList[i], valList[i]);
+ }
+ sw.Close();
+ }
+
+ //Build feature index
+ var filename_featureset = filename + ".feature";
+ var da = new CRFLite.Utils.DoubleArrayTrieBuilder(thread_num_);
+ if (da.build(keyList, valList, max_slot_usage_rate_threshold) == false)
+ {
+ return false;
+ }
+ //Save indexed feature set into file
+ da.save(filename_featureset);
+
+ if (string.IsNullOrWhiteSpace(modelFileName))
+ {
+ //Clean up all data
+ featureLexicalDict.Clear();
+ featureLexicalDict = null;
+ keyList = null;
+ valList = null;
+
+ GC.Collect();
+
+ //Create weight matrix
+ alpha_ = new double[feature_size() + 1];
+ }
+ else
+ {
+ //Create weight matrix
+ alpha_ = new double[feature_size() + 1];
+ var modelReader = new ModelReader(this.modelFileName);
+ modelReader.LoadModel();
+
+ if (modelReader.y_.Count == y_.Count)
+ {
+ for (var i = 0; i < keyList.Count; i++)
+ {
+ var index = modelReader.get_id(keyList[i]);
+ if (index < 0)
+ {
+ continue;
+ }
+ var size = (keyList[i][0] == 'U' ? y_.Count : y_.Count * y_.Count);
+ for (var j = 0; j < size; j++)
+ {
+ alpha_[valList[i] + j + 1] = modelReader.GetAlpha(index + j);
+ }
+ }
+ }
+ else
+ {
+ }
+
+ //Clean up all data
+ featureLexicalDict.Clear();
+ featureLexicalDict = null;
+ keyList = null;
+ valList = null;
+
+ GC.Collect();
+ }
+
+ return true;
+ }
+
+ //Save model meta data into file
+ public bool SaveModelMetaData(string filename)
+ {
+ var tofs = new StreamWriter(filename);
+
+ // header
+ tofs.WriteLine("version: " + BaseUtils.MODEL_TYPE_NORM);
+ tofs.WriteLine("cost-factor: " + cost_factor_);
+ tofs.WriteLine("maxid: " + maxid_);
+ tofs.WriteLine("xsize: " + xsize_);
+
+ tofs.WriteLine();
+
+ // y
+ for (var i = 0; i < y_.Count; ++i)
+ {
+ tofs.WriteLine(y_[i]);
+ }
+ tofs.WriteLine();
+
+ // template
+ for (var i = 0; i < unigram_templs_.Count; ++i)
+ {
+ tofs.WriteLine(unigram_templs_[i]);
+ }
+ for (var i = 0; i < bigram_templs_.Count; ++i)
+ {
+ tofs.WriteLine(bigram_templs_[i]);
+ }
+
+ tofs.Close();
+
+ return true;
+ }
+
+ ///
+ /// Save feature weights into file
+ ///
+ ///
+ ///
+ ///
+ public void SaveFeatureWeight(string filename, bool bVQ)
+ {
+ var filename_alpha = filename + ".alpha";
+ var tofs = new StreamWriter(filename_alpha, false);
+ var bw = new BinaryWriter(tofs.BaseStream);
+
+ if (bVQ == true)
+ {
+ //Build code book
+ CRFLite.Utils.VectorQuantization vq = new CRFLite.Utils.VectorQuantization();
+ for (long i = 1; i <= maxid_; i++)
+ {
+ vq.Add(alpha_[i]);
+ }
+
+ int vqSize = 256;
+ double distortion = vq.BuildCodebook(vqSize);
+
+ //VQ size
+ bw.Write(vqSize);
+
+ //Save VQ codebook into file
+ for (int j = 0; j < vqSize; j++)
+ {
+ bw.Write(vq.CodeBook[j]);
+ }
+
+ //Save weights
+ for (long i = 1; i <= maxid_; ++i)
+ {
+ bw.Write((byte)vq.ComputeVQ(alpha_[i]));
+ }
+ }
+ else
+ {
+ bw.Write(0);
+ //Save weights
+ for (long i = 1; i <= maxid_; ++i)
+ {
+ bw.Write((float)alpha_[i]);
+ }
+ }
+
+ bw.Close();
+ }
+
+ bool OpenTemplateFile(string filename)
+ {
+ var ifs = new StreamReader(filename);
+ unigram_templs_ = new List();
+ bigram_templs_ = new List();
+ while (ifs.EndOfStream == false)
+ {
+ var line = ifs.ReadLine();
+ if (line.Length == 0 || line[0] == '#')
+ {
+ continue;
+ }
+ if (line[0] == 'U')
+ {
+ unigram_templs_.Add(line);
+ }
+ else if (line[0] == 'B')
+ {
+ bigram_templs_.Add(line);
+ }
+ else
+ {
+ }
+ }
+ ifs.Close();
+ return true;
+ }
+
+ bool OpenTrainCorpusFile(string strTrainingCorpusFileName)
+ {
+ var ifs = new StreamReader(strTrainingCorpusFileName);
+ y_ = new List();
+ trainCorpusList = new List>>();
+ var hashCand = new HashSet();
+ var recordList = new List>();
+
+ var last_xsize = -1;
+ while (ifs.EndOfStream == false)
+ {
+ var line = ifs.ReadLine();
+ if (line.Length == 0 || line[0] == ' ' || line[0] == '\t')
+ {
+ //Current record is finished, save it into the list
+ if (recordList.Count > 0)
+ {
+ trainCorpusList.Add(recordList);
+ recordList = new List>();
+ }
+ continue;
+ }
+
+ var items = line.Split('\t');
+ var size = items.Length;
+ if (last_xsize >= 0 && last_xsize != size)
+ {
+ return false;
+ }
+ last_xsize = size;
+ xsize_ = (uint)(size - 1);
+ recordList.Add(new List(items));
+
+ if (hashCand.Contains(items[items.Length - 1]) == false)
+ {
+ hashCand.Add(items[items.Length - 1]);
+ y_.Add(items[items.Length - 1]);
+ }
+ }
+ ifs.Close();
+
+ return true;
+ }
+
+ //Get feature id from feature set by feature string
+ //If feature string is not existed in the set, generate a new id and return it
+ public bool BuildFeatures(EncoderTagger tagger)
+ {
+ var feature = new List();
+ using (var v = _buildersPool.GetOrCreate())
+ {
+ var localBuilder = v.Item;
+ //tagger.feature_id_ = tagger.feature_cache_.Count;
+ for (var cur = 0; cur < tagger.word_num; ++cur)
+ {
+ for (int index = 0; index < unigram_templs_.Count; index++)
+ {
+ var it = unigram_templs_[index];
+ var strFeature = apply_rule(it, cur, localBuilder, tagger);
+ if (strFeature == null)
+ {
+ }
+ else
+ {
+ var id = featureLexicalDict.GetOrAddId(strFeature.ToString());
+ feature.Add(id);
+ }
+ }
+ tagger.feature_cache_.Add(feature.ToArray());
+ feature.Clear();
+ }
+
+ for (var cur = 1; cur < tagger.word_num; ++cur)
+ {
+ for (int index = 0; index < bigram_templs_.Count; index++)
+ {
+ var it = bigram_templs_[index];
+ var strFeature = apply_rule(it, cur, localBuilder, tagger);
+ if (strFeature == null)
+ {
+ }
+ else
+ {
+ var id = featureLexicalDict.GetOrAddId(strFeature.ToString());
+ feature.Add(id);
+ }
+ }
+
+ tagger.feature_cache_.Add(feature.ToArray());
+ feature.Clear();
+
+ }
+
+ }
+
+ return true;
+ }
+
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/IGenerateFeature.cs b/BotSharp.MachineLearning/CRFLite/IGenerateFeature.cs
new file mode 100644
index 00000000..87748d0d
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/IGenerateFeature.cs
@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public interface IGenerateFeature
+ {
+ bool Initialize();
+ List> GenerateFeature(string strText);
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Node.cs b/BotSharp.MachineLearning/CRFLite/Node.cs
new file mode 100644
index 00000000..a17c47f2
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Node.cs
@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class Node
+ {
+ public int fid;
+ public short x;
+ public short y;
+ public double alpha;
+ public double beta;
+ public double cost;
+ public double bestCost;
+ public Node prev;
+
+ public List lpathList;
+ public List rpathList;
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Path.cs b/BotSharp.MachineLearning/CRFLite/Path.cs
new file mode 100644
index 00000000..7e003f63
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Path.cs
@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class Path
+ {
+ public int fid;
+ public Node rnode;
+ public Node lnode;
+ public double cost;
+
+ public Path()
+ {
+ rnode = null;
+ lnode = null;
+ cost = 0;
+ }
+
+ public void add(Node _lnode, Node _rnode)
+ {
+ lnode = _lnode;
+ rnode = _rnode;
+
+ lnode.rpathList.Add(this);
+ rnode.lpathList.Add(this);
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Pool.cs b/BotSharp.MachineLearning/CRFLite/Pool.cs
new file mode 100644
index 00000000..1f66ccef
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Pool.cs
@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ ///
+ /// Represents general purpose pool that has no restrictions (e.g. grows if it's required)
+ ///
+ ///
+ internal sealed class Pool
+ {
+ private int _totalCount;
+ private readonly ConcurrentStack _container = new ConcurrentStack();
+ private readonly Func, T> _creator;
+ private readonly Action _cleaner;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ public Pool(Func, T> creator, Action cleaner = null)
+ {
+ _creator = creator;
+ _cleaner = cleaner;
+ }
+
+ ///
+ /// Gets item from pool or creates a new item
+ ///
+ ///
+ public PoolItem GetOrCreate()
+ {
+ T item;
+ if (_container.TryPop(out item))
+ {
+ return new PoolItem(item, _cleaner, this);
+ }
+ var newItem = _creator(this);
+ if (newItem == null)
+ {
+ throw new ApplicationException("Unable to create new pool item");
+ }
+ Interlocked.Increment(ref _totalCount);
+ return new PoolItem(newItem, _cleaner, this);
+ }
+
+ ///
+ /// Returns amount of free items in the bag
+ ///
+ public int FreeCount { get { return _container.Count; } }
+
+ ///
+ /// Returns amount items created by pool
+ ///
+ public int TotalCount { get { return _totalCount; } }
+
+ private void Return(T item)
+ {
+ _container.Push(item);
+ }
+
+ ///
+ /// Pool item that is return when pool request is processed
+ ///
+ ///
+ internal struct PoolItem : IDisposable
+ {
+ ///
+ /// Pooled item
+ ///
+ public readonly T1 Item;
+ private readonly Pool _owner;
+ private readonly Action _cleaner;
+
+ ///
+ /// Creates a new pool item
+ ///
+ ///
+ ///
+ ///
+ internal PoolItem(T1 item, Action cleaner, Pool owner)
+ {
+ Item = item;
+ _cleaner = cleaner;
+ _owner = owner;
+ }
+
+ ///
+ /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+ ///
+ public void Dispose()
+ {
+ _cleaner?.Invoke(Item);
+ _owner.Return(Item);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs b/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs
new file mode 100644
index 00000000..d623dc97
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs
@@ -0,0 +1,131 @@
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class SegDecoderTagger : DecoderTagger
+ {
+ public SegDecoderTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+ : base(nbest, this_crf_max_word_num)
+ {
+ crf_max_word_num = this_crf_max_word_num;
+ }
+
+ int seg_termbuf_build(crf_seg_out term_buf)
+ {
+ term_buf.Clear();
+
+ //build raw result at first
+ var iRet = termbuf_build(term_buf);
+ if (iRet != BaseUtils.ERROR_SUCCESS)
+ {
+ return iRet;
+ }
+
+ //Then build token result
+ var term_len = 0;
+ var weight = 0.0;
+ var num = 0;
+ for (var i = 0; i < x_.Count; i++)
+ {
+ //Adding the length of current token
+ var strTag = term_buf.result_[i];
+ term_len += x_[i][0].Length;
+ weight += term_buf.weight_[i];
+ num++;
+
+ //Check if current term is the end of a token
+ if ((strTag.StartsWith("B_") == false &&
+ strTag.StartsWith("M_") == false) ||
+ i == x_.Count - 1)
+ {
+ var tkn = new SegToken();
+ tkn.length = term_len;
+ tkn.offset = term_buf.termTotalLength;
+
+ var spos = strTag.IndexOf('_');
+ if (spos < 0)
+ {
+ if (strTag == "NOR")
+ {
+ tkn.strTag = "";
+ }
+ else
+ {
+ tkn.strTag = strTag;
+ }
+ }
+ else
+ {
+ tkn.strTag = strTag.Substring(spos + 1);
+ }
+
+ term_buf.termTotalLength += term_len;
+ //Calculate each token's weight
+ switch (vlevel_)
+ {
+ case 0:
+ tkn.fWeight = 0.0;
+ break;
+ case 2:
+ tkn.fWeight = weight / num;
+ weight = 0.0;
+ num = 0;
+ break;
+ }
+
+ term_buf.tokenList.Add(tkn);
+ term_len = 0;
+ }
+ }
+
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+
+ public int output(crf_seg_out[] pout)
+ {
+ var n = 0;
+ var ret = 0;
+
+ if (nbest_ == 1)
+ {
+ //If only best result and no need probability, "next" is not to be used
+ ret = seg_termbuf_build(pout[0]);
+ if (ret < 0)
+ {
+ return ret;
+ }
+ }
+ else
+ {
+ //Fill the n best result
+ var iNBest = nbest_;
+ if (pout.Length < iNBest)
+ {
+ iNBest = pout.Length;
+ }
+
+ for (n = 0; n < iNBest; ++n)
+ {
+ ret = next();
+ if (ret < 0)
+ {
+ break;
+ }
+
+ ret = seg_termbuf_build(pout[n]);
+ if (ret < 0)
+ {
+ return ret;
+ }
+ }
+ }
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Tagger.cs b/BotSharp.MachineLearning/CRFLite/Tagger.cs
new file mode 100644
index 00000000..94f81824
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Tagger.cs
@@ -0,0 +1,154 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class Tagger
+ {
+ public List> x_;
+ public Node[,] node_; //Node matrix
+ public short ysize_;
+ public short word_num; //the number of tokens need to be labeled
+ public double Z_; //概率值
+ public double cost_; //The path cost
+ public short[] result_;
+ public List feature_cache_;
+
+ //Calculate the cost of each path. It's used for finding the best or N-best result
+ public int viterbi()
+ {
+ var bestc = double.MinValue;
+ Node bestNode = null;
+
+ for (var i = 0; i < word_num; ++i)
+ {
+ for (var j = 0; j < ysize_; ++j)
+ {
+ bestc = double.MinValue;
+ bestNode = null;
+
+ var node_i_j = node_[i, j];
+
+ for (int index = 0; index < node_i_j.lpathList.Count; ++index)
+ {
+ var p = node_i_j.lpathList[index];
+ var cost = p.lnode.bestCost + p.cost + node_i_j.cost;
+ if (cost > bestc)
+ {
+ bestc = cost;
+ bestNode = p.lnode;
+ }
+ }
+
+ node_i_j.prev = bestNode;
+ node_i_j.bestCost = bestNode != null ? bestc : node_i_j.cost;
+ }
+ }
+
+ bestc = double.MinValue;
+ bestNode = null;
+
+ var s = (short)(word_num - 1);
+ for (short j = 0; j < ysize_; ++j)
+ {
+ if (bestc < node_[s, j].bestCost)
+ {
+ bestNode = node_[s, j];
+ bestc = node_[s, j].bestCost;
+ }
+ }
+
+ var n = bestNode;
+ while (n != null)
+ {
+ result_[n.x] = n.y;
+ n = n.prev;
+ }
+
+ cost_ = -node_[s, result_[s]].bestCost;
+
+ return BaseUtils.ERROR_SUCCESS;
+ }
+
+ private void calcAlpha(int m, int n)
+ {
+ var nd = node_[m, n];
+ nd.alpha = 0.0;
+
+ var i = 0;
+ for (int index = 0; index < nd.lpathList.Count; index++)
+ {
+ var p = nd.lpathList[index];
+ nd.alpha = BaseUtils.logsumexp(nd.alpha, p.cost + p.lnode.alpha, (i == 0));
+ i++;
+ }
+ nd.alpha += nd.cost;
+ }
+
+ private void calcBeta(int m, int n)
+ {
+ var nd = node_[m, n];
+ nd.beta = 0.0f;
+ if (m + 1 < word_num)
+ {
+ var i = 0;
+ for (int index = 0; index < nd.rpathList.Count; index++)
+ {
+ var p = nd.rpathList[index];
+ nd.beta = BaseUtils.logsumexp(nd.beta, p.cost + p.rnode.beta, (i == 0));
+ i++;
+ }
+ }
+ nd.beta += nd.cost;
+ }
+
+ public void forwardbackward()
+ {
+ for (int i = 0, k = word_num - 1; i < word_num; ++i, --k)
+ {
+ for (var j = 0; j < ysize_; ++j)
+ {
+ calcAlpha(i, j);
+ calcBeta(k, j);
+ }
+ }
+
+ Z_ = 0.0;
+ for (var j = 0; j < ysize_; ++j)
+ {
+ Z_ = BaseUtils.logsumexp(Z_, node_[0, j].beta, j == 0);
+ }
+ }
+
+
+ //Assign feature ids to node and path
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public int RebuildFeatures()
+ {
+ var fid = 0;
+ for (short cur = 0; cur < word_num; ++cur)
+ {
+ for (short i = 0; i < ysize_; ++i)
+ {
+ node_[cur, i].fid = fid;
+ if (cur > 0)
+ {
+ Node previousNode = node_[cur - 1, i];
+ for (int index = 0; index < previousNode.rpathList.Count; ++index)
+ {
+ Path path = previousNode.rpathList[index];
+ path.fid = fid + word_num - 1;
+ }
+ }
+ }
+
+ ++fid;
+ }
+
+ return 0;
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils.cs b/BotSharp.MachineLearning/CRFLite/Utils.cs
new file mode 100644
index 00000000..90d85fb8
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils.cs
@@ -0,0 +1,160 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+ public class QueueElement
+ {
+ public Node node;
+ public QueueElement next;
+ public double fx;
+ public double gx;
+ };
+
+ public class Heap
+ {
+ public int capacity;
+ public int elem_size; //size of elem_list
+ public int size; // size of elem_ptr_list
+ public List elem_ptr_list;
+ public List elem_list;
+ };
+
+ public class BaseUtils
+ {
+ public const double eps = 1e-7;
+
+
+ public const int MINUS_LOG_EPSILON = 13;
+ public const int DEFAULT_CRF_MAX_WORD_NUM = 100;
+
+ public const int MODEL_TYPE_NORM = 100;
+
+
+ public const int ERROR_INVALIDATED_FEATURE = -8;
+ public const int ERROR_HEAP_SIZE_TOO_BIG = -7;
+ public const int ERROR_INSERT_HEAP_FAILED = -6;
+ public const int ERROR_EMPTY_FEATURE = -5;
+ public const int ERROR_INVALIDATED_PARAMETER = -4;
+ public const int ERROR_WRONG_STATUS = -3;
+ public const int ERROR_TOO_LONG_WORD = -2;
+ public const int ERROR_UNKNOWN = -1;
+ public const int ERROR_SUCCESS = 0;
+
+ public static Heap heap_init(int max_size)
+ {
+ Heap H;
+
+ H = new Heap();
+ H.capacity = max_size;
+ H.size = 0;
+ H.elem_size = 0;
+
+ H.elem_ptr_list = new List(max_size + 1);
+ H.elem_list = new List(max_size + 1);
+
+ for (var z = 0; z < max_size; z++)
+ {
+ H.elem_list.Add(new QueueElement());
+ H.elem_ptr_list.Add(null);
+ }
+ H.elem_list[0].fx = double.MinValue;
+ H.elem_ptr_list.Add(H.elem_list[0]);
+
+ return H;
+ }
+
+ public static QueueElement allc_from_heap(Heap H)
+ {
+ if (H.elem_size >= H.capacity)
+ {
+ return null;
+ }
+ else
+ {
+ return H.elem_list[++H.elem_size];
+ }
+ }
+
+ public static int heap_insert(QueueElement qe, Heap H)
+ {
+ if (H.size >= H.capacity)
+ {
+ return BaseUtils.ERROR_HEAP_SIZE_TOO_BIG;
+ }
+ var i = ++H.size;
+ while (i != 1 && H.elem_ptr_list[i / 2].fx > qe.fx)
+ {
+ H.elem_ptr_list[i] = H.elem_ptr_list[i / 2]; //此时i还没有进行i/2操作
+ i /= 2;
+ }
+ H.elem_ptr_list[i] = qe;
+ return 0;
+ }
+
+ public static QueueElement heap_delete_min(Heap H)
+ {
+ var min_elem = H.elem_ptr_list[1]; //堆是从第1号元素开始的
+ var last_elem = H.elem_ptr_list[H.size--];
+ int i = 1, ci = 2;
+ while (ci <= H.size)
+ {
+ if (ci < H.size && H.elem_ptr_list[ci].fx > H.elem_ptr_list[ci + 1].fx)
+ {
+ ci++;
+ }
+ if (last_elem.fx <= H.elem_ptr_list[ci].fx)
+ {
+ break;
+ }
+ H.elem_ptr_list[i] = H.elem_ptr_list[ci];
+ i = ci;
+ ci *= 2;
+ }
+ H.elem_ptr_list[i] = last_elem;
+ return min_elem;
+ }
+
+ public static bool is_heap_empty(Heap H)
+ {
+ return H.size == 0;
+ }
+
+ public static void heap_reset(Heap H)
+ {
+ if (H != null)
+ {
+ H.size = 0;
+ H.elem_size = 0;
+ }
+ }
+
+ public static double logsumexp(double x, double y, bool flg)
+ {
+ if (flg)
+ {
+ return y; // init mode
+ }
+ double vmin;
+ double vmax;
+ if (x > y)
+ {
+ vmin = y;
+ vmax = x;
+ }
+ else
+ {
+ vmin = x;
+ vmax = y;
+ }
+
+ if (vmax > vmin + MINUS_LOG_EPSILON)
+ {
+ return vmax;
+ }
+ return vmax + Math.Log(Math.Exp(vmin - vmax) + 1.0);
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/BTreeDictionary.cs b/BotSharp.MachineLearning/CRFLite/Utils/BTreeDictionary.cs
new file mode 100644
index 00000000..8895ba04
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/BTreeDictionary.cs
@@ -0,0 +1,1337 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ ///
+ /// Represents a generic interface of an ordered collection.
+ ///
+ /// The type of elements in the collection.
+ public interface ISortedCollection : ICollection
+ {
+ ///
+ /// Gets the comparer used to order items in the collection.
+ ///
+ IComparer Comparer
+ {
+ get;
+ }
+
+ ///
+ /// Get all items equal to or greater than the specified value, starting with the lowest index and moving forwards.
+ ///
+ IEnumerable WhereGreaterOrEqual(T value);
+
+ ///
+ /// Get all items less than or equal to the specified value, starting with the highest index and moving backwards.
+ ///
+ IEnumerable WhereLessOrEqualBackwards(T value);
+
+ ///
+ /// Gets the index of the first item greater than the specified value.
+ /// ///
+ int FirstIndexWhereGreaterThan(T value);
+
+ ///
+ /// Gets the index of the last item less than the specified key.
+ ///
+ int LastIndexWhereLessThan(T value);
+
+ ///
+ /// Gets the item at the specified index.
+ ///
+ T At(int index);
+
+ ///
+ /// Removes the item at the specified index.
+ ///
+ void RemoveAt(int index);
+
+ ///
+ /// Get all items starting at the index, and moving forward.
+ ///
+ IEnumerable ForwardFromIndex(int index);
+
+ ///
+ /// Get all items starting at the index, and moving backward.
+ ///
+ IEnumerable BackwardFromIndex(int index);
+ }
+
+ ///
+ /// Represents a generic interface of ordered key/value pairs.
+ ///
+ /// The type of the key.
+ /// The type of the value.
+ public interface ISortedDictionary : IDictionary
+ {
+ ///
+ /// Get all items having a key equal to or greater than the specified key, starting with the lowest index and moving forwards.
+ ///
+ IEnumerable> WhereGreaterOrEqual(TKey key);
+
+ ///
+ /// Get all items less than or equal to the specified value, starting with the highest index and moving backwards.
+ ///
+ IEnumerable> WhereLessOrEqualBackwards(TKey keyUpperBound);
+
+ ///
+ /// Gets the sorted collection of keys.
+ ///
+ new ISortedCollection Keys
+ {
+ get;
+ }
+
+ ///
+ /// Gets the item at the specified index.
+ ///
+ KeyValuePair At(int index);
+
+ ///
+ /// Removes the item at the specified index.
+ ///
+ void RemoveAt(int index);
+
+ ///
+ /// Sets the value at the specified index.
+ ///
+ ///
+ ///
+ void SetValueAt(int index, TValue value);
+
+ ///
+ /// Get all items starting at the index, and moving forward.
+ ///
+ IEnumerable> ForwardFromIndex(int index);
+
+ ///
+ /// Get all items starting at the index, and moving backward.
+ ///
+ IEnumerable> BackwardFromIndex(int index);
+ }
+
+ ///
+ /// An O(log N) implementation of the ISortedDictionary interface.
+ ///
+ /// The type for the sorted keys.
+ /// The type for the associated values.
+ public class BTreeDictionary : ISortedDictionary
+ {
+ #region Fields
+
+ Node root;
+ readonly Node first;
+ readonly KeyCollection keys;
+ readonly ValueCollection values;
+ readonly IComparer keyComparer;
+
+ private void ObjectInvariant()
+ {
+
+ }
+
+ #endregion
+
+ #region Construction
+
+ ///
+ /// Initializes a new BTreeDictionary instance optimized for the specified node capacity.
+ ///
+ /// The capacity in keys for each node in the tree structure.
+ public BTreeDictionary(int nodeCapacity = 128)
+ : this(Comparer.Default, nodeCapacity)
+ {
+
+ }
+
+ ///
+ /// Initializes a new BTreeDictionary instance.
+ ///
+ /// The comparer for ordering keys in the structure.
+ /// The capacity in keys for each node in the tree structure.
+ public BTreeDictionary(IComparer keyComparer, int nodeCapacity)
+ {
+
+
+ this.keyComparer = keyComparer;
+ this.first = new Node(nodeCapacity);
+ this.root = this.first;
+
+ this.keys = new KeyCollection(this);
+ this.values = new ValueCollection(this);
+ }
+
+ #endregion
+
+ #region Properties
+
+ ///
+ /// Gets the value associated with the specified key. An arbitrary value will be chosen if key is a duplicate.
+ ///
+ /// The key for which to retrieve the associated value.
+ /// The value associated with the key.
+ public TValue this[TKey key]
+ {
+ get
+ {
+ TValue result;
+ if (this.TryGetValue(key, out result))
+ return result;
+ throw new InvalidOperationException("error");
+ }
+ set
+ {
+
+ Node leaf;
+ int pos;
+ if (Node.Find(root, key, KeyComparer, 0, out leaf, out pos))
+ leaf.SetValue(pos, value);
+ else
+ {
+ Node.Insert(key, ref leaf, ref pos, ref root);
+ leaf.SetValue(pos, value);
+ }
+ }
+ }
+
+ ///
+ /// Gets the number of key value pairs in the dictionary.
+ ///
+ public int Count
+ {
+ get
+ {
+
+ return this.root.TotalCount;
+ }
+ }
+
+ ///
+ /// Gets the key comparer.
+ ///
+ public IComparer KeyComparer
+ {
+ get
+ {
+
+ return this.keyComparer;
+ }
+ }
+
+ ///
+ /// Gets the collection of keys in the dictionary.
+ ///
+ public ISortedCollection Keys
+ {
+ get
+ {
+
+ return this.keys;
+ }
+ }
+
+ public IList KeyList
+ {
+ get
+ {
+ return this.keys;
+ }
+ }
+
+ ///
+ /// Gets the collection of values in the dictionary.
+ ///
+ public ICollection Values
+ {
+ get
+ {
+
+ return this.values;
+ }
+ }
+
+ public IList ValueList
+ {
+ get
+ {
+ return this.values;
+ }
+ }
+
+ ///
+ /// Gets or sets indication whether this dictionary is readonly or mutable.
+ ///
+ public bool IsReadOnly
+ {
+ get;
+ set;
+ }
+
+
+ #endregion
+
+ #region Methods
+
+ ///
+ /// Gets indication of whether the dictionary contains an entry for the specified key.
+ ///
+ /// The key.
+ /// True if the dictionary contains the key; otherwise, false.
+ public bool ContainsKey(TKey key)
+ {
+ Node leaf;
+ int pos;
+ var found = Node.Find(root, key, KeyComparer, 0, out leaf, out pos);
+ return found;
+ }
+
+ ///
+ /// Tries to get the value for the specified key. An arbitrary value will be chosen if key is a duplicate.
+ ///
+ /// The key for which to try to get the value.
+ /// The value found for the specified key, or a default value if not found.
+ /// True if the value was found; otherwise, false.
+ public bool TryGetValue(TKey key, out TValue value)
+ {
+ try
+ {
+ Node leaf;
+ int pos;
+ var found = Node.Find(root, key, KeyComparer, 0, out leaf, out pos);
+ value = found ? leaf.GetValue(pos) : default(TValue);
+ return found;
+ }
+ catch (System.Exception)
+ {
+ value = default(TValue);
+ return false;
+ }
+
+ }
+
+ ///
+ /// Adds the specified key and value to the dictionary.
+ ///
+ /// The key to add.
+ /// The value to associate with the key.
+ public void Add(TKey key, TValue value)
+ {
+
+ Node leaf;
+ int pos;
+ var found = Node.Find(root, key, KeyComparer, 0, out leaf, out pos);
+ if (found)
+ {
+ //The key is already in the dictionary, throw exception out
+ throw new InvalidOperationException("The key is already in the dictionary");
+ }
+
+ Node.Insert(key, ref leaf, ref pos, ref root);
+ leaf.SetValue(pos, value);
+ }
+
+ ///
+ /// Gets the key value pair at the specified index.
+ ///
+ /// The index at which to get the key value pair.
+ /// The key value pair at the specified index.
+ public KeyValuePair At(int index)
+ {
+
+ var leaf = Node.LeafAt(root, ref index);
+ return new KeyValuePair(leaf.GetKey(index), leaf.GetValue(index));
+ }
+
+ ///
+ /// Clears the dictionary of all items.
+ ///
+ public void Clear()
+ {
+
+ Node.Clear(first);
+ root = first;
+ }
+
+ ///
+ /// Remove the key and associated value from the dictionary.
+ ///
+ /// The key to remove.
+ /// True if the key was removed; otherwise, false if key was not found.
+ public bool Remove(TKey key)
+ {
+
+ Node leaf;
+ int pos;
+ if (!Node.Find(root, key, KeyComparer, 0, out leaf, out pos))
+ return false;
+
+ Node.Remove(leaf, pos, ref root);
+ return true;
+ }
+
+ ///
+ /// Removes the key and associated value from the dictionary at the specified index.
+ ///
+ /// The index at which to remove the key value pair.
+ public void RemoveAt(int index)
+ {
+
+ var leaf = Node.LeafAt(root, ref index);
+ Node.Remove(leaf, index, ref root);
+ }
+
+
+ ///
+ /// Get all items starting at the index, and moving forward.
+ ///
+ public IEnumerable> ForwardFromIndex(int index)
+ {
+
+ var node = Node.LeafAt(root, ref index);
+ return Node.ForwardFromIndex(node, index);
+ }
+
+ ///
+ /// Get all items starting at the index, and moving backward.
+ ///
+ public IEnumerable> BackwardFromIndex(int index)
+ {
+
+ var node = Node.LeafAt(root, ref index);
+ return Node.BackwardFromIndex(node, index);
+ }
+
+ ///
+ /// Sets the value at the specified index, leaving the key unchanged.
+ ///
+ /// The index at which to set the value.
+ /// The value to associate at the specified index.
+ public void SetValueAt(int index, TValue value)
+ {
+
+ var leaf = Node.LeafAt(root, ref index);
+ leaf.SetValue(index, value);
+ }
+
+ ///
+ /// Gets an enumerator of key value pairs for the entire collection in sorted ascending order.
+ ///
+ ///
+ public IEnumerator> GetEnumerator()
+ {
+
+ return Node.ForwardFromIndex(first, 0).GetEnumerator();
+ }
+
+ ///
+ /// Gets an enumerator of key value pairs in ascending order by key, for all items with a key
+ /// equal or greater than the specified key lower bound.
+ ///
+ /// The value at which to start returning keys.
+ /// All key value pairs having a key equal or greater than the lower bound, in key ascending order.
+ public IEnumerable> WhereGreaterOrEqual(TKey keyLowerBound)
+ {
+
+ Node leaf;
+ int leafPos;
+ Node.Find(root, keyLowerBound, KeyComparer, 0, out leaf, out leafPos);
+ return Node.ForwardFromIndex(leaf, leafPos);
+ }
+
+ ///
+ /// Gets an enumerator of key value pairs in descending order by key, for all items with a key
+ /// less than or equal than the specified key upper bound.
+ ///
+ /// The value at which to start returning keys.
+ /// All key value pairs having a key equal or less than the upper bound, in key descending order.
+ public IEnumerable> WhereLessOrEqualBackwards(TKey keyUpperBound)
+ {
+
+ Node leaf;
+ int leafPos;
+ var found = Node.Find(root, keyUpperBound, KeyComparer, 0, out leaf, out leafPos);
+ if (!found)
+ --leafPos;
+ return Node.BackwardFromIndex(leaf, leafPos);
+ }
+
+ ///
+ /// Copy the entire dictionary to the specified array, starting at the specified array index.
+ ///
+ /// The array into which to copy.
+ /// The index at which to start copying.
+ public void CopyTo(KeyValuePair[] array, int arrayIndex)
+ {
+ foreach (var item in this)
+ array[arrayIndex++] = item;
+ }
+
+ #endregion
+
+ #region Implementation - Nested Types
+
+ sealed class Node
+ {
+ #region Fields
+
+ readonly TKey[] keys;
+ readonly TValue[] values;
+ readonly Node[] nodes;
+
+ int nodeCount;
+ int totalCount;
+
+ Node parent;
+ Node next;
+ Node prev;
+ private void ObjectInvariant()
+ {
+
+ }
+
+ #endregion
+
+ #region Construction
+
+ ///
+ /// Initialize the first node in the BTree structure.
+ ///
+ public Node(int nodeCapacity)
+ : this(nodeCapacity, true)
+ {
+ }
+
+ #endregion
+
+ #region Properties
+
+ public int TotalCount
+ {
+ get
+ {
+ return this.totalCount;
+ }
+ }
+
+ public bool IsRoot
+ {
+ get
+ {
+ return this.parent == null;
+ }
+ }
+
+ public bool IsLeaf
+ {
+ get
+ {
+ return nodes == null;
+ }
+ }
+
+ public int NodeCount
+ {
+ get
+ {
+ return this.nodeCount;
+ }
+ }
+
+ #endregion
+
+ #region Methods
+
+ ///
+ /// Gets the key at the specified position.
+ ///
+ public TKey GetKey(int pos)
+ {
+ return this.keys[pos];
+ }
+
+ ///
+ /// Gets the value at the specified position.
+ ///
+ public TValue GetValue(int pos)
+ {
+ return this.values[pos];
+ }
+
+ ///
+ /// Sets the value at the specified position.
+ ///
+ public void SetValue(int pos, TValue value)
+ {
+ this.values[pos] = value;
+ }
+
+ ///
+ /// Get the leaf node at the specified index in the tree defined by the specified root.
+ ///
+ public static Node LeafAt(Node root, ref int pos)
+ {
+
+ int nodeIndex = 0;
+ while (true)
+ {
+ if (root.nodes == null)
+ {
+ return root;
+ }
+
+ var node = root.nodes[nodeIndex];
+ if (pos < node.totalCount)
+ {
+ root = node;
+ nodeIndex = 0;
+ }
+ else
+ {
+ pos -= node.totalCount;
+ ++nodeIndex;
+ }
+ }
+ }
+
+ ///
+ /// Find the node and index in the tree defined by the specified root.
+ ///
+ public static bool Find(Node root, TKey key, IComparer keyComparer, int duplicatesBias, out Node leaf, out int pos)
+ {
+
+ pos = Array.BinarySearch(root.keys, 0, root.nodeCount, key, keyComparer);
+ while (root.nodes != null)
+ {
+ if (pos >= 0)
+ {
+ if (duplicatesBias != 0)
+ MoveToDuplicatesBoundary(key, keyComparer, duplicatesBias, ref root, ref pos);
+
+ // Found an exact match. Move down one level.
+ root = root.nodes[pos];
+ }
+ else
+ {
+ // No exact match. Find greatest lower bound.
+ pos = ~pos;
+ if (pos > 0)
+ --pos;
+ root = root.nodes[pos];
+ }
+
+ pos = Array.BinarySearch(root.keys, 0, root.nodeCount, key, keyComparer);
+ }
+
+ leaf = root;
+ if (pos < 0)
+ {
+ pos = ~pos;
+ return false;
+ }
+
+ if (duplicatesBias != 0)
+ MoveToDuplicatesBoundary(key, keyComparer, duplicatesBias, ref leaf, ref pos);
+
+ return true;
+ }
+
+ ///
+ /// Insert a new key into the leaf node at the specified position.
+ ///
+ public static void Insert(TKey key, ref Node leaf, ref int pos, ref Node root)
+ {
+ // Make sure there is space for the new key.
+ if (EnsureSpace(leaf, ref root) && pos > leaf.nodeCount)
+ {
+ pos -= leaf.nodeCount;
+ leaf = leaf.next;
+ }
+
+ // Insert the key.
+ int moveCount = leaf.nodeCount - pos;
+ Array.Copy(leaf.keys, pos, leaf.keys, pos + 1, moveCount);
+ leaf.keys[pos] = key;
+ ++leaf.nodeCount;
+ EnsureParentKey(leaf, pos);
+
+ // Insert space for the value. Caller is responsible for filling in value.
+ Array.Copy(leaf.values, pos, leaf.values, pos + 1, moveCount);
+
+ // Update total counts.
+ for (var node = leaf; node != null; node = node.parent)
+ ++node.totalCount;
+ }
+
+ ///
+ /// Remove the item from the node at the specified position.
+ ///
+ public static bool Remove(Node leaf, int pos, ref Node root)
+ {
+
+ // Update total counts.
+ for (var node = leaf; node != null; node = node.parent)
+ --node.totalCount;
+
+ // Remove the key and value from the node.
+ --leaf.nodeCount;
+ Array.Copy(leaf.keys, pos + 1, leaf.keys, pos, leaf.nodeCount - pos);
+ Array.Copy(leaf.values, pos + 1, leaf.values, pos, leaf.nodeCount - pos);
+ leaf.keys[leaf.nodeCount] = default(TKey);
+ leaf.values[leaf.nodeCount] = default(TValue);
+
+ // Make sure parent keys index correctly into this node.
+ if (leaf.nodeCount > 0)
+ EnsureParentKey(leaf, pos);
+
+ // Merge this node with others if it is below the node capacity threshold.
+ Merge(leaf, ref root);
+ return true;
+ }
+
+ ///
+ /// Get an ascending enumerable for the collection, starting an the index in the specified leaf node.
+ ///
+ public static IEnumerable> ForwardFromIndex(Node leaf, int pos)
+ {
+
+ while (leaf != null)
+ {
+ while (pos < leaf.nodeCount)
+ {
+ yield return new KeyValuePair(leaf.GetKey(pos), leaf.GetValue(pos));
+ ++pos;
+ }
+ pos -= leaf.nodeCount;
+ leaf = leaf.next;
+ }
+ }
+
+ ///
+ /// Get a descending enumerable, starting at the index in the specified leaf node.
+ ///
+ public static IEnumerable> BackwardFromIndex(Node leaf, int pos)
+ {
+
+ if (pos == -1)
+ {
+ // Handle special case to start moving in the previous node.
+ leaf = leaf.prev;
+ if (leaf != null)
+ pos = leaf.nodeCount - 1;
+ else
+ pos = 0;
+ }
+ else if (pos == leaf.NodeCount)
+ {
+ // Handle special case to start moving in the next node.
+ if (leaf.next == null)
+ --pos;
+ else
+ {
+ leaf = leaf.next;
+ pos = 0;
+ }
+ }
+
+ // Loop thru collection, yielding each value in sequence.
+ while (leaf != null)
+ {
+ while (pos >= 0)
+ {
+ yield return new KeyValuePair(leaf.GetKey(pos), leaf.GetValue(pos));
+ --pos;
+ }
+ leaf = leaf.prev;
+ if (leaf != null)
+ pos += leaf.nodeCount;
+ }
+ }
+
+ ///
+ /// Clear all keys and values from the specified node.
+ ///
+ public static void Clear(Node firstNode)
+ {
+
+ int clearCount = firstNode.nodeCount;
+
+ Array.Clear(firstNode.keys, 0, clearCount);
+ Array.Clear(firstNode.values, 0, clearCount);
+ firstNode.nodeCount = 0;
+ firstNode.totalCount = 0;
+
+ firstNode.parent = null;
+ firstNode.next = null;
+ }
+
+ ///
+ /// Get the index relative to the root node, for the position in the specified leaf.
+ ///
+ public static int GetRootIndex(Node leaf, int pos)
+ {
+ var node = leaf;
+ var rootIndex = pos;
+ while (node.parent != null)
+ {
+ int nodePos = Array.IndexOf(node.parent.nodes, node, 0, node.parent.nodeCount);
+ for (int i = 0; i < nodePos; ++i)
+ rootIndex += node.parent.nodes[i].totalCount;
+ node = node.parent;
+ }
+ return rootIndex;
+ }
+
+ #endregion
+
+ #region Implementation
+
+ Node(int nodeCapacity, bool leaf)
+ {
+ this.keys = new TKey[nodeCapacity];
+
+ if (leaf)
+ {
+ this.values = new TValue[nodeCapacity];
+ this.nodes = null;
+ }
+ else
+ {
+ this.values = null;
+ this.nodes = new Node[nodeCapacity];
+ }
+
+ this.nodeCount = 0;
+ this.totalCount = 0;
+ this.parent = null;
+ this.next = null;
+ this.prev = null;
+ }
+
+ ///
+ /// (Assumes: key is a duplicate in node at pos) Move to the side on the range of duplicates,
+ /// as indicated by the sign of duplicatesBias.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ static void MoveToDuplicatesBoundary(TKey key, IComparer keyComparer, int duplicatesBias, ref Node node, ref int pos)
+ {
+ // Technically, we could adjust the binary search to perform most of this step, but duplicates
+ // are usually unexpected.. algorithm is still O(log N), because scan include at most a scan thru two nodes
+ // worth of keys, for each level.
+ // Also, the binary search option would still need the ugliness of the special case for moving into the
+ // previous node; it would only be a little faster, on average, assuming large numbers of duplicates were common.
+
+ if (duplicatesBias < 0)
+ {
+ // Move backward over duplicates.
+ while (pos > 0 && 0 == keyComparer.Compare(node.keys[pos - 1], key))
+ --pos;
+
+ // Special case: duplicates can span backwards into the previous node because the parent
+ // key pivot might be in the center for the duplicates.
+ if (pos == 0 && node.prev != null)
+ {
+ var prev = node.prev;
+ var prevPos = prev.NodeCount;
+ while (prevPos > 0 && 0 == keyComparer.Compare(prev.keys[prevPos - 1], key))
+ {
+ --prevPos;
+ }
+ if (prevPos < prev.NodeCount)
+ {
+ node = prev;
+ pos = prevPos;
+ }
+ }
+ }
+ else
+ {
+ // Move forward over duplicates.
+ while (pos < node.NodeCount - 1 && 0 == keyComparer.Compare(node.keys[pos + 1], key))
+ ++pos;
+ }
+ }
+
+ static bool EnsureSpace(Node node, ref Node root)
+ {
+ if (node.nodeCount < node.keys.Length)
+ return false;
+
+ EnsureParent(node, ref root);
+ EnsureSpace(node.parent, ref root);
+
+ var sibling = new Node(node.keys.Length, node.nodes == null);
+ sibling.next = node.next;
+ sibling.prev = node;
+ sibling.parent = node.parent;
+
+ if (node.next != null)
+ node.next.prev = sibling;
+ node.next = sibling;
+
+ int pos = Array.IndexOf(node.parent.nodes, node, 0, node.parent.nodeCount);
+ int siblingPos = pos + 1;
+
+ Array.Copy(node.parent.keys, siblingPos, node.parent.keys, siblingPos + 1, node.parent.nodeCount - siblingPos);
+ Array.Copy(node.parent.nodes, siblingPos, node.parent.nodes, siblingPos + 1, node.parent.nodeCount - siblingPos);
+ ++node.parent.nodeCount;
+ node.parent.nodes[siblingPos] = sibling;
+
+ int half = node.nodeCount / 2;
+ int halfCount = node.nodeCount - half;
+ Move(node, half, sibling, 0, halfCount);
+ return true;
+ }
+
+ static void Move(Node source, int sourceIndex, Node target, int targetIndex, int moveCount)
+ {
+ Move(source.keys, sourceIndex, source.nodeCount, target.keys, targetIndex, target.nodeCount, moveCount);
+ if (source.values != null)
+ Move(source.values, sourceIndex, source.nodeCount, target.values, targetIndex, target.nodeCount, moveCount);
+
+ int totalMoveCount;
+ if (source.nodes == null)
+ {
+ totalMoveCount = moveCount;
+ }
+ else
+ {
+ Move(source.nodes, sourceIndex, source.nodeCount, target.nodes, targetIndex, target.nodeCount, moveCount);
+ totalMoveCount = 0;
+ for (int i = 0; i < moveCount; ++i)
+ {
+ var child = target.nodes[targetIndex + i];
+ child.parent = target;
+ totalMoveCount += child.totalCount;
+ }
+ }
+
+ source.nodeCount -= moveCount;
+ target.nodeCount += moveCount;
+
+ var sn = source;
+ var tn = target;
+ while (sn != null && sn != tn)
+ {
+ sn.totalCount -= totalMoveCount;
+ tn.totalCount += totalMoveCount;
+ sn = sn.parent;
+ tn = tn.parent;
+ }
+
+ EnsureParentKey(source, sourceIndex);
+ EnsureParentKey(target, targetIndex);
+ }
+
+ static void Move(TItem[] source, int sourceIndex, int sourceTotal, TItem[] target, int targetIndex, int targetTotal, int count)
+ {
+ Array.Copy(target, targetIndex, target, targetIndex + count, targetTotal - targetIndex);
+ Array.Copy(source, sourceIndex, target, targetIndex, count);
+ Array.Copy(source, sourceIndex + count, source, sourceIndex, sourceTotal - sourceIndex - count);
+ Array.Clear(source, sourceTotal - count, count);
+ }
+
+ static void EnsureParent(Node node, ref Node root)
+ {
+ if (node.parent != null)
+ return;
+
+ var parent = new Node(node.keys.Length, false);
+ parent.totalCount = node.totalCount;
+ parent.nodeCount = 1;
+ parent.keys[0] = node.keys[0];
+ parent.nodes[0] = node;
+
+ node.parent = parent;
+ root = parent;
+ }
+
+ static void EnsureParentKey(Node node, int pos)
+ {
+ while (pos == 0 && node.parent != null)
+ {
+ pos = Array.IndexOf(node.parent.nodes, node, 0, node.parent.nodeCount);
+ node.parent.keys[pos] = node.keys[0];
+ node = node.parent;
+ }
+ }
+
+ static void Merge(Node node, ref Node root)
+ {
+ if (node.nodeCount == 0)
+ {
+ // Handle special case: Empty node.
+ if (node.parent == null)
+ return;
+
+ // Remove the node from the parent nodes.
+ int pos = Array.IndexOf(node.parent.nodes, node, 0, node.parent.nodeCount);
+ --node.parent.nodeCount;
+ Array.Copy(node.parent.keys, pos + 1, node.parent.keys, pos, node.parent.nodeCount - pos);
+ Array.Copy(node.parent.nodes, pos + 1, node.parent.nodes, pos, node.parent.nodeCount - pos);
+ node.parent.keys[node.parent.nodeCount] = default(TKey);
+ node.parent.nodes[node.parent.nodeCount] = null;
+
+ // Make sure parent (of the parent) keys link down correctly.
+ if (node.parent.nodeCount > 0)
+ EnsureParentKey(node.parent, pos);
+
+ // Delete the node from the next/prev linked list.
+ node.prev.next = node.next;
+ if (node.next != null)
+ node.next.prev = node.prev;
+
+ // Merge the parent node.
+ Merge(node.parent, ref root);
+ return;
+ }
+
+ if (node.next == null)
+ {
+ if (node.parent == null && node.nodeCount == 1 && node.nodes != null)
+ {
+ root = node.nodes[0];
+ root.parent = null;
+ }
+
+ return;
+ }
+
+ if (node.nodeCount >= node.keys.Length / 2)
+ return;
+
+ int count = node.next.nodeCount;
+ if (node.nodeCount + count > node.keys.Length)
+ count -= (node.nodeCount + count) / 2;
+
+ Move(node.next, 0, node, node.nodeCount, count);
+ Merge(node.next, ref root);
+ }
+
+ #endregion
+ }
+
+ abstract class KeyValueCollectionBase : ICollection
+ {
+ #region Fields
+
+ protected readonly BTreeDictionary tree;
+
+ #endregion
+
+ #region Construction
+
+ public KeyValueCollectionBase(BTreeDictionary tree)
+ {
+ this.tree = tree;
+ }
+
+ #endregion
+
+ #region Properties
+
+ public int Count
+ {
+ get
+ {
+ return tree.Count;
+ }
+ }
+
+ #endregion
+
+ #region Methods
+
+ public abstract bool Contains(T item);
+
+ public void CopyTo(T[] array, int arrayIndex)
+ {
+ foreach (var item in this)
+ array[arrayIndex++] = item;
+ }
+
+ public abstract IEnumerator GetEnumerator();
+
+ #endregion
+
+ #region ICollection<> members
+
+ void ICollection.Add(T item)
+ {
+ throw new NotSupportedException();
+ }
+
+ void ICollection.Clear()
+ {
+ throw new NotSupportedException();
+ }
+
+ bool ICollection.IsReadOnly
+ {
+ get
+ {
+ return true;
+ }
+ }
+
+ public bool Remove(T item)
+ {
+ throw new NotSupportedException();
+ }
+
+ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+ {
+ return this.GetEnumerator();
+ }
+
+ #endregion
+ }
+
+ sealed class ValueCollection : KeyValueCollectionBase, IList
+ {
+ #region Construction
+
+ public ValueCollection(BTreeDictionary tree)
+ : base(tree)
+ {
+ }
+
+ #endregion
+
+ #region Methods
+
+ public override bool Contains(TValue item)
+ {
+ return this.tree.Any(keyValue => object.Equals(item, keyValue.Value));
+ }
+
+ public override IEnumerator GetEnumerator()
+ {
+ return this.tree.Select(keyValue => keyValue.Value).GetEnumerator();
+ }
+
+ #endregion
+
+ int IList.IndexOf(TValue item)
+ {
+ throw new NotImplementedException();
+ }
+
+ void IList.Insert(int index, TValue item)
+ {
+ throw new NotImplementedException();
+ }
+
+ void IList.RemoveAt(int index)
+ {
+ throw new NotImplementedException();
+ }
+
+ public TValue this[int index]
+ {
+ get
+ {
+ return this.tree.At(index).Value;
+ }
+ set
+ {
+ throw new NotImplementedException();
+ }
+ }
+ }
+
+ sealed class KeyCollection : KeyValueCollectionBase, ISortedCollection, IList
+ {
+ #region Construction
+
+ public KeyCollection(BTreeDictionary tree)
+ : base(tree)
+ {
+ }
+
+ #endregion
+
+ #region Properties
+
+ public IComparer Comparer
+ {
+ get
+ {
+ return tree.KeyComparer;
+ }
+ }
+
+ #endregion
+
+ #region Methods
+
+ public int FirstIndexWhereGreaterThan(TKey value)
+ {
+ Node leaf;
+ int pos;
+ var found = Node.Find(tree.root, value, tree.KeyComparer, 0, out leaf, out pos);
+ int result = Node.GetRootIndex(leaf, pos);
+ if (found)
+ ++result;
+ return result;
+ }
+
+ public int LastIndexWhereLessThan(TKey value)
+ {
+ Node leaf;
+ int pos;
+ var found = Node.Find(tree.root, value, tree.KeyComparer, 0, out leaf, out pos);
+ int result = Node.GetRootIndex(leaf, pos);
+ if (found)
+ --result;
+ return result;
+ }
+
+ public TKey At(int index)
+ {
+ return this.tree.At(index).Key;
+ }
+
+ public override bool Contains(TKey item)
+ {
+ return tree.ContainsKey(item);
+ }
+
+ public override IEnumerator GetEnumerator()
+ {
+ return tree.Select(keyValue => keyValue.Key).GetEnumerator();
+ }
+
+ public IEnumerable WhereGreaterOrEqual(TKey lowerBound)
+ {
+ return tree.WhereGreaterOrEqual(lowerBound).Select(keyValue => keyValue.Key);
+ }
+
+ public IEnumerable WhereLessOrEqualBackwards(TKey upperBound)
+ {
+ return tree.WhereLessOrEqualBackwards(upperBound).Select(keyValue => keyValue.Key);
+ }
+
+ public IEnumerable ForwardFromIndex(int index)
+ {
+ return this.tree.ForwardFromIndex(index).Select(item => item.Key);
+ }
+
+ public IEnumerable BackwardFromIndex(int index)
+ {
+ return this.tree.BackwardFromIndex(index).Select(item => item.Key);
+ }
+
+ #endregion
+
+ #region IEnumerable members
+
+ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+ {
+ return this.GetEnumerator();
+ }
+
+ #endregion
+
+ #region ISortedCollection<> members
+
+ void ISortedCollection.RemoveAt(int index)
+ {
+ throw new NotSupportedException();
+ }
+
+ #endregion
+
+ int IList.IndexOf(TKey item)
+ {
+ throw new NotImplementedException();
+ }
+
+ void IList.Insert(int index, TKey item)
+ {
+ throw new NotImplementedException();
+ }
+
+ void IList.RemoveAt(int index)
+ {
+ throw new NotImplementedException();
+ }
+
+ public TKey this[int index]
+ {
+ get
+ {
+ return this.tree.At(index).Key;
+ }
+ set
+ {
+ throw new NotImplementedException();
+ }
+ }
+
+ }
+
+ #endregion
+
+ #region IDictionary<> members
+
+ ICollection IDictionary.Keys
+ {
+ get
+ {
+ return this.Keys;
+ }
+ }
+
+ #endregion
+
+ #region ICollection<> members
+
+ void ICollection>.Add(KeyValuePair item)
+ {
+ this.Add(item.Key, item.Value);
+ }
+
+ bool ICollection>.Contains(KeyValuePair item)
+ {
+ TValue value;
+ return this.TryGetValue(item.Key, out value) && object.Equals(item.Value, value);
+ }
+
+ bool ICollection>.IsReadOnly
+ {
+ get
+ {
+ return false;
+ }
+ }
+
+ bool ICollection>.Remove(KeyValuePair item)
+ {
+ TValue value;
+ if (this.TryGetValue(item.Key, out value) && object.Equals(item.Value, value))
+ {
+ this.Remove(item.Key);
+ return true;
+ }
+ return false;
+ }
+
+ #endregion
+
+ #region IEnumerable members
+
+ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+ {
+ return this.GetEnumerator();
+ }
+
+ #endregion
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/BigArray.cs b/BotSharp.MachineLearning/CRFLite/Utils/BigArray.cs
new file mode 100644
index 00000000..20c52f0e
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/BigArray.cs
@@ -0,0 +1,433 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+using System.Threading.Tasks;
+#endif
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ abstract public class BigArray : IList where T : IComparable
+ {
+ public const long sizePerBlock = 1024 * 1024 * 64; //(<<26bits)
+ public const int moveBit = 26;
+ public long size_;
+ public List arrList;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ private ParallelOptions parallelOption;
+ private LimitedConcurrencyLevelTaskScheduler lcts;
+#endif
+ public BigArray()
+ {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ parallelOption = new ParallelOptions();
+#endif
+ }
+
+ public int IndexOf(T item)
+ {
+ throw new NotImplementedException();
+ }
+
+ public void Insert(int index, T item)
+ {
+ throw new NotImplementedException();
+ }
+
+ public void RemoveAt(int index)
+ {
+ throw new NotImplementedException();
+ }
+
+ public T this[int i]
+ {
+ get
+ {
+ return this[(long)i];
+ }
+ set
+ {
+ this[(long)i] = value;
+ }
+ }
+
+ public abstract T this[long i]
+ {
+ get;
+ set;
+ }
+
+ public void Add(T item)
+ {
+ throw new NotImplementedException();
+ }
+
+ public void Clear()
+ {
+ foreach (T[] item in arrList)
+ {
+ Array.Clear(item, 0, item.Length);
+ }
+ }
+
+ public bool Contains(T item)
+ {
+ throw new NotImplementedException();
+ }
+
+ public void CopyTo(T[] array, int arrayIndex)
+ {
+ throw new NotImplementedException();
+ }
+
+ public int Count
+ {
+ get
+ {
+ return (int)LongLength;
+ }
+ }
+
+ public long LongLength
+ {
+ get
+ {
+ return size_;
+ }
+ }
+
+ public bool IsReadOnly
+ {
+ get { throw new NotImplementedException(); }
+ }
+
+ public bool Remove(T item)
+ {
+ throw new NotImplementedException();
+ }
+
+ public IEnumerator GetEnumerator()
+ {
+ throw new NotImplementedException();
+ }
+
+ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+ {
+ throw new NotImplementedException();
+ }
+
+
+ void swap(long pos1, long pos2)
+ {
+ int nBlock1 = (int)(pos1 >> moveBit);
+ int offset1 = (int)(pos1 & (sizePerBlock - 1));
+
+ int nBlock2 = (int)(pos2 >> moveBit);
+ int offset2 = (int)(pos2 & (sizePerBlock - 1));
+
+ T tmp = arrList[nBlock1][offset1];
+ arrList[nBlock1][offset1] = arrList[nBlock2][offset2];
+ arrList[nBlock2][offset2] = tmp;
+ }
+
+
+ private long med3(long a, long b, long c)
+ {
+ return this[a].CompareTo(this[b]) < 0 ? (this[b].CompareTo(this[c]) < 0 ? b : this[a].CompareTo(this[c]) < 0 ? c : a) : this[b].CompareTo(this[c]) > 0 ? b : this[a].CompareTo(this[c]) > 0 ? c : a;
+ }
+
+ private void vecswap(long a, long b, long n)
+ {
+#if NO_SUPPORT_PARALLEL_LIB
+ for (long i = 0;i < n;i++)
+#else
+ Parallel.For(0, n, parallelOption, i =>
+#endif
+ {
+ int nBlock1 = (int)((a + i) >> moveBit);
+ int offset1 = (int)((a + i) & (sizePerBlock - 1));
+
+ int nBlock2 = (int)((b + i) >> moveBit);
+ int offset2 = (int)((b + i) & (sizePerBlock - 1));
+
+ T tmp = arrList[nBlock1][offset1];
+ arrList[nBlock1][offset1] = arrList[nBlock2][offset2];
+ arrList[nBlock2][offset2] = tmp;
+ }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ );
+#endif
+ }
+
+ const int INSERT_SORT_THRESHOLD = 7;
+ public void QuickSort(long left, long right)
+ {
+ if (left >= right)
+ {
+ return;
+ }
+
+ //use insert sort to handle small data
+ long len = right - left + 1;
+ if (len < INSERT_SORT_THRESHOLD)
+ {
+ for (long i = left; i <= right; i++)
+ {
+ T t = this[i];
+ long j = i;
+ for (; j > left && this[j - 1].CompareTo(t) > 0; j--)
+ {
+ this[j] = this[j - 1];
+ }
+ this[j] = t;
+ }
+ return;
+ }
+
+ //Choose the pivot value
+ long mid = left + (len >> 1);
+ if (len > INSERT_SORT_THRESHOLD)
+ {
+ //Split the list into three parts, and find middle value in each part,
+ //and finally, use middle value in above three middle values as pivot.
+ long leftMid = left;
+ long rightMid = right;
+ if (len > 40)
+ {
+ long size = len / 8;
+ leftMid = med3(leftMid, leftMid + size, leftMid + 2 * size);
+ mid = med3(mid - size, mid, mid + size);
+ rightMid = med3(right - 2 * size, right - size, right);
+ }
+ mid = med3(leftMid, mid, rightMid);
+ }
+
+ T v = this[mid];
+
+ //Scan the list from two directions
+ long pivotLeftSide = left, leftScanIndex = pivotLeftSide;
+ long rightScanIndex = right, pivotRightSide = rightScanIndex;
+
+
+ int leftScanIndexBlock = (int)(leftScanIndex >> moveBit);
+ int leftScanIndexOffset = (int)(leftScanIndex & (sizePerBlock - 1));
+ T[] arrayLeft = arrList[leftScanIndexBlock];
+
+ int rightScanIndexBlock = (int)(rightScanIndex >> moveBit);
+ int rightScanIndexOffset = (int)(rightScanIndex & (sizePerBlock - 1));
+ T[] arrayRight = arrList[rightScanIndexBlock];
+ while (true)
+ {
+ //Try to find item which is bigger than pivot
+ while (leftScanIndex <= rightScanIndex)
+ {
+ int cmpRst = arrayLeft[leftScanIndexOffset].CompareTo(v);
+ if (cmpRst > 0)
+ {
+ //Found one.
+ break;
+ }
+ else if (cmpRst == 0)
+ {
+ //If the item is equal to pivot, exchange it with the item in left-side.
+ swap(pivotLeftSide++, leftScanIndex);
+ }
+ leftScanIndex++;
+
+ leftScanIndexOffset++;
+ if (leftScanIndexOffset == sizePerBlock)
+ {
+ leftScanIndexOffset = 0;
+ leftScanIndexBlock++;
+ if (leftScanIndexBlock == arrList.Count)
+ {
+ break;
+ }
+ arrayLeft = arrList[leftScanIndexBlock];
+ }
+ }
+
+ //Try to find item which is smaller than pivot
+ while (rightScanIndex >= leftScanIndex)
+ {
+ int cmpRst = arrayRight[rightScanIndexOffset].CompareTo(v);
+ if (cmpRst < 0)
+ {
+ //Found one.
+ break;
+ }
+ else if (cmpRst == 0)
+ {
+ //If the item is equal to pivot, exchange it with the item in left-side.
+ swap(rightScanIndex, pivotRightSide--);
+ }
+ rightScanIndex--;
+
+ rightScanIndexOffset--;
+ if (rightScanIndexOffset < 0)
+ {
+ rightScanIndexOffset = (int)(sizePerBlock - 1);
+ rightScanIndexBlock--;
+ if (rightScanIndexBlock < 0)
+ {
+ break;
+ }
+ arrayRight = arrList[rightScanIndexBlock];
+ }
+ }
+
+ if (leftScanIndex > rightScanIndex)
+ {
+ //Scan finished
+ break;
+ }
+
+ //Exchange two found items between pivot
+ T temp = arrayLeft[(int)leftScanIndexOffset];
+ arrayLeft[(int)leftScanIndexOffset] = arrayRight[(int)rightScanIndexOffset];
+ arrayRight[(int)rightScanIndexOffset] = temp;
+
+ leftScanIndex++;
+ rightScanIndex--;
+
+ leftScanIndexOffset++;
+ if (leftScanIndexOffset == sizePerBlock)
+ {
+ leftScanIndexOffset = 0;
+ leftScanIndexBlock++;
+ if (leftScanIndexBlock == arrList.Count)
+ {
+ break;
+ }
+ arrayLeft = arrList[leftScanIndexBlock];
+ }
+
+ rightScanIndexOffset--;
+ if (rightScanIndexOffset < 0)
+ {
+ rightScanIndexOffset = (int)(sizePerBlock - 1);
+ rightScanIndexBlock--;
+ if (rightScanIndexBlock < 0)
+ {
+ break;
+ }
+ arrayRight = arrList[rightScanIndexBlock];
+ }
+ }
+
+ //Continue to sort two sub-sections
+ long splitIndexLeft = leftScanIndex - pivotLeftSide;
+ long splitIndexRight = pivotRightSide - rightScanIndex;
+ if (splitIndexLeft > 1 && splitIndexRight > 1)
+ {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ Parallel.Invoke(parallelOption,
+ () =>
+#endif
+ {
+ //exchange items with same value into middle of the list
+ long size = Math.Min(pivotLeftSide - left, leftScanIndex - pivotLeftSide);
+ vecswap(left, leftScanIndex - size, size);
+ QuickSort(left, splitIndexLeft + left - 1);
+ }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ ,
+ () =>
+#endif
+ {
+ //exchange items with same value into middle of the list
+ long size = Math.Min(pivotRightSide - rightScanIndex, right - pivotRightSide);
+ vecswap(leftScanIndex, right - size + 1, size);
+ QuickSort(right - splitIndexRight + 1, right);
+ }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ );
+#endif
+ }
+ else
+ {
+ //exchange items with same value into middle of the list
+ long size = Math.Min(pivotLeftSide - left, leftScanIndex - pivotLeftSide);
+ vecswap(left, leftScanIndex - size, size);
+
+ size = Math.Min(pivotRightSide - rightScanIndex, right - pivotRightSide);
+ vecswap(leftScanIndex, right - size + 1, size);
+
+ if (splitIndexLeft > 1)
+ {
+ QuickSort(left, splitIndexLeft + left - 1);
+ }
+
+ if (splitIndexRight > 1)
+ {
+ QuickSort(right - splitIndexRight + 1, right);
+ }
+ }
+ }
+
+
+
+ public void Sort(long startIndex, long size, int threadnum = -1)
+ {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ parallelOption.MaxDegreeOfParallelism = threadnum;
+ if (threadnum > 0)
+ {
+ lcts = new LimitedConcurrencyLevelTaskScheduler(threadnum * 2);
+ parallelOption.TaskScheduler = lcts;
+ }
+#endif
+ QuickSort(startIndex, startIndex + size - 1);
+ }
+
+ public void Sort(int threadnum = -1)
+ {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ parallelOption.MaxDegreeOfParallelism = threadnum;
+ if (threadnum > 0)
+ {
+ lcts = new LimitedConcurrencyLevelTaskScheduler(threadnum * 2);
+ parallelOption.TaskScheduler = lcts;
+ }
+#endif
+ QuickSort(0, Count - 1);
+ }
+
+
+ public T BinarySearch(long low, long high, T goal)
+ {
+ long mid = 0;
+
+ while (low <= high)
+ {
+ mid = (high + low) / 2;
+ if (this[mid].CompareTo(goal) == 0)
+ {
+ return this[mid];
+ }
+ else if (this[mid].CompareTo(goal) > 0)
+ {
+ high = mid - 1;
+ }
+ else
+ {
+ low = mid + 1;
+ }
+ }
+ return default(T);
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/DoubleArrayTrie.cs b/BotSharp.MachineLearning/CRFLite/Utils/DoubleArrayTrie.cs
new file mode 100644
index 00000000..51a9487c
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/DoubleArrayTrie.cs
@@ -0,0 +1,631 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using System.Threading;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+using System.Threading.Tasks;
+#endif
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ public class unit_t : IComparable
+ {
+ public int base1;
+ public int check;
+
+ public int CompareTo(unit_t obj)
+ {
+ throw new NotImplementedException();
+ }
+ }
+
+ public struct sunit_t : IComparable
+ {
+ public int base1;
+ public int check;
+
+ public sunit_t(int b, int c)
+ {
+ base1 = b;
+ check = c;
+ }
+
+ public int CompareTo(sunit_t obj)
+ {
+ throw new NotImplementedException();
+ }
+ }
+
+ public class DoubleArrayTrieSearch
+ {
+#if NO_SUPPORT_VERY_BIG_OBJECT
+ private VarBigArray array;
+#else
+ private sunit_t[] array;
+#endif
+
+ ///
+ /// Loads ArrayTrie from file
+ ///
+ /// path to file
+ ///
+ /// Number of elements (2 int32) in read buffer.
+ /// Default is 2048 (16K buffer size)
+ ///
+ public void Load(string fileName, int numberOfElementsInChunk = 2048)
+ {
+ if(!File.Exists(fileName))
+ throw new FileNotFoundException(
+ "Please check that the specified file exists", fileName);
+ using (var stream = File.OpenRead(fileName))
+ Load(stream, numberOfElementsInChunk);
+ }
+
+ ///
+ /// Loads ArrayTrie from an arbitrary .
+ /// is closed and
+ /// disposed once this method completes.
+ ///
+ ///
+ /// A containing the model.
+ ///
+ ///
+ /// Number of elements (2 int32) in read buffer.
+ /// Default is 2048 (16K buffer size)
+ public void Load(Stream sourceStream, int numberOfElementsInChunk = 2048)
+ {
+ const int int32Size = sizeof(int);
+ const int elementSize = int32Size * 2;
+ var fileSizeInBytes = sourceStream.Length;
+ var numberOfElements = fileSizeInBytes / elementSize;
+#if NO_SUPPORT_VERY_BIG_OBJECT
+ array = new VarBigArray(numberOfElements);
+#else
+ array = new sunit_t[numberOfElements];
+#endif
+ using (var sr = new StreamReader(sourceStream))
+ using (var br = new BinaryReader(sr.BaseStream))
+ {
+ var buffersize = elementSize * numberOfElementsInChunk;
+ var buffer = new byte[elementSize * numberOfElementsInChunk];
+ var index = 0;
+ for (long j = 0; j <= numberOfElements / numberOfElementsInChunk; j++)
+ {
+ var numberOfReadBytes = br.Read(buffer, 0, buffersize);
+ if (numberOfReadBytes == buffersize)
+ {
+ for (int i = 0; i < numberOfElementsInChunk; i++, index++)
+ {
+ var base1 = BitConverter.ToInt32(buffer, elementSize * i);
+ var check = BitConverter.ToInt32(buffer, (elementSize * i) + int32Size);
+ array[index] = new sunit_t(base1, check);
+ }
+ }
+ else
+ {
+ for (int i = 0; i < numberOfReadBytes / elementSize; i++)
+ {
+ var base1 = BitConverter.ToInt32(buffer, elementSize * i);
+ var check = BitConverter.ToInt32(buffer, (elementSize * i) + int32Size);
+ array[index++] = new sunit_t(base1, check);
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ //Match indexed key which is perfect matched with given string
+ public int SearchByPerfectMatch(string key)
+ {
+ int b = array[0].base1;
+ int p;
+ for (int index = 0; index < key.Length; index++)
+ {
+ char ch = key[index];
+ p = b + ch + 1;
+ if (p >= array.Length)
+ {
+ return -1;
+ }
+
+ if (b == array[p].check)
+ {
+ b = array[p].base1;
+ }
+ else
+ {
+ return -1;
+ }
+ }
+
+ if (b >= array.Length)
+ {
+ return -1;
+ }
+
+ int n = array[b].base1;
+ if (b == array[b].check && n < 0)
+ {
+ return -n - 1;
+ }
+ return -1;
+ }
+
+ ///
+ /// Match all indexed keys which is prefix of the given string.
+ ///
+ public int SearchAsKeyPrefix(string key, List result)
+ {
+ int len = key.Length;
+ int b = array[0].base1;
+ int n, p;
+ result.Clear();
+
+ for (int i = 0; i < len; i++)
+ {
+ p = b;
+ if (p >= array.Length)
+ {
+ return result.Count;
+ }
+
+ n = array[p].base1;
+
+ if (b == array[p].check && n < 0)
+ {
+ result.Add(-n - 1);
+ }
+
+ p = b + (int)key[i] + 1;
+ if (p >= array.Length)
+ {
+ return result.Count;
+ }
+
+ if (b == array[p].check)
+ {
+ b = array[p].base1;
+ }
+ else
+ {
+ return result.Count;
+ }
+ }
+
+ p = b;
+ if (p >= array.Length)
+ {
+ return result.Count;
+ }
+ n = array[p].base1;
+
+ if (b == array[p].check && n < 0)
+ {
+ result.Add(-n - 1);
+ }
+
+ return result.Count;
+ }
+
+
+ ///
+ /// Search keys by their prefix string
+ ///
+ public int SearchByPrefix(string strKeyPrefx, List result)
+ {
+ int b = array[0].base1;
+ int p;
+
+ result.Clear();
+ for (int index = 0; index < strKeyPrefx.Length; index++)
+ {
+ var ch = strKeyPrefx[index];
+ p = b + (int)ch + 1;
+ if (p >= array.Length)
+ {
+ //The given string isn't existed in the DART
+ return -1;
+ }
+
+ if (b == array[p].check)
+ {
+ b = array[p].base1;
+ }
+ else
+ {
+ return -1;
+ }
+ }
+
+ Queue queue = new Queue();
+ queue.Enqueue(b);
+
+ while (queue.Count > 0)
+ {
+ b = queue.Dequeue();
+ if (b >= array.Length)
+ {
+ //Invalidated base, skip it
+ continue;
+ }
+
+ if (b == array[b].check && array[b].base1 < 0)
+ {
+ result.Add(-array[b].base1 - 1);
+ }
+
+ for (int i = 0; i <= 65535; i++)
+ {
+ p = b + i + 1;
+ if (p >= array.Length)
+ {
+ //Out of the size of array, skip current search
+ break;
+ }
+
+ if (b == array[p].check)
+ {
+ queue.Enqueue(array[p].base1);
+ }
+ }
+ }
+
+ return result.Count;
+ }
+ }
+
+ public class DoubleArrayTrieBuilder
+ {
+ private VarBigArray array;
+ private VarBigArray used;
+ private IList key_;
+ private IList val_;
+ private int next_chk_pos_;
+ private int progress_;
+ private int thread_num_;
+ private static double MAX_SLOT_USAGE_RATE_THRESHOLD = 0.95;
+ private static double MIN_SLOT_USAGE_RATE_THRESHOLD = 0.05;
+ private double slot_usage_rate_threshold_ = MAX_SLOT_USAGE_RATE_THRESHOLD;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ private ParallelOptions parallelOption;
+ private LimitedConcurrencyLevelTaskScheduler lcts;
+#endif
+
+ private DateTime startDT;
+ private double lastQPS;
+ private double lastQPSDelta;
+
+ public class Node
+ {
+ public int code;
+ public int depth;
+ public int left;
+ public int right;
+ };
+
+ public DoubleArrayTrieBuilder(int thread_num)
+ {
+ array = null;
+ thread_num_ = thread_num;
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+ lcts = new LimitedConcurrencyLevelTaskScheduler(thread_num_ * 2);
+ parallelOption = new ParallelOptions();
+ parallelOption.TaskScheduler = lcts;
+#endif
+ lastQPS = 0.0;
+ lastQPSDelta = 0.0;
+ }
+
+ int fetch(Node parent, List siblings)
+ {
+ int prev = 0;
+
+ int i = parent.left;
+ for (int j = parent.left; j < parent.right; j++)
+ {
+ string key = key_[j];
+ if (key.Length < parent.depth)
+ continue;
+ int cur = 0;
+ if (key.Length != parent.depth)
+ {
+ cur = ((int)key[parent.depth]) + 1;
+ }
+ if (prev > cur)
+ {
+ throw new Exception("Fatal: given strings are not sorted.\n");
+ }
+ if (cur != prev || siblings.Count == 0)
+ {
+ Node tmp_node = new Node();
+ tmp_node.depth = parent.depth + 1;
+ tmp_node.code = cur;
+ tmp_node.left = i;
+ if (siblings.Count != 0)
+ siblings[siblings.Count - 1].right = i;
+ siblings.Add(tmp_node);
+ }
+ prev = cur;
+ i++;
+ }
+ if (siblings.Count != 0)
+ siblings[siblings.Count - 1].right = parent.right;
+ return siblings.Count;
+ }
+
+ int insert(List siblings)
+ {
+ Random rnd = new Random(DateTime.Now.Millisecond + Thread.CurrentThread.ManagedThreadId);
+ int begin = 0;
+ bool cont = true;
+ int nonzeronum = 0;
+
+ while (used[next_chk_pos_] == 1)
+ {
+ Interlocked.Increment(ref next_chk_pos_);
+ }
+
+ int pos = next_chk_pos_;
+ int startpos = pos;
+
+ //search begin position
+ pos--;
+ while (cont == true)
+ {
+ pos++;
+ if (used[pos] == 0)
+ {
+ //Check whether slots are available, if not go on to search,
+ cont = false;
+ foreach (Node n in siblings)
+ {
+ if (used[pos + n.code] == 1 || array[pos + n.code] != null)
+ {
+ cont = true;
+ break;
+ }
+ }
+ }
+ else
+ {
+ nonzeronum++;
+ }
+ }
+ begin = pos;
+
+ //check average slot usage rate. If the rate is no less than the threshold, update next_chk_pos_ to
+ //pos whose slot range has much less conflict.
+ //note that, the higher rate threshold, the higher slot space usage rate, however, the timing-cost for tri-tree build
+ //will also become more higher.
+ if ((double)nonzeronum / (double)(pos - startpos + 1) >= slot_usage_rate_threshold_ &&
+ pos > next_chk_pos_)
+ {
+ System.Threading.Interlocked.Exchange(ref next_chk_pos_, pos);
+ }
+
+ //double check whether slots are available
+ //the reason why double check is because:
+ //1. in entire slots space, conflict rate is different. the conflict rate of array's tail
+ // is much lower than that of its header and body
+ //2. roll back cost is heavy. So in high conflict rate range, we just check conflict and no other action (first check)
+ // once we find a availabe range without conflict, we try to allocate memory on this range and double check conflict
+ bool bAllNull;
+ bool bZeroCode = false;
+ foreach (Node n in siblings)
+ {
+ if (n.code == 0)
+ {
+ bZeroCode = true;
+ break;
+ }
+ }
+
+ if (bZeroCode == false)
+ {
+ Node sNode = new Node();
+ sNode.code = 0;
+ siblings.Add(sNode);
+ }
+
+ do
+ {
+ bAllNull = true;
+ //Test conflict in multi-threads
+ int cnt = 0;
+ foreach (Node n in siblings)
+ {
+ int nBlock = (begin + n.code) >> VarBigArray.moveBit;
+ long offset = (begin + n.code) & (VarBigArray.sizePerBlock - 1);
+
+ if (used[begin + n.code] == 1 ||
+ System.Threading.Interlocked.CompareExchange(ref used.arrList[nBlock][offset], 1, 0) != 0)
+ {
+ bAllNull = false;
+ foreach (Node revertNode in siblings.GetRange(0, cnt))
+ {
+ used[begin + revertNode.code] = 0;
+ }
+ begin += rnd.Next(thread_num_) + 1;
+ break;
+ }
+ cnt++;
+ }
+ } while (bAllNull == false);
+
+ if (bZeroCode == false)
+ {
+ siblings.RemoveAt(siblings.Count - 1);
+ }
+
+#if NO_SUPPORT_PARALLEL_LIB
+ for (int i = 0;i < siblings.Count;i++)
+#else
+ Parallel.For(0, siblings.Count, parallelOption, i =>
+#endif
+ {
+ List new_siblings = new List();
+ Node sibling = siblings[i];
+ int offset = begin + sibling.code;
+
+ array[offset] = new unit_t();
+ array[offset].check = begin;
+ if (fetch(sibling, new_siblings) == 0)
+ {
+ array[offset].base1 = -val_[sibling.left] - 1;
+ if (Interlocked.Increment(ref progress_) % 10000 == 0)
+ {
+ //Try to adjust slot usage rate in order to keep high performance
+ TimeSpan ts = DateTime.Now - startDT;
+ double currQPS = progress_ / (ts.TotalSeconds + 1);
+ double currQPSDelta = currQPS - lastQPS;
+
+ if (currQPS < lastQPS && currQPSDelta < lastQPSDelta)
+ {
+ //Average QPS becomes slow down, need to reduce slot usage rate
+ slot_usage_rate_threshold_ -= 0.1;
+ if (slot_usage_rate_threshold_ < MIN_SLOT_USAGE_RATE_THRESHOLD)
+ {
+ slot_usage_rate_threshold_ = MIN_SLOT_USAGE_RATE_THRESHOLD;
+ }
+ }
+ else
+ {
+ //Average QPS becomes fast, need to add slot usage rate
+ slot_usage_rate_threshold_ += 0.1;
+ if (slot_usage_rate_threshold_ > MAX_SLOT_USAGE_RATE_THRESHOLD)
+ {
+ slot_usage_rate_threshold_ = MAX_SLOT_USAGE_RATE_THRESHOLD;
+ }
+ }
+
+ lastQPSDelta = currQPSDelta;
+ lastQPS = currQPS;
+
+ if (progress_ % 100000 == 0)
+ {
+ //Show current progress on console
+ Console.Write("{0}...", progress_);
+ }
+ }
+ }
+ else
+ {
+ int b = insert(new_siblings);
+ array[offset].base1 = b;
+ }
+ }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+);
+#endif
+
+ return begin;
+ }
+
+ void clear()
+ {
+ array = null;
+ }
+
+ public bool build(IDictionary keyvalueList, double max_slot_usage_rate_threshold = 0.95)
+ {
+ FixedBigArray keyList = new FixedBigArray(keyvalueList.Count, 0);
+ FixedBigArray valList = new FixedBigArray(keyvalueList.Count, 0);
+ long index = 0;
+ foreach (KeyValuePair pair in keyvalueList)
+ {
+ keyList[index] = pair.Key;
+ valList[index] = pair.Value;
+ index++;
+ }
+
+ return build(keyList, valList, max_slot_usage_rate_threshold);
+ }
+
+ public bool build(IList keyList, IList valList, double max_slot_usage_rate_threshold = 0.95)
+ {
+ if (keyList == null)
+ {
+ Console.WriteLine("Key list is empty");
+ return false;
+ }
+ if (valList == null)
+ {
+ Console.WriteLine("Value list is empty");
+ return false;
+ }
+
+ if (keyList.Count != valList.Count)
+ {
+ Console.WriteLine("The size of key list and value list is not equal");
+ return false;
+ }
+
+ for (int i = 0; i < valList.Count; i++)
+ {
+ if (valList[i] <= -1)
+ {
+ Console.WriteLine("Invalidated value {0} at index {1}", valList[i], i);
+ return false;
+ }
+ }
+
+ MAX_SLOT_USAGE_RATE_THRESHOLD = max_slot_usage_rate_threshold;
+ slot_usage_rate_threshold_ = max_slot_usage_rate_threshold;
+ progress_ = 0;
+ key_ = keyList;
+ val_ = valList;
+
+ startDT = DateTime.Now;
+ array = new VarBigArray(key_.Count * 5);
+ used = new VarBigArray(key_.Count * 5);
+ array[0] = new unit_t();
+ array[0].base1 = 1;
+ used[0] = 1;
+ next_chk_pos_ = 0;
+ Node root_node = new Node();
+ root_node.left = 0;
+ root_node.right = key_.Count;
+ root_node.depth = 0;
+ List siblings = new List();
+ fetch(root_node, siblings);
+ insert(siblings);
+
+ return true;
+ }
+
+ public void save(string file)
+ {
+ StreamWriter sw = new StreamWriter(file);
+ BinaryWriter bw = new BinaryWriter(sw.BaseStream);
+
+ long r_length = array.LongLength;
+ while (array[r_length - 1] == null)
+ {
+ r_length--;
+ }
+
+ for (long i = 0; i < r_length; i++)
+ {
+ if (array[i] == null)
+ {
+ bw.Write(0);
+ bw.Write(0);
+ }
+ else
+ {
+ bw.Write(array[i].base1);
+ bw.Write(array[i].check);
+ }
+ }
+ bw.Close();
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/FixedBigArray.cs b/BotSharp.MachineLearning/CRFLite/Utils/FixedBigArray.cs
new file mode 100644
index 00000000..1b8a6305
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/FixedBigArray.cs
@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ public sealed class FixedBigArray : BigArray where T : IComparable
+ {
+ public int lowBounding_;
+
+ public override T this[long i]
+ {
+ get
+ {
+ long offset = (i - lowBounding_);
+ int nBlock = (int)(offset >> moveBit);
+ return arrList[nBlock][offset & (sizePerBlock - 1)];
+ }
+ set
+ {
+ long offset = (i - lowBounding_);
+ int nBlock = (int)(offset >> moveBit);
+ arrList[nBlock][offset & (sizePerBlock - 1)] = value;
+ }
+ }
+
+
+
+ //construct big array
+ //size is array's default length
+ //lowBounding is the lowest bounding of the array
+ public FixedBigArray(long size, int lowBounding)
+ {
+ size_ = size;
+ lowBounding_ = lowBounding;
+ arrList = new List();
+
+ for (long i = 0; i < size_; i += sizePerBlock)
+ {
+ if (i + sizePerBlock < size_)
+ {
+ arrList.Add(new T[sizePerBlock]);
+ }
+ else
+ {
+ arrList.Add(new T[size_ - i]);
+ }
+ }
+ }
+
+
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/LimitedConcurrencyLevelTaskScheduler.cs b/BotSharp.MachineLearning/CRFLite/Utils/LimitedConcurrencyLevelTaskScheduler.cs
new file mode 100644
index 00000000..2da789c4
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/LimitedConcurrencyLevelTaskScheduler.cs
@@ -0,0 +1,140 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+using System.Threading.Tasks;
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ ///
+ /// Provides a task scheduler that ensures a maximum concurrency level while
+ /// running on top of the ThreadPool.
+ ///
+ public class LimitedConcurrencyLevelTaskScheduler : TaskScheduler
+ {
+ /// Whether the current thread is processing work items.
+ [ThreadStatic]
+ private static bool _currentThreadIsProcessingItems;
+ /// The list of tasks to be executed.
+ private readonly LinkedList _tasks = new LinkedList(); // protected by lock(_tasks)
+ /// The maximum concurrency level allowed by this scheduler.
+ private readonly int _maxDegreeOfParallelism;
+ /// Whether the scheduler is currently processing work items.
+ private int _delegatesQueuedOrRunning = 0; // protected by lock(_tasks)
+
+ ///
+ /// Initializes an instance of the LimitedConcurrencyLevelTaskScheduler class with the
+ /// specified degree of parallelism.
+ ///
+ /// The maximum degree of parallelism provided by this scheduler.
+ public LimitedConcurrencyLevelTaskScheduler(int maxDegreeOfParallelism)
+ {
+ if (maxDegreeOfParallelism < 1) throw new ArgumentOutOfRangeException("maxDegreeOfParallelism");
+ _maxDegreeOfParallelism = maxDegreeOfParallelism;
+ }
+
+ /// Queues a task to the scheduler.
+ /// The task to be queued.
+ protected sealed override void QueueTask(Task task)
+ {
+ // Add the task to the list of tasks to be processed. If there aren't enough
+ // delegates currently queued or running to process tasks, schedule another.
+ lock (_tasks)
+ {
+ _tasks.AddLast(task);
+ if (_delegatesQueuedOrRunning < _maxDegreeOfParallelism)
+ {
+ ++_delegatesQueuedOrRunning;
+ NotifyThreadPoolOfPendingWork();
+ }
+ }
+ }
+
+ ///
+ /// Informs the ThreadPool that there's work to be executed for this scheduler.
+ ///
+ private void NotifyThreadPoolOfPendingWork()
+ {
+ ThreadPool.UnsafeQueueUserWorkItem(_ =>
+ {
+ // Note that the current thread is now processing work items.
+ // This is necessary to enable inlining of tasks into this thread.
+ _currentThreadIsProcessingItems = true;
+ try
+ {
+ // Process all available items in the queue.
+ while (true)
+ {
+ Task item;
+ lock (_tasks)
+ {
+ // When there are no more items to be processed,
+ // note that we're done processing, and get out.
+ if (_tasks.Count == 0)
+ {
+ --_delegatesQueuedOrRunning;
+ break;
+ }
+
+ // Get the next item from the queue
+ item = _tasks.First.Value;
+ _tasks.RemoveFirst();
+ }
+
+ // Execute the task we pulled out of the queue
+ base.TryExecuteTask(item);
+ }
+ }
+ // We're done processing items on the current thread
+ finally { _currentThreadIsProcessingItems = false; }
+ }, null);
+ }
+
+ /// Attempts to execute the specified task on the current thread.
+ /// The task to be executed.
+ ///
+ /// Whether the task could be executed on the current thread.
+ protected sealed override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued)
+ {
+ // If this thread isn't already processing a task, we don't support inlining
+ if (!_currentThreadIsProcessingItems) return false;
+
+ // If the task was previously queued, remove it from the queue
+ if (taskWasPreviouslyQueued) TryDequeue(task);
+
+ // Try to run the task.
+ return base.TryExecuteTask(task);
+ }
+
+ /// Attempts to remove a previously scheduled task from the scheduler.
+ /// The task to be removed.
+ /// Whether the task could be found and removed.
+ protected sealed override bool TryDequeue(Task task)
+ {
+ lock (_tasks) return _tasks.Remove(task);
+ }
+
+ /// Gets the maximum concurrency level supported by this scheduler.
+ public sealed override int MaximumConcurrencyLevel { get { return _maxDegreeOfParallelism; } }
+
+ /// Gets an enumerable of the tasks currently scheduled on this scheduler.
+ /// An enumerable of the tasks currently scheduled.
+ protected sealed override IEnumerable GetScheduledTasks()
+ {
+ bool lockTaken = false;
+ try
+ {
+ Monitor.TryEnter(_tasks, ref lockTaken);
+ if (lockTaken) return _tasks.ToArray();
+ else throw new NotSupportedException();
+ }
+ finally
+ {
+ if (lockTaken) Monitor.Exit(_tasks);
+ }
+ }
+ }
+}
+#endif
\ No newline at end of file
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/MD5.cs b/BotSharp.MachineLearning/CRFLite/Utils/MD5.cs
new file mode 100644
index 00000000..3d334e51
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/MD5.cs
@@ -0,0 +1,554 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ public sealed class MD5
+ {
+ private const int BLOCK_SIZE_BYTES = 64;
+ private const int HASH_SIZE_BYTES = 16;
+
+ private uint _state0;
+ private uint _state1;
+ private uint _state2;
+ private uint _state3;
+
+ private uint _decodeBuf0;
+ private uint _decodeBuf1;
+ private uint _decodeBuf2;
+ private uint _decodeBuf3;
+ private uint _decodeBuf4;
+ private uint _decodeBuf5;
+ private uint _decodeBuf6;
+ private uint _decodeBuf7;
+ private uint _decodeBuf8;
+ private uint _decodeBuf9;
+ private uint _decodeBuf10;
+ private uint _decodeBuf11;
+ private uint _decodeBuf12;
+ private uint _decodeBuf13;
+ private uint _decodeBuf14;
+ private uint _decodeBuf15;
+
+ private ulong count;
+ private byte[] _ProcessingBuffer; // Used to start data when passed less than a block worth.
+ private int _ProcessingBufferCount; // Counts how much data we have stored that still needs processed.
+ private byte[] hash;
+ private byte[] fooBuffer;
+
+ public MD5()
+ {
+ fooBuffer = new byte[BLOCK_SIZE_BYTES * 4096];
+ hash = new byte[16];
+ _ProcessingBuffer = new byte[BLOCK_SIZE_BYTES];
+
+ Initialize();
+ }
+
+ ~MD5()
+ {
+ Dispose(false);
+ }
+
+ private void Dispose(bool disposing)
+ {
+ if (_ProcessingBuffer != null)
+ {
+ Array.Clear(_ProcessingBuffer, 0, _ProcessingBuffer.Length);
+ _ProcessingBuffer = null;
+ }
+ }
+
+ private void HashCore(byte[] rgb, int start, int size)
+ {
+ int i;
+ if (_ProcessingBufferCount != 0)
+ {
+ if (size < (BLOCK_SIZE_BYTES - _ProcessingBufferCount))
+ {
+ System.Buffer.BlockCopy(rgb, start, _ProcessingBuffer, _ProcessingBufferCount, size);
+ _ProcessingBufferCount += size;
+ return;
+ }
+ else
+ {
+ i = (BLOCK_SIZE_BYTES - _ProcessingBufferCount);
+ System.Buffer.BlockCopy(rgb, start, _ProcessingBuffer, _ProcessingBufferCount, i);
+ ProcessBlock(_ProcessingBuffer, 0);
+ _ProcessingBufferCount = 0;
+ start += i;
+ size -= i;
+ }
+ }
+
+ for (i = 0; i < size - size % BLOCK_SIZE_BYTES; i += BLOCK_SIZE_BYTES)
+ {
+ ProcessBlock(rgb, start + i);
+ }
+
+ if (size % BLOCK_SIZE_BYTES != 0)
+ {
+ System.Buffer.BlockCopy(rgb, size - size % BLOCK_SIZE_BYTES + start, _ProcessingBuffer, 0, size % BLOCK_SIZE_BYTES);
+ _ProcessingBufferCount = size % BLOCK_SIZE_BYTES;
+ }
+ }
+
+ public long Compute64BitHash(byte[] buffer)
+ {
+ HashCore(buffer, 0, buffer.Length);
+
+ ProcessFinalBlock(_ProcessingBuffer, 0, _ProcessingBufferCount);
+ long longRst = (((long)_state1 << 32) | (long)_state0);
+ this.Initialize();
+
+ return longRst;
+ }
+
+ public byte[] ComputeHash(byte [] buffer)
+ {
+ HashCore(buffer, 0, buffer.Length);
+
+ ProcessFinalBlock(_ProcessingBuffer, 0, _ProcessingBufferCount);
+
+ hash[0] = (byte)(_state0);
+ hash[1] = (byte)(_state0 >> 8);
+ hash[2] = (byte)(_state0 >> 16);
+ hash[3] = (byte)(_state0 >> 24);
+ hash[4] = (byte)(_state1);
+ hash[5] = (byte)(_state1 >> 8);
+ hash[6] = (byte)(_state1 >> 16);
+ hash[7] = (byte)(_state1 >> 24);
+ hash[8] = (byte)(_state2);
+ hash[9] = (byte)(_state2 >> 8);
+ hash[10] = (byte)(_state2 >> 16);
+ hash[11] = (byte)(_state2 >> 24);
+ hash[12] = (byte)(_state3);
+ hash[13] = (byte)(_state3 >> 8);
+ hash[14] = (byte)(_state3 >> 16);
+ hash[15] = (byte)(_state3 >> 24);
+
+ return hash;
+ }
+
+ public void Initialize()
+ {
+ count = 0;
+ _ProcessingBufferCount = 0;
+
+ _state0 = 0x67452301;
+ _state1 = 0xefcdab89;
+ _state2 = 0x98badcfe;
+ _state3 = 0x10325476;
+ }
+
+ private void ProcessBlock(byte[] inputBuffer, int inputOffset)
+ {
+ uint a, b, c, d;
+
+ count += BLOCK_SIZE_BYTES;
+
+ _decodeBuf0 = ((uint)(inputBuffer[inputOffset] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 1] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 2] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 3]) << 24);
+
+ _decodeBuf1 = ((uint)(inputBuffer[inputOffset + 4] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 5] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 6] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 7]) << 24);
+
+ _decodeBuf2 = ((uint)(inputBuffer[inputOffset + 8] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 9] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 10] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 11]) << 24);
+
+ _decodeBuf3 = ((uint)(inputBuffer[inputOffset + 12] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 13] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 14] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 15]) << 24);
+
+ _decodeBuf4 = ((uint)(inputBuffer[inputOffset + 16] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 17] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 18] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 19]) << 24);
+
+ _decodeBuf5 = ((uint)(inputBuffer[inputOffset + 20] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 21] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 22] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 23]) << 24);
+
+ _decodeBuf6 = ((uint)(inputBuffer[inputOffset + 24] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 25] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 26] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 27]) << 24);
+
+ _decodeBuf7 = ((uint)(inputBuffer[inputOffset + 28] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 29] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 30] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 31]) << 24);
+
+ _decodeBuf8 = ((uint)(inputBuffer[inputOffset + 32] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 33] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 34] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 35]) << 24);
+
+ _decodeBuf9 = ((uint)(inputBuffer[inputOffset + 36] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 37] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 38] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 39]) << 24);
+
+ _decodeBuf10 = ((uint)(inputBuffer[inputOffset + 40] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 41] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 42] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 43]) << 24);
+
+ _decodeBuf11 = ((uint)(inputBuffer[inputOffset + 44] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 45] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 46] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 47]) << 24);
+
+ _decodeBuf12 = ((uint)(inputBuffer[inputOffset + 48] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 49] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 50] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 51]) << 24);
+
+ _decodeBuf13 = ((uint)(inputBuffer[inputOffset + 52] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 53] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 54] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 55]) << 24);
+
+ _decodeBuf14 = ((uint)(inputBuffer[inputOffset + 56] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 57] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 58] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 59]) << 24);
+
+ _decodeBuf15 = ((uint)(inputBuffer[inputOffset + 60] & 0xff)) |
+ (((uint)(inputBuffer[inputOffset + 61] & 0xff)) << 8) |
+ (((uint)(inputBuffer[inputOffset + 62] & 0xff)) << 16) |
+ (((uint)inputBuffer[inputOffset + 63]) << 24);
+
+ a = _state0;
+ b = _state1;
+ c = _state2;
+ d = _state3;
+
+ // ---- Round 1 --------
+
+ // ---- Round 1 --------
+
+ a += (((c ^ d) & b) ^ d) + (uint)0xd76aa478 + _decodeBuf0;
+ a = (a << 7) | (a >> 25);
+ a += b;
+
+ d += (((b ^ c) & a) ^ c) + (uint)0xe8c7b756 + _decodeBuf1;
+ d = (d << 12) | (d >> 20);
+ d += a;
+
+ c += (((a ^ b) & d) ^ b) + (uint)0x242070db + _decodeBuf2;
+ c = (c << 17) | (c >> 15);
+ c += d;
+
+ b += (((d ^ a) & c) ^ a) + (uint)0xc1bdceee + _decodeBuf3;
+ b = (b << 22) | (b >> 10);
+ b += c;
+
+ a += (((c ^ d) & b) ^ d) + (uint)0xf57c0faf + _decodeBuf4;
+ a = (a << 7) | (a >> 25);
+ a += b;
+
+ d += (((b ^ c) & a) ^ c) + (uint)0x4787c62a + _decodeBuf5;
+ d = (d << 12) | (d >> 20);
+ d += a;
+
+ c += (((a ^ b) & d) ^ b) + (uint)0xa8304613 + _decodeBuf6;
+ c = (c << 17) | (c >> 15);
+ c += d;
+
+ b += (((d ^ a) & c) ^ a) + (uint)0xfd469501 + _decodeBuf7;
+ b = (b << 22) | (b >> 10);
+ b += c;
+
+ a += (((c ^ d) & b) ^ d) + (uint)0x698098d8 + _decodeBuf8;
+ a = (a << 7) | (a >> 25);
+ a += b;
+
+ d += (((b ^ c) & a) ^ c) + (uint)0x8b44f7af + _decodeBuf9;
+ d = (d << 12) | (d >> 20);
+ d += a;
+
+ c += (((a ^ b) & d) ^ b) + (uint)0xffff5bb1 + _decodeBuf10;
+ c = (c << 17) | (c >> 15);
+ c += d;
+
+ b += (((d ^ a) & c) ^ a) + (uint)0x895cd7be + _decodeBuf11;
+ b = (b << 22) | (b >> 10);
+ b += c;
+
+ a += (((c ^ d) & b) ^ d) + (uint)0x6b901122 + _decodeBuf12;
+ a = (a << 7) | (a >> 25);
+ a += b;
+
+ d += (((b ^ c) & a) ^ c) + (uint)0xfd987193 + _decodeBuf13;
+ d = (d << 12) | (d >> 20);
+ d += a;
+
+ c += (((a ^ b) & d) ^ b) + (uint)0xa679438e + _decodeBuf14;
+ c = (c << 17) | (c >> 15);
+ c += d;
+
+ b += (((d ^ a) & c) ^ a) + (uint)0x49b40821 + _decodeBuf15;
+ b = (b << 22) | (b >> 10);
+ b += c;
+
+
+ // ---- Round 2 --------
+
+ a += ((b & d) | (c & ~d)) + (uint)0xf61e2562 + _decodeBuf1;
+ a = (a << 5) | (a >> 27);
+ a += b;
+
+ d += ((a & c) | (b & ~c)) + (uint)0xc040b340 + _decodeBuf6;
+ d = (d << 9) | (d >> 23);
+ d += a;
+
+ c += ((d & b) | (a & ~b)) + (uint)0x265e5a51 + _decodeBuf11;
+ c = (c << 14) | (c >> 18);
+ c += d;
+
+ b += ((c & a) | (d & ~a)) + (uint)0xe9b6c7aa + _decodeBuf0;
+ b = (b << 20) | (b >> 12);
+ b += c;
+
+ a += ((b & d) | (c & ~d)) + (uint)0xd62f105d + _decodeBuf5;
+ a = (a << 5) | (a >> 27);
+ a += b;
+
+ d += ((a & c) | (b & ~c)) + (uint)0x02441453 + _decodeBuf10;
+ d = (d << 9) | (d >> 23);
+ d += a;
+
+ c += ((d & b) | (a & ~b)) + (uint)0xd8a1e681 + _decodeBuf15;
+ c = (c << 14) | (c >> 18);
+ c += d;
+
+ b += ((c & a) | (d & ~a)) + (uint)0xe7d3fbc8 + _decodeBuf4;
+ b = (b << 20) | (b >> 12);
+ b += c;
+
+ a += ((b & d) | (c & ~d)) + (uint)0x21e1cde6 + _decodeBuf9;
+ a = (a << 5) | (a >> 27);
+ a += b;
+
+ d += ((a & c) | (b & ~c)) + (uint)0xc33707d6 + _decodeBuf14;
+ d = (d << 9) | (d >> 23);
+ d += a;
+
+ c += ((d & b) | (a & ~b)) + (uint)0xf4d50d87 + _decodeBuf3;
+ c = (c << 14) | (c >> 18);
+ c += d;
+
+ b += ((c & a) | (d & ~a)) + (uint)0x455a14ed + _decodeBuf8;
+ b = (b << 20) | (b >> 12);
+ b += c;
+
+ a += ((b & d) | (c & ~d)) + (uint)0xa9e3e905 + _decodeBuf13;
+ a = (a << 5) | (a >> 27);
+ a += b;
+
+ d += ((a & c) | (b & ~c)) + (uint)0xfcefa3f8 + _decodeBuf2;
+ d = (d << 9) | (d >> 23);
+ d += a;
+
+ c += ((d & b) | (a & ~b)) + (uint)0x676f02d9 + _decodeBuf7;
+ c = (c << 14) | (c >> 18);
+ c += d;
+
+ b += ((c & a) | (d & ~a)) + (uint)0x8d2a4c8a + _decodeBuf12;
+ b = (b << 20) | (b >> 12);
+ b += c;
+
+
+ // ---- Round 3 --------
+
+ a += (b ^ c ^ d) + (uint)0xfffa3942 + _decodeBuf5;
+ a = (a << 4) | (a >> 28);
+ a += b;
+
+ d += (a ^ b ^ c) + (uint)0x8771f681 + _decodeBuf8;
+ d = (d << 11) | (d >> 21);
+ d += a;
+
+ c += (d ^ a ^ b) + (uint)0x6d9d6122 + _decodeBuf11;
+ c = (c << 16) | (c >> 16);
+ c += d;
+
+ b += (c ^ d ^ a) + (uint)0xfde5380c + _decodeBuf14;
+ b = (b << 23) | (b >> 9);
+ b += c;
+
+ a += (b ^ c ^ d) + (uint)0xa4beea44 + _decodeBuf1;
+ a = (a << 4) | (a >> 28);
+ a += b;
+
+ d += (a ^ b ^ c) + (uint)0x4bdecfa9 + _decodeBuf4;
+ d = (d << 11) | (d >> 21);
+ d += a;
+
+ c += (d ^ a ^ b) + (uint)0xf6bb4b60 + _decodeBuf7;
+ c = (c << 16) | (c >> 16);
+ c += d;
+
+ b += (c ^ d ^ a) + (uint)0xbebfbc70 + _decodeBuf10;
+ b = (b << 23) | (b >> 9);
+ b += c;
+
+ a += (b ^ c ^ d) + (uint)0x289b7ec6 + _decodeBuf13;
+ a = (a << 4) | (a >> 28);
+ a += b;
+
+ d += (a ^ b ^ c) + (uint)0xeaa127fa + _decodeBuf0;
+ d = (d << 11) | (d >> 21);
+ d += a;
+
+ c += (d ^ a ^ b) + (uint)0xd4ef3085 + _decodeBuf3;
+ c = (c << 16) | (c >> 16);
+ c += d;
+
+ b += (c ^ d ^ a) + (uint)0x04881d05 + _decodeBuf6;
+ b = (b << 23) | (b >> 9);
+ b += c;
+
+ a += (b ^ c ^ d) + (uint)0xd9d4d039 + _decodeBuf9;
+ a = (a << 4) | (a >> 28);
+ a += b;
+
+ d += (a ^ b ^ c) + (uint)0xe6db99e5 + _decodeBuf12;
+ d = (d << 11) | (d >> 21);
+ d += a;
+
+ c += (d ^ a ^ b) + (uint)0x1fa27cf8 + _decodeBuf15;
+ c = (c << 16) | (c >> 16);
+ c += d;
+
+ b += (c ^ d ^ a) + (uint)0xc4ac5665 + _decodeBuf2;
+ b = (b << 23) | (b >> 9);
+ b += c;
+
+
+ // ---- Round 4 --------
+
+ a += (((~d) | b) ^ c) + (uint)0xf4292244 + _decodeBuf0;
+ a = (a << 6) | (a >> 26);
+ a += b;
+
+ d += (((~c) | a) ^ b) + (uint)0x432aff97 + _decodeBuf7;
+ d = (d << 10) | (d >> 22);
+ d += a;
+
+ c += (((~b) | d) ^ a) + (uint)0xab9423a7 + _decodeBuf14;
+ c = (c << 15) | (c >> 17);
+ c += d;
+
+ b += (((~a) | c) ^ d) + (uint)0xfc93a039 + _decodeBuf5;
+ b = (b << 21) | (b >> 11);
+ b += c;
+
+ a += (((~d) | b) ^ c) + (uint)0x655b59c3 + _decodeBuf12;
+ a = (a << 6) | (a >> 26);
+ a += b;
+
+ d += (((~c) | a) ^ b) + (uint)0x8f0ccc92 + _decodeBuf3;
+ d = (d << 10) | (d >> 22);
+ d += a;
+
+ c += (((~b) | d) ^ a) + (uint)0xffeff47d + _decodeBuf10;
+ c = (c << 15) | (c >> 17);
+ c += d;
+
+ b += (((~a) | c) ^ d) + (uint)0x85845dd1 + _decodeBuf1;
+ b = (b << 21) | (b >> 11);
+ b += c;
+
+ a += (((~d) | b) ^ c) + (uint)0x6fa87e4f + _decodeBuf8;
+ a = (a << 6) | (a >> 26);
+ a += b;
+
+ d += (((~c) | a) ^ b) + (uint)0xfe2ce6e0 + _decodeBuf15;
+ d = (d << 10) | (d >> 22);
+ d += a;
+
+ c += (((~b) | d) ^ a) + (uint)0xa3014314 + _decodeBuf6;
+ c = (c << 15) | (c >> 17);
+ c += d;
+
+ b += (((~a) | c) ^ d) + (uint)0x4e0811a1 + _decodeBuf13;
+ b = (b << 21) | (b >> 11);
+ b += c;
+
+ a += (((~d) | b) ^ c) + (uint)0xf7537e82 + _decodeBuf4;
+ a = (a << 6) | (a >> 26);
+ a += b;
+
+ d += (((~c) | a) ^ b) + (uint)0xbd3af235 + _decodeBuf11;
+ d = (d << 10) | (d >> 22);
+ d += a;
+
+ c += (((~b) | d) ^ a) + (uint)0x2ad7d2bb + _decodeBuf2;
+ c = (c << 15) | (c >> 17);
+ c += d;
+
+ b += (((~a) | c) ^ d) + (uint)0xeb86d391 + _decodeBuf9;
+ b = (b << 21) | (b >> 11);
+ b += c;
+
+ _state0 += a;
+ _state1 += b;
+ _state2 += c;
+ _state3 += d;
+ }
+
+ private void ProcessFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount)
+ {
+ ulong total = count + (ulong)inputCount;
+ int paddingSize = (int)(56 - total % BLOCK_SIZE_BYTES);
+
+ if (paddingSize < 1)
+ paddingSize += BLOCK_SIZE_BYTES;
+
+ for (int i = 0; i < inputCount; i++)
+ {
+ fooBuffer[i] = inputBuffer[i + inputOffset];
+ }
+
+ fooBuffer[inputCount] = 0x80;
+ for (int i = inputCount + 1; i < inputCount + paddingSize; i++)
+ {
+ fooBuffer[i] = 0x00;
+ }
+
+ // I deal in bytes. The algorithm deals in bits.
+ ulong size = total << 3;
+ AddLength(size, fooBuffer, inputCount + paddingSize);
+ ProcessBlock(fooBuffer, 0);
+
+ if (inputCount + paddingSize + 8 == 128)
+ {
+ ProcessBlock(fooBuffer, 64);
+ }
+ }
+
+ internal void AddLength(ulong length, byte[] buffer, int position)
+ {
+ buffer[position++] = (byte)(length);
+ buffer[position++] = (byte)(length >> 8);
+ buffer[position++] = (byte)(length >> 16);
+ buffer[position++] = (byte)(length >> 24);
+ buffer[position++] = (byte)(length >> 32);
+ buffer[position++] = (byte)(length >> 40);
+ buffer[position++] = (byte)(length >> 48);
+ buffer[position] = (byte)(length >> 56);
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/VarBigArray.cs b/BotSharp.MachineLearning/CRFLite/Utils/VarBigArray.cs
new file mode 100644
index 00000000..634b6c7a
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/VarBigArray.cs
@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ public sealed class VarBigArray : BigArray where T : IComparable
+ {
+ long blockSizeInTotal_;
+ private object ll = new object();
+
+ public override T this[long offset]
+ {
+ get
+ {
+ if (offset >= size_)
+ {
+ //resize array size, it need to be synced,
+ //for high performance, we use double check to avoid useless resize call and save memory
+ lock (ll)
+ {
+ if (offset >= size_)
+ {
+ Resize(offset + 1);
+ }
+ }
+ }
+
+ long nBlock = offset >> moveBit;
+ return arrList[(int)nBlock][offset & (sizePerBlock-1)];
+ }
+ set
+ {
+ if (offset >= size_)
+ {
+ //resize array size, it need to be synced,
+ //for high performance, we use double check to avoid useless resize call and save memory
+ lock (ll)
+ {
+ if (offset >= size_)
+ {
+ Resize(offset + 1);
+ }
+ }
+ }
+
+ long nBlock = offset >> moveBit;
+ arrList[(int)nBlock][offset & (sizePerBlock-1)] = value;
+ }
+ }
+
+
+
+ private void Resize(long new_size)
+ {
+ while (blockSizeInTotal_ <= new_size)
+ {
+ arrList.Add(new T[sizePerBlock]);
+ blockSizeInTotal_ += sizePerBlock;
+ }
+
+ size_ = new_size;
+ }
+
+ //construct variable size big array
+ //size is array's default length
+ //lowBounding is the lowest bounding of the array
+ //when accessing the position which is outer bounding, the big array will be extend automatically.
+ public VarBigArray(long size)
+ {
+ size_ = size;
+ arrList = new List();
+
+ for (blockSizeInTotal_ = 0; blockSizeInTotal_ < size_;
+ blockSizeInTotal_ += sizePerBlock)
+ {
+ arrList.Add(new T[sizePerBlock]);
+ }
+ }
+ }
+}
diff --git a/BotSharp.MachineLearning/CRFLite/Utils/VectorQuantization.cs b/BotSharp.MachineLearning/CRFLite/Utils/VectorQuantization.cs
new file mode 100644
index 00000000..8aaa4546
--- /dev/null
+++ b/BotSharp.MachineLearning/CRFLite/Utils/VectorQuantization.cs
@@ -0,0 +1,377 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.IO;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+ public class VQCluster : IComparer
+ {
+ public int iStart, iEnd;
+ public double variance, mean;
+
+ public VQCluster(double m, double v, int i, int j)
+ {
+ mean = m;
+ variance = v;
+ iStart = i;
+ iEnd = j;
+ }
+
+ public VQCluster() { }
+
+ public int Compare(VQCluster X, VQCluster Y)
+ {
+ if (X.mean > Y.mean) return 1;
+ if (X.mean < Y.mean) return -1;
+ return 0;
+ }
+ };
+
+
+ public class VectorQuantization
+ {
+ protected List vqClusters;
+ protected double[] codebook;
+ protected VarBigArray dataSet;
+ protected int dataSetSize;
+
+ public double[] CodeBook { get { return codebook; } }
+
+ public VectorQuantization()
+ {
+ dataSet = new VarBigArray(1024 * 1024);
+ dataSetSize = 0;
+ }
+
+ ///
+ /// Add a set of data into data set
+ ///
+ ///
+ public void Add(double[] values)
+ {
+ foreach (double value in values)
+ {
+ Add(value);
+ }
+ }
+
+ ///
+ /// Add a single data into data set
+ ///
+ ///
+ public void Add(double value)
+ {
+ dataSet[dataSetSize] = value;
+ dataSetSize++;
+ }
+
+ public int ComputeVQ(double value)
+ {
+ return BinarySearch(value);
+ }
+
+ ///
+ /// Build codebook according given data set
+ ///
+ ///
+ ///
+ public double BuildCodebook(int vqSize)
+ {
+ if (vqSize > dataSetSize)
+ {
+ return -1;
+ }
+
+ dataSet.Sort(0, dataSetSize);
+
+ //Set entire data as a single cluster, and then split it
+ double mean, var;
+ ComputeVariables(0, dataSetSize - 1, out mean, out var);
+ VQCluster c = new VQCluster(mean, var, 0, dataSetSize - 1);
+ vqClusters = new List();
+ vqClusters.Add(c);
+
+ //Split clusters according its variance values
+ while (vqClusters.Count < vqSize)
+ {
+ int maxVarClusterId = MaxVarianceClusterId();
+ if (maxVarClusterId < 0) break; // no more to split
+
+ //Split the cluster into two and remove the orginal one
+ SplitCluster(vqClusters[maxVarClusterId].iStart, vqClusters[maxVarClusterId].iEnd, 0, 1);
+ vqClusters.RemoveAt(maxVarClusterId);
+ }
+
+ //Adjust clusters according their mean values
+ AdjustCluster();
+
+ //Final codebook
+ vqSize = vqClusters.Count;
+ codebook = new double[vqSize];
+ double distortion = 0;
+ for (int i = 0; i < vqSize; i++)
+ {
+ codebook[i] = vqClusters[i].mean;
+ for (int j = vqClusters[i].iStart; j <= vqClusters[i].iEnd; j++)
+ {
+ double diff = dataSet[j] - codebook[i];
+ distortion += diff * diff;
+ }
+ }
+
+ distortion = Math.Sqrt(distortion / dataSetSize);
+ return distortion;
+ }
+
+ public bool WriteCodebook(string filename)
+ {
+ using (StreamWriter sw = new StreamWriter(filename))
+ {
+ sw.WriteLine("Codeword\tMean\tCount");
+ for (int i = 0; i < codebook.Length; i++)
+ {
+ int count = (int)(vqClusters[i].iEnd - vqClusters[i].iStart + 1);
+ sw.WriteLine("{0,8} {1}\t{2}", i, codebook[i], count);
+ }
+ }
+
+ return true;
+ }
+
+ public void ReadCodebook(string filename)
+ {
+ using (StreamReader sr = new StreamReader(filename))
+ {
+ //Skip column title
+ sr.ReadLine();
+
+ //Read each line
+ string line = null;
+ List cb = new List();
+ while ((line = sr.ReadLine()) != null)
+ {
+ string[] words = line.Split();
+ double mean = 0;
+
+ int n = int.Parse(words[0]);
+ mean = double.Parse(words[1]);
+ int count = int.Parse(words[2]);
+ cb.Add(mean);
+ }
+ codebook = cb.ToArray();
+ }
+ }
+
+ ///
+ /// Adjust cluster boundary according mean values
+ ///
+ void AdjustCluster()
+ {
+ int vqsize = vqClusters.Count;
+ bool updateCluster = true;
+ double mean, var;
+
+ vqClusters.Sort(new VQCluster());
+
+ for (int iter = 0; iter < 20 && updateCluster; iter++)
+ {
+ updateCluster = false;
+ for (int i = 1; i < vqsize; i++)
+ {
+ int j = (int)vqClusters[i - 1].iEnd;
+ while (true)
+ {
+ double d1 = dataSet[j] - vqClusters[i - 1].mean;
+ if (d1 <= 0) break;
+
+ double d2 = vqClusters[i].mean - dataSet[j];
+
+ if (d1 <= d2) break;
+ j--;
+ }
+
+ if (j < vqClusters[i - 1].iEnd)
+ {
+ ComputeVariables((int)vqClusters[i - 1].iStart, j, out mean, out var);
+ UpdateCluster(i - 1, (int)vqClusters[i - 1].iStart, j, mean, var);
+
+ ComputeVariables(j + 1, (int)vqClusters[i].iEnd, out mean, out var);
+ UpdateCluster(i, j + 1, (int)vqClusters[i].iEnd, mean, var);
+
+ updateCluster = true;
+ continue;
+ }
+
+ j = (int)vqClusters[i].iStart;
+ while (true)
+ {
+ double d1 = vqClusters[i].mean - dataSet[j];
+ if (d1 <= 0) break;
+
+ double d2 = dataSet[j] - vqClusters[i - 1].mean;
+
+ if (d1 <= d2) break;
+ j++;
+ }
+ if (j > vqClusters[i].iStart)
+ {
+ ComputeVariables((int)vqClusters[i - 1].iStart, j - 1, out mean, out var);
+ UpdateCluster(i - 1, (int)vqClusters[i - 1].iStart, j - 1, mean, var);
+
+ ComputeVariables(j, (int)vqClusters[i].iEnd, out mean, out var);
+ UpdateCluster(i, j, (int)vqClusters[i].iEnd, mean, var);
+
+ updateCluster = true;
+ }
+ }
+ }
+ }
+
+ ///
+ /// Search codebook and get the index which value is the nearest to given value
+ ///
+ ///
+ ///
+ private int BinarySearch(double value)
+ {
+ int low = 0, high = codebook.Length, mid = 0;
+ while (low < high)
+ {
+ mid = (int)((high - low) / 2) + low;
+ if (value > codebook[mid])
+ low = mid + 1;
+ else if (value < codebook[mid])
+ high = mid;
+ else
+ return mid;
+ }
+
+ int cw = mid;
+ double delta = Math.Abs(value - codebook[cw]);
+ if (mid + 1 < codebook.Length)
+ {
+ double d2 = Math.Abs(value - codebook[mid + 1]);
+ if (d2 < delta)
+ {
+ cw = mid + 1;
+ delta = d2;
+ }
+ }
+ if (mid - 1 >= 0)
+ {
+ double d2 = Math.Abs(value - codebook[mid - 1]);
+ if (d2 < delta)
+ {
+ cw = mid - 1;
+ delta = d2;
+ }
+ }
+ return cw;
+ }
+
+ ///
+ /// Return the cluster id which has the biggest variance value
+ ///
+ ///
+ private int MaxVarianceClusterId()
+ {
+ double maxVar = -1;
+ int c = -1;
+ for (int i = 0; i < vqClusters.Count; i++)
+ {
+ if (vqClusters[i].variance > maxVar)
+ {
+ maxVar = vqClusters[i].variance;
+ c = i;
+ }
+ }
+ return c;
+ }
+
+ ///
+ /// Computing the mean and variance of given data set
+ ///
+ ///
+ ///
+ ///
+ ///
+ private void ComputeVariables(int iStart, int iEnd, out double mean, out double variance)
+ {
+ double sum = 0;
+
+ mean = 0;
+ variance = 0;
+ for (int i = iStart; i <= iEnd; i++)
+ sum += dataSet[i];
+ mean = sum / (iEnd - iStart + 1);
+
+ sum = 0;
+ if (dataSet[iStart] < mean && dataSet[iEnd] > mean)
+ {
+ for (int i = iStart; i <= iEnd; i++)
+ {
+ double diff = dataSet[i] - mean;
+ sum += diff * diff;
+ }
+ }
+ variance = sum;
+ }
+
+ ///
+ /// Update given cluster's values
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ private void UpdateCluster(int index, int iStart, int iEnd, double mean, double variance)
+ {
+ vqClusters[index].iStart = iStart;
+ vqClusters[index].iEnd = iEnd;
+ vqClusters[index].mean = mean;
+ vqClusters[index].variance = variance;
+
+ }
+
+ ///
+ /// Split one cluster into two clusters according its mean
+ ///
+ ///
+ ///
+ ///
+ ///
+ private void SplitCluster(int iStart, int iEnd, int depth, int maxDepth)
+ {
+ if (iStart > iEnd)
+ {
+ return;
+ }
+
+ double mean, variance;
+ ComputeVariables(iStart, iEnd, out mean, out variance);
+ if (depth == maxDepth)
+ {
+ VQCluster c = new VQCluster(mean, variance, iStart, iEnd);
+ vqClusters.Add(c);
+ }
+ else
+ {
+ //Split the cluster into two clusters according mean value
+ int i;
+ for (i = iStart; i <= iEnd; i++)
+ {
+ //The following data will be greater than mean value, so we split it here
+ if (dataSet[i] > mean)
+ break;
+ }
+
+ SplitCluster(iStart, i - 1, depth + 1, maxDepth);
+ SplitCluster(i, iEnd, depth + 1, maxDepth);
+ }
+ }
+ }
+}
diff --git a/BotSharp.sln b/BotSharp.sln
index 1465645b..630d1360 100644
--- a/BotSharp.sln
+++ b/BotSharp.sln
@@ -17,6 +17,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NL
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.MachineLearning.UnitTest", "BotSharp.MachineLearning.UnitTest\BotSharp.MachineLearning.UnitTest.csproj", "{B876F0E9-40F0-48B7-91D5-E09E0B442266}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -51,6 +53,10 @@ Global
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.Build.0 = Release|Any CPU
+ {B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE