initial CRFLite algrithm.

BotSharp.MachineLearning.UnitTest/BotSharp.MachineLearning.UnitTest.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.7.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="1.2.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="1.2.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\BotSharp.MachineLearning\BotSharp.MachineLearning.csproj" />
+  </ItemGroup>
+
+</Project>

BotSharp.MachineLearning.UnitTest/DecoderTest.cs

@@ -0,0 +1,287 @@
+using BotSharp.MachineLearning.CRFLite;
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.UnitTest
+{
+    [TestClass]
+    public class DecoderTest
+    {
+        [TestMethod]
+        public void TestDecode()
+        {
+            var encoder = new CRFDecoder();
+            bool bRet = Decode(new DecoderOptions
+            {
+
+            });
+        }
+
+        object rdLocker = new object();
+
+        bool Decode(DecoderOptions options)
+        {
+            var parallelOption = new ParallelOptions();
+            var watch = Stopwatch.StartNew();
+            if (File.Exists(options.strInputFileName) == false)
+            {
+                //Logger.WriteLine("FAILED: Open {0} file failed.", options.strInputFileName);
+                return false;
+            }
+
+            if (File.Exists(options.strModelFileName) == false)
+            {
+                //Logger.WriteLine("FAILED: Open {0} file failed.", options.strModelFileName);
+                return false;
+            }
+
+            var sr = new StreamReader(options.strInputFileName);
+            StreamWriter sw = null, swSeg = null;
+
+            if (options.strOutputFileName != null && options.strOutputFileName.Length > 0)
+            {
+                sw = new StreamWriter(options.strOutputFileName);
+            }
+            if (options.strOutputSegFileName != null && options.strOutputSegFileName.Length > 0)
+            {
+                swSeg = new StreamWriter(options.strOutputSegFileName);
+            }
+
+            //Create CRFSharp wrapper instance. It's a global instance
+            var crfWrapper = new CRFDecoder();
+
+            //Load encoded model from file
+            //Logger.WriteLine("Loading model from {0}", options.strModelFileName);
+            crfWrapper.LoadModel(options.strModelFileName);
+
+            var queueRecords = new ConcurrentQueue<List<List<string>>>();
+            var queueSegRecords = new ConcurrentQueue<List<List<string>>>();
+
+            parallelOption.MaxDegreeOfParallelism = options.thread;
+            Parallel.For(0, options.thread, parallelOption, t =>
+            {
+
+                //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
+                var tagger = crfWrapper.CreateTagger(options.nBest, options.maxword);
+                tagger.set_vlevel(options.probLevel);
+
+                //Initialize result
+                var crf_out = new crf_seg_out[options.nBest];
+                for (var i = 0; i < options.nBest; i++)
+                {
+                    crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
+                }
+
+                var inbuf = new List<List<string>>();
+                while (true)
+                {
+                    lock (rdLocker)
+                    {
+                        if (ReadRecord(inbuf, sr) == false)
+                        {
+                            break;
+                        }
+
+                        queueRecords.Enqueue(inbuf);
+                        queueSegRecords.Enqueue(inbuf);
+                    }
+
+                    //Call CRFSharp wrapper to predict given string's tags
+                    if (swSeg != null)
+                    {
+                        crfWrapper.Segment(crf_out, tagger, inbuf);
+                    }
+                    else
+                    {
+                        crfWrapper.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf);
+                    }
+
+                    List<List<string>> peek = null;
+                    //Save segmented tagged result into file
+                    if (swSeg != null)
+                    {
+                        var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out);
+                        while (peek != inbuf)
+                        {
+                            queueSegRecords.TryPeek(out peek);
+                        }
+                        for (int index = 0; index < rstList.Count; index++)
+                        {
+                            var item = rstList[index];
+                            swSeg.WriteLine(item);
+                        }
+                        queueSegRecords.TryDequeue(out peek);
+                        peek = null;
+                    }
+
+                    //Save raw tagged result (with probability) into file
+                    if (sw != null)
+                    {
+                        while (peek != inbuf)
+                        {
+                            queueRecords.TryPeek(out peek);
+                        }
+                        OutputRawResultToFile(inbuf, crf_out, tagger, sw);
+                        queueRecords.TryDequeue(out peek);
+
+                    }
+                }
+            });
+
+
+            sr.Close();
+
+            if (sw != null)
+            {
+                sw.Close();
+            }
+            if (swSeg != null)
+            {
+                swSeg.Close();
+            }
+            watch.Stop();
+            //Logger.WriteLine("Elapsed: {0} ms", watch.ElapsedMilliseconds);
+
+            return true;
+        }
+
+        private bool ReadRecord(List<List<string>> inbuf, StreamReader sr)
+        {
+            inbuf.Clear();
+
+            while (true)
+            {
+                var strLine = sr.ReadLine();
+                if (strLine == null)
+                {
+                    //At the end of current file
+                    if (inbuf.Count == 0)
+                    {
+                        return false;
+                    }
+                    else
+                    {
+                        return true;
+                    }
+                }
+                strLine = strLine.Trim();
+                if (strLine.Length == 0)
+                {
+                    return true;
+                }
+
+                //Read feature set for each record
+                var items = strLine.Split(new char[] { '\t' });
+                inbuf.Add(new List<string>());
+                for (int index = 0; index < items.Length; index++)
+                {
+                    var item = items[index];
+                    inbuf[inbuf.Count - 1].Add(item);
+                }
+            }
+        }
+
+        //Output raw result with probability
+        private void OutputRawResultToFile(List<List<string>> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw)
+        {
+            for (var k = 0; k < crf_out.Length; k++)
+            {
+                if (crf_out[k] == null)
+                {
+                    //No more result
+                    break;
+                }
+
+                var sb = new StringBuilder();
+
+                var crf_seg_out = crf_out[k];
+                //Show the entire sequence probability
+                //For each token
+                for (var i = 0; i < inbuf.Count; i++)
+                {
+                    //Show all features
+                    for (var j = 0; j < inbuf[i].Count; j++)
+                    {
+                        sb.Append(inbuf[i][j]);
+                        sb.Append("\t");
+                    }
+
+                    //Show the best result and its probability
+                    sb.Append(crf_seg_out.result_[i]);
+
+                    if (tagger.vlevel_ > 1)
+                    {
+                        sb.Append("\t");
+                        sb.Append(crf_seg_out.weight_[i]);
+
+                        //Show the probability of all tags
+                        sb.Append("\t");
+                        for (var j = 0; j < tagger.ysize_; j++)
+                        {
+                            sb.Append(tagger.yname(j));
+                            sb.Append("/");
+                            sb.Append(tagger.prob(i, j));
+
+                            if (j < tagger.ysize_ - 1)
+                            {
+                                sb.Append("\t");
+                            }
+                        }
+                    }
+                    sb.AppendLine();
+                }
+                if (tagger.vlevel_ > 0)
+                {
+                    sw.WriteLine("#{0}", crf_seg_out.prob);
+                }
+                sw.WriteLine(sb.ToString().Trim());
+                sw.WriteLine();
+            }
+        }
+
+        //Convert CRFSharp output format to string list
+        private List<string> ConvertCRFTermOutToStringList(List<List<string>> inbuf, crf_seg_out[] crf_out)
+        {
+            var sb = new StringBuilder();
+            for (var i = 0; i < inbuf.Count; i++)
+            {
+                sb.Append(inbuf[i][0]);
+            }
+
+            var strText = sb.ToString();
+            var rstList = new List<string>();
+            for (var i = 0; i < crf_out.Length; i++)
+            {
+                if (crf_out[i] == null)
+                {
+                    //No more result
+                    break;
+                }
+
+                sb.Clear();
+                var crf_term_out = crf_out[i];
+                for (var j = 0; j < crf_term_out.Count; j++)
+                {
+                    var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length);
+                    var strNE = crf_term_out.tokenList[j].strTag;
+
+                    sb.Append(str);
+                    if (strNE.Length > 0)
+                    {
+                        sb.Append("[" + strNE + "]");
+                    }
+                    sb.Append(" ");
+                }
+                rstList.Add(sb.ToString().Trim());
+            }
+
+            return rstList;
+        }
+    }
+}

BotSharp.MachineLearning.UnitTest/EncoderTest.cs

@@ -0,0 +1,22 @@
+using BotSharp.MachineLearning.CRFLite;
+using BotSharp.MachineLearning.CRFLite.Encoder;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace BotSharp.MachineLearning.UnitTest
+{
+    [TestClass]
+    public class EncoderTest
+    {
+        [TestMethod]
+        public void TestEncode()
+        {
+            var encoder = new CRFEncoder();
+            bool bRet = encoder.Learn(new EncoderOptions
+            {
+                TrainingCorpusFileName = "",
+                TemplateFileName = "",
+                ModelFileName = ""
+            });
+        }
+    }
+}

BotSharp.MachineLearning/BotSharp.MachineLearning.csproj

@@ -9,4 +9,10 @@
     <Folder Include="Fasttext\" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Reference Include="System.ComponentModel.Annotations">
+      <HintPath>..\..\..\..\..\..\Program Files\dotnet\sdk\NuGetFallbackFolder\microsoft.netcore.app\2.1.0\ref\netcoreapp2.1\System.ComponentModel.Annotations.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
 </Project>

BotSharp.MachineLearning/CRFLite/BaseModel.cs

@@ -0,0 +1,154 @@
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class BaseModel
+    {
+        public long maxid_;
+        public double cost_factor_;
+
+        public List<string> unigram_templs_;
+        public List<string> bigram_templs_;
+
+        //Labeling tag list
+        public List<string> y_;
+        public uint ysize() { return (uint)y_.Count; }
+
+        //The dimension training corpus
+        public uint xsize_;
+
+        //Feature set value array
+        public double[] alpha_;
+
+        public BaseModel()
+        {
+            cost_factor_ = 1.0;
+        }
+
+        //获取类别i的字符表示
+        public string y(int i) { return y_[i]; }
+
+        public long feature_size() { return maxid_; }
+
+        public StringBuilder apply_rule(string p, int pos, StringBuilder resultContainer, Tagger tagger)
+        {
+            resultContainer.Clear();
+            for (var i = 0; i < p.Length; i++)
+            {
+                if (p[i] == '%')
+                {
+                    i++;
+                    if (p[i] == 'x')
+                    {
+                        i++;
+                        var res = get_index(p, pos, i, tagger);
+                        i = res.idx;
+                        if (res.value == null)
+                        {
+                            return null;
+                        }
+                        resultContainer.Append(res.value);
+                    }
+                    else
+                    {
+                        return null;
+                    }
+                }
+                else
+                {
+                    resultContainer.Append(p[i]);
+                }
+            }
+            return resultContainer;
+        }
+
+        Index get_index(string p, int pos, int i, Tagger tagger)
+        {
+            if (p[i] != '[')
+            {
+                return new Index(null, i);
+            }
+            i++;
+            var isInRow = true;
+            var col = 0;
+            var row = 0;
+            var neg = 1;
+
+            if (p[i] == '-')
+            {
+                neg = -1;
+                i++;
+            }
+
+            for (; i < p.Length; i++)
+            {
+                var c = p[i];
+                if (isInRow)
+                {
+                    if (c >= '0' && c <= '9')
+                    {
+                        row = 10 * row + (c - '0');
+                    }
+                    else if (c == ',')
+                    {
+                        isInRow = false;
+                    }
+                    else
+                    {
+                        return new Index(null, i);
+                    }
+                }
+                else
+                {
+                    if (c >= '0' && c <= '9')
+                    {
+                        col = 10 * col + (c - '0');
+                    }
+                    else if (c == ']')
+                    {
+                        break;
+                    }
+                    else
+                    {
+                        return new Index(null, i);
+                    }
+                }
+            }
+
+            row *= neg;
+
+            if (col < 0 || col >= xsize_)
+            {
+                return new Index(null, i);
+            }
+            var idx = pos + row;
+            if (idx < 0)
+            {
+                return new Index("_B-" + (-idx).ToString(), i); ;
+            }
+            if (idx >= tagger.word_num)
+            {
+                return new Index("_B+" + (idx - tagger.word_num + 1).ToString(), i);
+            }
+
+            return new Index(tagger.x_[idx][col], i);
+
+        }
+
+        private struct Index
+        {
+            public int idx;
+            public string value;
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="T:System.Object"/> class.
+            /// </summary>
+            public Index(string value, int idx)
+            {
+                this.idx = idx;
+                this.value = value;
+            }
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/CRFDecoder.cs

@@ -0,0 +1,144 @@
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class CRFDecoder
+    {
+        ModelReader _modelReader;
+
+        /// <summary>
+        /// Load encoded model from file
+        /// </summary>
+        /// <param name="modelFilename">
+        /// The model path.
+        /// </param>
+        /// <returns></returns>
+        public void LoadModel(string modelFilename)
+        {
+            _modelReader = new ModelReader(modelFilename);
+            _modelReader.LoadModel();
+        }
+
+        /// <summary>
+        /// Loads an encoded model using the specified delegate.
+        /// Using this overload you can read the model e.g. 
+        /// from network, zipped archives or other locations, as you wish.
+        /// </summary>
+        /// <param name="modelLoader">
+        /// Allows reading the model from arbitrary formats and sources.
+        /// </param>
+        /// <param name="modelFilename">
+        /// The model file name, as used by the given <paramref name="modelLoader"/>
+        /// for file resolution.
+        /// </param>
+        /// <returns></returns>
+        public void LoadModel(Func<string, Stream> modelLoader, string modelFilename)
+        {
+            this._modelReader = new ModelReader(modelLoader, modelFilename);
+            _modelReader.LoadModel();
+        }
+
+        public SegDecoderTagger CreateTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+        {
+            if (_modelReader == null)
+            {
+                return null;
+            }
+
+            var tagger = new SegDecoderTagger(nbest, this_crf_max_word_num);
+            tagger.init_by_model(_modelReader);
+
+            return tagger;
+        }
+
+        //Segment given text
+        public int Segment(crf_seg_out[] pout, //segment result
+            SegDecoderTagger tagger, //Tagger per thread
+            List<List<string>> inbuf //feature set for segment
+            )
+        {
+            var ret = 0;
+            if (inbuf.Count == 0)
+            {
+                //Empty input string
+                return BaseUtils.ERROR_SUCCESS;
+            }
+
+            ret = tagger.reset();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            ret = tagger.add(inbuf);
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            //parse
+            ret = tagger.parse();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            //wrap result
+            ret = tagger.output(pout);
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+
+
+        //Segment given text
+        public int Segment(CRFTermOut[] pout, //segment result
+            DecoderTagger tagger, //Tagger per thread
+            List<List<string>> inbuf //feature set for segment
+            )
+        {
+            var ret = 0;
+            if (inbuf.Count == 0)
+            {
+                //Empty input string
+                return BaseUtils.ERROR_SUCCESS;
+            }
+
+            ret = tagger.reset();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            ret = tagger.add(inbuf);
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            //parse
+            ret = tagger.parse();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            //wrap result
+            ret = tagger.output(pout);
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/CRFEncoder.cs

@@ -0,0 +1,317 @@
+using BotSharp.MachineLearning.CRFLite.Encoder;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class CRFEncoder
+    {
+        public enum REG_TYPE { L1, L2 };
+
+        //encoding CRF model from training corpus
+        public bool Learn(EncoderOptions args)
+        {
+            if (args.MinDifference <= 0.0)
+            {
+                return false;
+            }
+
+            if (args.CostFactor < 0.0)
+            {
+                return false;
+            }
+
+            if (args.ThreadsNum <= 0)
+            {
+                return false;
+            }
+
+            if (args.HugeLexMemLoad > 0)
+            {
+            }
+
+            var modelWriter = new ModelWriter(args.ThreadsNum, args.CostFactor,
+                args.HugeLexMemLoad, args.RetrainModelFileName);
+
+            if (modelWriter.Open(args.TemplateFileName, args.TrainingCorpusFileName) == false)
+            {
+                return false;
+            }
+
+            var xList = modelWriter.ReadAllRecords();
+
+
+            modelWriter.Shrink(xList, args.MinFeatureFreq);
+
+            if (!modelWriter.SaveModelMetaData(args.ModelFileName))
+            {
+                return false;
+            }
+            else
+            {
+            }
+
+            if (!modelWriter.BuildFeatureSetIntoIndex(args.ModelFileName, args.SlotUsageRateThreshold, args.DebugLevel))
+            {
+                return false;
+            }
+            else
+            {
+            }
+
+            if (xList.Length == 0)
+            {
+                return false;
+            }
+
+            var orthant = false;
+            if (args.RegType == REG_TYPE.L1)
+            {
+                orthant = true;
+            }
+            if (runCRF(xList, modelWriter, orthant, args) == false)
+            {
+            }
+
+            modelWriter.SaveFeatureWeight(args.ModelFileName, args.BVQ);
+
+            return true;
+        }
+
+        bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args)
+        {
+            var old_obj = double.MaxValue;
+            var converge = 0;
+            var lbfgs = new LBFGS(args.ThreadsNum);
+            lbfgs.expected = new double[modelWriter.feature_size() + 1];
+
+            var processList = new List<CRFEncoderThread>();
+            var parallelOption = new ParallelOptions();
+            parallelOption.MaxDegreeOfParallelism = args.ThreadsNum;
+
+            //Initialize encoding threads
+            for (var i = 0; i < args.ThreadsNum; i++)
+            {
+                var thread = new CRFEncoderThread();
+                thread.start_i = i;
+                thread.thread_num = args.ThreadsNum;
+                thread.x = x;
+                thread.lbfgs = lbfgs;
+                thread.Init();
+                processList.Add(thread);
+            }
+
+            //Statistic term and result tags frequency
+            var termNum = 0;
+            int[] yfreq;
+            yfreq = new int[modelWriter.y_.Count];
+            for (int index = 0; index < x.Length; index++)
+            {
+                var tagger = x[index];
+                termNum += tagger.word_num;
+                for (var j = 0; j < tagger.word_num; j++)
+                {
+                    yfreq[tagger.answer_[j]]++;
+                }
+            }
+
+            //Iterative training
+            var startDT = DateTime.Now;
+            var dMinErrRecord = 1.0;
+            for (var itr = 0; itr < args.MaxIteration; ++itr)
+            {
+                //Clear result container
+                lbfgs.obj = 0.0f;
+                lbfgs.err = 0;
+                lbfgs.zeroone = 0;
+
+                Array.Clear(lbfgs.expected, 0, lbfgs.expected.Length);
+
+                var threadList = new List<Thread>();
+                for (var i = 0; i < args.ThreadsNum; i++)
+                {
+                    var thread = new Thread(processList[i].Run);
+                    thread.Start();
+                    threadList.Add(thread);
+                }
+
+                int[,] merr;
+                merr = new int[modelWriter.y_.Count, modelWriter.y_.Count];
+                for (var i = 0; i < args.ThreadsNum; ++i)
+                {
+                    threadList[i].Join();
+                    lbfgs.obj += processList[i].obj;
+                    lbfgs.err += processList[i].err;
+                    lbfgs.zeroone += processList[i].zeroone;
+
+                    //Calculate error
+                    for (var j = 0; j < modelWriter.y_.Count; j++)
+                    {
+                        for (var k = 0; k < modelWriter.y_.Count; k++)
+                        {
+                            merr[j, k] += processList[i].merr[j, k];
+                        }
+                    }
+                }
+
+                long num_nonzero = 0;
+                var fsize = modelWriter.feature_size();
+                var alpha = modelWriter.alpha_;
+                if (orthant == true)
+                {
+                    //L1 regularization
+                    Parallel.For<double>(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) =>
+                    {
+                        subtotal += Math.Abs(alpha[k] / modelWriter.cost_factor_);
+                        if (alpha[k] != 0.0)
+                        {
+                            Interlocked.Increment(ref num_nonzero);
+                        }
+                        return subtotal;
+                    },
+                   (subtotal) => // lock free accumulator
+                   {
+                       double initialValue;
+                       double newValue;
+                       do
+                       {
+                           initialValue = lbfgs.obj; // read current value
+                           newValue = initialValue + subtotal;  //calculate new value
+                       }
+                       while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue));
+                   });
+                }
+                else
+                {
+                    //L2 regularization
+                    num_nonzero = fsize;
+                    Parallel.For<double>(1, fsize + 1, parallelOption, () => 0, (k, loop, subtotal) =>
+                   {
+                       subtotal += (alpha[k] * alpha[k] / (2.0 * modelWriter.cost_factor_));
+                       lbfgs.expected[k] += (alpha[k] / modelWriter.cost_factor_);
+                       return subtotal;
+                   },
+                   (subtotal) => // lock free accumulator
+                   {
+                       double initialValue;
+                       double newValue;
+                       do
+                       {
+                           initialValue = lbfgs.obj; // read current value
+                           newValue = initialValue + subtotal;  //calculate new value
+                       }
+                       while (initialValue != Interlocked.CompareExchange(ref lbfgs.obj, newValue, initialValue));
+                   });
+                }
+
+                //Show each iteration result
+                var diff = (itr == 0 ? 1.0f : Math.Abs(old_obj - lbfgs.obj) / old_obj);
+                old_obj = lbfgs.obj;
+
+                ShowEvaluation(x.Length, modelWriter, lbfgs, termNum, itr, merr, yfreq, diff, startDT, num_nonzero, args);
+                if (diff < args.MinDifference)
+                {
+                    converge++;
+                }
+                else
+                {
+                    converge = 0;
+                }
+                if (itr > args.MaxIteration || converge == 3)
+                {
+                    break;  // 3 is ad-hoc
+                }
+
+                if (args.DebugLevel > 0 && (double)lbfgs.zeroone / (double)x.Length < dMinErrRecord)
+                {
+                    var cc = Console.ForegroundColor;
+                    Console.ForegroundColor = ConsoleColor.Red;
+                    Console.Write("[Debug Mode] ");
+                    Console.ForegroundColor = cc;
+
+                    //Save current best feature weight into file
+                    dMinErrRecord = (double)lbfgs.zeroone / (double)x.Length;
+                    modelWriter.SaveFeatureWeight("feature_weight_tmp", false);
+                }
+
+                int iret;
+                iret = lbfgs.optimize(alpha, modelWriter.cost_factor_, orthant);
+                if (iret <= 0)
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private static void ShowEvaluation(int recordNum, ModelWriter feature_index, LBFGS lbfgs, int termNum, int itr, int[,] merr, int[] yfreq, double diff, DateTime startDT, long nonzero_feature_num, EncoderOptions args)
+        {
+            var ts = DateTime.Now - startDT;
+
+            if (args.DebugLevel > 1)
+            {
+                for (var i = 0; i < feature_index.y_.Count; i++)
+                {
+                    var total_merr = 0;
+                    var sdict = new SortedDictionary<double, List<string>>();
+                    for (var j = 0; j < feature_index.y_.Count; j++)
+                    {
+                        total_merr += merr[i, j];
+                        var v = (double)merr[i, j] / (double)yfreq[i];
+                        if (v > 0.0001)
+                        {
+                            if (sdict.ContainsKey(v) == false)
+                            {
+                                sdict.Add(v, new List<string>());
+                            }
+                            sdict[v].Add(feature_index.y_[j]);
+                        }
+                    }
+                    var vet = (double)total_merr / (double)yfreq[i];
+                    vet = vet * 100.0F;
+
+                    Console.ForegroundColor = ConsoleColor.Green;
+                    Console.Write("{0} ", feature_index.y_[i]);
+                    Console.ResetColor();
+                    Console.Write("[FR={0}, TE=", yfreq[i]);
+                    Console.ForegroundColor = ConsoleColor.Yellow;
+                    Console.Write("{0:0.00}%", vet);
+                    Console.ResetColor();
+                    Console.WriteLine("]");
+
+                    var n = 0;
+                    foreach (var pair in sdict.Reverse())
+                    {
+                        for (int index = 0; index < pair.Value.Count; index++)
+                        {
+                            var item = pair.Value[index];
+                            n += item.Length + 1 + 7;
+                            if (n > 80)
+                            {
+                                //only show data in one line, more data in tail will not be show.
+                                break;
+                            }
+                            Console.Write("{0}:", item);
+                            Console.ForegroundColor = ConsoleColor.Red;
+                            Console.Write("{0:0.00}% ", pair.Key * 100);
+                            Console.ResetColor();
+                        }
+                        if (n > 80)
+                        {
+                            break;
+                        }
+                    }
+                    Console.WriteLine();
+                }
+            }
+
+            var act_feature_rate = (double)(nonzero_feature_num) / (double)(feature_index.feature_size()) * 100.0;
+            //Logger.WriteLine("iter={0} terr={1:0.00000} serr={2:0.00000} diff={3:0.000000} fsize={4}({5:0.00}% act)", itr, 1.0 * lbfgs.err / termNum, 1.0 * lbfgs.zeroone / recordNum, diff, feature_index.feature_size(), act_feature_rate);
+            //Logger.WriteLine("Time span: {0}, Aver. time span per iter: {1}", ts, new TimeSpan(0, 0, (int)(ts.TotalSeconds / (itr + 1))));
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/CRFSharpHelper.cs

@@ -0,0 +1,41 @@
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class SegToken
+    {
+        public int offset;
+        public int length;
+        public string strTag; //CRF对应于term组合后的Tag字符串
+        public double fWeight;   //对应属性id的概率值，或者得分
+    };
+
+    public class crf_seg_out : CRFTermOut
+    {
+        //Segmented token by merging raw CRF model output
+        public int termTotalLength;          // the total term length in character
+        public List<SegToken> tokenList;
+
+        public int Count
+        {
+            get { return tokenList.Count; }
+        }
+
+        public void Clear()
+        {
+            termTotalLength = 0;
+            tokenList.Clear();
+        }
+
+        public crf_seg_out(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM):
+            base(max_word_num)
+        {
+            termTotalLength = 0;
+            tokenList = new List<SegToken>();
+        }
+    };
+
+}

BotSharp.MachineLearning/CRFLite/Decoder/CRFTermOut.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+    public class CRFTermOut
+    {
+        //Sequence label probability
+        public double prob;
+
+        //Raw CRF model output
+        public string[] result_;
+        public double[] weight_;
+
+        public CRFTermOut(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+        {
+            prob = 0;
+            result_ = new string[max_word_num];
+            weight_ = new double[max_word_num];
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Decoder/DecoderOptions.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+    public class DecoderOptions
+    {
+        [Required]
+        public string strModelFileName;
+        [Required]
+        public string strInputFileName;
+        public string strOutputFileName;
+        public string strOutputSegFileName;
+        public int nBest;
+        public int thread;
+        public int probLevel;
+        public int maxword;
+
+        public DecoderOptions()
+        {
+            thread = 1;
+            nBest = 1;
+            probLevel = 0;
+            maxword = 100;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Decoder/DecoderTagger.cs

@@ -0,0 +1,471 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+    public class DecoderTagger : Tagger
+    {
+        private readonly Pool<StringBuilder> _buildersPool =
+            new Pool<StringBuilder>(p => new StringBuilder(100), b => b.Clear());
+
+        public int forward_backward_stat; //前向后向过程运行状态，0为未运行，1为已经运行
+
+        //概率计算函数
+        double toprob(Node n, double Z)
+        {
+            return Math.Exp(n.alpha + n.beta - n.cost - Z);
+        }
+
+        //To get the fastest decoded result, please set vlevel=0 and nbest=1, since it only outputs 1-best result without probability (forward-backward and A* aren't performed, only run viterbi)
+        public int vlevel_; //Need to calculate probability 0 - no need to calculate, 1 - calculate sequence label probability, 2 - calculate both sequence label and individual entity probability
+        protected int nbest_; //output top N-best result
+        //CrfModel model;
+        ModelReader featureIndex;
+
+        Node node(int i, int j)
+        {
+            return node_[i, j];
+        }
+
+        Heap heap_queue; //Using min-heap to get next result, it's only used when nbest > 1
+        public int crf_max_word_num;
+
+        public DecoderTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+        {
+            crf_max_word_num = this_crf_max_word_num;
+            vlevel_ = 0;
+            nbest_ = nbest;
+            cost_ = 0.0;
+            Z_ = 0;
+
+            ysize_ = 0;
+            word_num = 0;
+            heap_queue = null;
+            node_ = null;
+            x_ = null;
+            result_ = null;
+        }
+
+        public void InitializeFeatureCache()
+        {
+            feature_cache_ = new List<long[]>();
+            var feature_cache_every_row_size = 0;
+            if (featureIndex.unigram_templs_.Count > featureIndex.bigram_templs_.Count)
+            {
+                feature_cache_every_row_size = featureIndex.unigram_templs_.Count + 1;
+            }
+            else
+            {
+                feature_cache_every_row_size = featureIndex.bigram_templs_.Count + 1;
+            }
+            for (var i = 0; i < crf_max_word_num * 2; i++)
+            {
+                var features = new long[feature_cache_every_row_size];
+                for (var j = 0; j < feature_cache_every_row_size; j++)
+                {
+                    features[j] = -1;
+                }
+                feature_cache_.Add(features);
+            }
+        }
+
+        //获取序列的词数
+        public short get_word_num()
+        {
+            return word_num;
+        }
+
+        public double prob(int i, int j)
+        {
+            return toprob(node_[i, j], Z_);
+        }
+
+        //Get the probability of the i-th word's best result
+        public double prob(int i)
+        {
+            return toprob(node_[i, result_[i]], Z_);
+        }
+
+        //Get entire sequence probability
+        public double prob()
+        {
+            return Math.Exp(-cost_ - Z_);
+        }
+
+        //Get the string of i-th tag
+        public string yname(int i) { return featureIndex.y(i); }
+
+        //设置vlevel
+        public void set_vlevel(int vlevel_value)
+        {
+            vlevel_ = vlevel_value;
+        }
+
+        //使用模型初始化tag，必须先使用该函数初始化才能使用add和parse                                                                    
+        //正常返回为0， 错误返回<0
+        public int init_by_model(ModelReader model_p)
+        {
+            featureIndex = model_p;
+            ysize_ = (short)model_p.ysize();
+
+            if (nbest_ > 1)
+            {
+                //Only allocate heap when nbest is more than 1
+                heap_queue = BaseUtils.heap_init((int)(crf_max_word_num * ysize_ * ysize_));
+            }
+
+            //Initialize feature set cache according unigram and bigram templates
+            InitializeFeatureCache();
+
+            node_ = new Node[crf_max_word_num, ysize_];
+            result_ = new short[crf_max_word_num];
+
+            //Create node and path cache
+            for (short cur = 0; cur < crf_max_word_num; cur++)
+            {
+                for (short i = 0; i < ysize_; i++)
+                {
+                    var n = new Node();
+                    node_[cur, i] = n;
+
+                    n.lpathList = new List<Path>();
+                    n.rpathList = new List<Path>();
+                    n.x = cur;
+                    n.y = i;
+                }
+            }
+
+            for (var cur = 1; cur < crf_max_word_num; cur++)
+            {
+                for (var j = 0; j < ysize_; ++j)
+                {
+                    for (var i = 0; i < ysize_; ++i)
+                    {
+                        var p = new Path();
+                        p.add(node_[cur - 1, j], node_[cur, i]);
+                    }
+                }
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+        public int initNbest()
+        {
+            var k = (int)word_num - 1;
+            for (var i = 0; i < ysize_; ++i)
+            {
+                var eos = BaseUtils.allc_from_heap(heap_queue);
+                eos.node = node_[k, i];
+                eos.fx = -node_[k, i].bestCost;
+                eos.gx = -node_[k, i].cost;
+                eos.next = null;
+                if (BaseUtils.heap_insert(eos, heap_queue) < 0)
+                {
+                    return BaseUtils.ERROR_INSERT_HEAP_FAILED;
+                }
+            }
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+        public int next()
+        {
+            while (!BaseUtils.is_heap_empty(heap_queue))
+            {
+                var top = BaseUtils.heap_delete_min(heap_queue);
+                var rnode = top.node;
+
+                if (rnode.x == 0)
+                {
+                    for (var n = top; n != null; n = n.next)
+                    {
+                        result_[n.node.x] = n.node.y;
+                    }
+                    cost_ = top.gx;
+                    return 0;
+                }
+
+                for (int index = 0; index < rnode.lpathList.Count; index++)
+                {
+                    var p = rnode.lpathList[index];
+                    var n = BaseUtils.allc_from_heap(heap_queue);
+                    var x_num = (rnode.x) - 1;
+                    n.node = p.lnode;
+                    n.gx = -p.lnode.cost - p.cost + top.gx;
+                    n.fx = -p.lnode.bestCost - p.cost + top.gx;
+                    //          |              h(x)                 |  |  g(x)  |
+                    n.next = top;
+                    if (BaseUtils.heap_insert(n, heap_queue) < 0)
+                    {
+                        return BaseUtils.ERROR_INSERT_HEAP_FAILED;
+                    }
+                }
+            }
+            return 0;
+        }
+
+        public int reset()
+        {
+            word_num = 0;
+            Z_ = cost_ = 0.0;
+
+            BaseUtils.heap_reset(heap_queue);
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        int buildLattice()
+        {
+            //Generate feature ids for all nodes and paths
+            RebuildFeatures();
+
+            for (int i = 0; i < word_num; ++i)
+            {
+                for (int j = 0; j < ysize_; ++j)
+                {
+                    var currentNode = node_[i, j];
+                    calcCost(currentNode);
+                    for (int index = 0; index < currentNode.lpathList.Count; ++index)
+                    {
+                        var p = currentNode.lpathList[index];
+                        calcCost(p);
+                    }
+                }
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+        public int add(List<List<string>> row_p)
+        {
+            x_ = row_p;
+            word_num = (short)x_.Count;
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+
+        public int termbuf_build(CRFTermOut term_buf)
+        {
+            if (vlevel_ > 0)
+            {
+                //Calcuate the sequence label probability
+                term_buf.prob = prob();
+            }
+
+            var this_word_num = get_word_num();
+
+            for (var i = 0; i < this_word_num; ++i)
+            {
+                term_buf.result_[i] = yname(result_[i]);
+                switch (vlevel_)
+                {
+                    case 0:
+                        term_buf.weight_[i] = 0.0;
+                        break;
+                    case 2:
+                        term_buf.weight_[i] = prob(i);
+                        break;
+                }
+            }
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+        //Label input string. The result is saved as result []
+        //If nbest > 1, get nbest result by "next"
+        //Returen value: Successed - 0, Failed < 0
+        public int parse()
+        {
+            var ret = 0;
+            //no word need to be labeled
+            if (word_num == 0)
+            {
+                return BaseUtils.ERROR_SUCCESS;
+            }
+
+            //building feature set
+            ret = buildFeatures();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+
+            ret = buildLattice();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            //4.forward-backward when we need to calcuate probability
+            if (vlevel_ > 0)
+            {
+                forwardbackward();
+            }
+
+
+            //5.using viterbi to search best result path
+            ret = viterbi();
+            if (ret < 0)
+            {
+                return ret;
+            }
+
+            //6.initNbest
+            //	求nbest(n>1)时的数据结构初始化，此后可以调用next()来获取nbest结果
+            if (nbest_ > 1)
+            {
+                //如果只求1-best，不需要使用initNbest()和next()获取结果
+                ret = initNbest();
+                if (ret < 0)
+                {
+                    return ret;
+                }
+
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+
+        public int buildFeatures()
+        {
+            if (word_num <= 0)
+            {
+                return BaseUtils.ERROR_INVALIDATED_PARAMETER;
+            }
+            using (var v = _buildersPool.GetOrCreate())
+            {
+                var builder = v.Item;
+                var id = 0;
+                var feature_cache_row_size = 0;
+                var feature_cache_size = 0;
+                for (var cur = 0; cur < word_num; cur++)
+                {
+                    feature_cache_row_size = 0;
+                    for (int index = 0; index < featureIndex.unigram_templs_.Count; index++)
+                    {
+                        var templ = featureIndex.unigram_templs_[index];
+                        var res = featureIndex.apply_rule(templ, cur, builder, this);
+                        if (res == null)
+                        {
+                            return BaseUtils.ERROR_EMPTY_FEATURE;
+                        }
+                        id = featureIndex.get_id(res.ToString());
+                        if (id != -1)
+                        {
+                            feature_cache_[feature_cache_size][feature_cache_row_size] = id;
+                            feature_cache_row_size++;
+                        }
+                    }
+                    feature_cache_[feature_cache_size][feature_cache_row_size] = -1;
+                    feature_cache_size++;
+                }
+
+                for (var cur = 0; cur < word_num; cur++)
+                {
+                    feature_cache_row_size = 0;
+                    for (int index = 0; index < featureIndex.bigram_templs_.Count; index++)
+                    {
+                        var templ = featureIndex.bigram_templs_[index];
+                        var strFeature = featureIndex.apply_rule(templ, cur, builder, this);
+                        if (strFeature == null)
+                        {
+                            return BaseUtils.ERROR_EMPTY_FEATURE;
+                        }
+
+                        id = featureIndex.get_id(strFeature.ToString());
+                        if (id != -1)
+                        {
+                            feature_cache_[feature_cache_size][feature_cache_row_size] = id;
+                            feature_cache_row_size++;
+                        }
+                    }
+                    feature_cache_[feature_cache_size][feature_cache_row_size] = -1;
+                    feature_cache_size++;
+                }
+
+                return BaseUtils.ERROR_SUCCESS;
+            }
+        }
+
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void calcCost(Node n)
+        {
+            double c = 0;
+            var f = feature_cache_[n.fid];
+
+            for (int i = 0; i < f.Length; ++i)
+            {
+                int fCurrent = (int)f[i];
+                if (fCurrent == -1)
+                    break;
+                c += featureIndex.GetAlpha(fCurrent + n.y);
+            }
+
+            n.cost = featureIndex.cost_factor_ * c;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void calcCost(Path p)
+        {
+            double c = 0;
+            long[] f = feature_cache_[p.fid];
+            for (int i = 0; i < f.Length; ++i)
+            {
+                int fCurrent = (int)f[i];
+                if (fCurrent == -1)
+                    break;
+                c += featureIndex.GetAlpha((fCurrent + p.lnode.y * ysize_ + p.rnode.y));
+            }
+
+            p.cost = featureIndex.cost_factor_ * c;
+        }
+
+
+        public int output(CRFTermOut[] pout)
+        {
+            var n = 0;
+            var ret = 0;
+
+            if (nbest_ == 1)
+            {
+                //If only best result and no need probability, "next" is not to be used
+                ret = termbuf_build(pout[0]);
+                if (ret < 0)
+                {
+                    return ret;
+                }
+            }
+            else
+            {
+                //Fill the n best result
+                var iNBest = nbest_;
+                if (pout.Length < iNBest)
+                {
+                    iNBest = pout.Length;
+                }
+
+                for (n = 0; n < iNBest; ++n)
+                {
+                    ret = next();
+                    if (ret < 0)
+                    {
+                        break;
+                    }
+
+                    ret = termbuf_build(pout[n]);
+                    if (ret < 0)
+                    {
+                        return ret;
+                    }
+                }
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Decoder/ModelReader.cs

@@ -0,0 +1,251 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using BotSharp.MachineLearning.CRFLite.Utils;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+    public class ModelReader : BaseModel
+    {
+        private readonly Func<string, Stream> modelLoader = null;
+             
+        public uint version; //模型版本号,读取模型时读入
+        private CRFLite.Utils.DoubleArrayTrieSearch da; //特征集合
+
+        /// <summary>
+        /// Returns the model path.
+        /// </summary>
+        public string ModelPath { get; private set; }
+
+        /// <summary>
+        /// Creates a new <see cref="ModelReader"/>
+        /// that will load the model from the file system,
+        /// using the given <paramref name="modelPath"/>.
+        /// </summary>
+        /// <param name="modelPath">
+        /// Path to the model.
+        /// </param>
+        public ModelReader(string modelPath) :
+            this(GetStreamFromFileSystem, modelPath)
+        {
+            
+        }
+
+        /// <summary>
+        /// Creates a new <see cref="ModelReader"/>
+        /// that will load the model from the file system,
+        /// using the given <paramref name="modelPath"/>.
+        /// </summary>
+        /// <param name="modelLoader">
+        /// A delegate capable of resolving 
+        /// the given <paramref name="modelPath"/>
+        /// into a stream with the model file.
+        /// </param>
+        /// <param name="modelPath">
+        /// Path to the model.
+        /// </param>
+        public ModelReader(Func<string, Stream> modelLoader,
+            string modelPath)
+        {
+            this.modelLoader = modelLoader;
+            this.ModelPath = modelPath;
+        }
+
+        /// <summary>
+        /// Loads the model into memory.
+        /// </summary>
+        public void LoadModel()
+        {
+            //Load model meta data
+            LoadMetadata();
+
+            //Load all feature set data
+            LoadFeatureSet();
+
+            //Load all features alpha data
+            LoadFeatureWeights();
+        }
+
+        //获取key对应的特征id
+        public virtual int get_id(string str)
+        {
+            return da.SearchByPerfectMatch(str);
+        }
+
+        public virtual double GetAlpha(long index)
+        {
+            return alpha_[index];
+        }
+
+        /// <summary>
+        /// The default model loading strategy -
+        /// load files from the file system.
+        /// </summary>
+        /// <param name="path">
+        /// Model file path.</param>
+        /// <returns>
+        /// A stream containing the requested file.
+        /// </returns>
+        private static Stream GetStreamFromFileSystem(string path)
+        {
+            path.ThrowIfNotExists();
+            return File.OpenRead(path);
+        }
+
+        /// <summary>
+        /// Provides access to the metadata stream.
+        /// </summary>
+        /// <returns>
+        /// A <see cref="Stream"/> instance
+        /// that points to the model metadata file.
+        /// </returns>
+        private Stream GetMetadataStream()
+        {
+            string path = ModelPath.ToMetadataModelName();
+
+            return modelLoader(path);
+        }
+
+        /// <summary>
+        /// Provides access to the feature set stream.
+        /// </summary>
+        /// <returns>
+        /// A <see cref="Stream"/> instance
+        /// that allows accessing the model feature set file.
+        /// </returns>
+        private Stream GetFeatureSetStream()
+        {
+            string path = ModelPath.ToFeatureSetFileName();
+
+            return modelLoader(path);
+        }
+
+        /// <summary>
+        /// Provides access to the feature set stream.
+        /// </summary>
+        /// <returns>
+        /// A <see cref="Stream"/> instance
+        /// that allows accessing the model feature weight file.
+        /// </returns>
+        private Stream GetFeatureWeightStream()
+        {
+            string path = ModelPath.ToFeatureWeightFileName();
+
+            return modelLoader(path);
+        }
+
+        private void LoadMetadata()
+        {
+            using (Stream metadataStream = GetMetadataStream())
+            {
+                var sr = new StreamReader(metadataStream);
+                string strLine;
+
+                //读入版本号
+                strLine = sr.ReadLine();
+                version = uint.Parse(strLine.Split(':')[1].Trim());
+
+                //读入cost_factor
+                strLine = sr.ReadLine();
+                cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());
+
+                //读入maxid
+                strLine = sr.ReadLine();
+                maxid_ = long.Parse(strLine.Split(':')[1].Trim());
+
+                //读入xsize
+                strLine = sr.ReadLine();
+                xsize_ = uint.Parse(strLine.Split(':')[1].Trim());
+
+                //读入空行
+                strLine = sr.ReadLine();
+
+                //读入待标注的标签
+                y_ = new List<string>();
+                while (true)
+                {
+                    strLine = sr.ReadLine();
+                    if (strLine.Length == 0)
+                    {
+                        break;
+                    }
+                    y_.Add(strLine);
+                }
+
+                //读入unigram和bigram模板
+                unigram_templs_ = new List<string>();
+                bigram_templs_ = new List<string>();
+                while (sr.EndOfStream == false)
+                {
+                    strLine = sr.ReadLine();
+                    if (strLine.Length == 0)
+                    {
+                        break;
+                    }
+                    if (strLine[0] == 'U')
+                    {
+                        unigram_templs_.Add(strLine);
+                    }
+                    if (strLine[0] == 'B')
+                    {
+                        bigram_templs_.Add(strLine);
+                    }
+                }
+                sr.Close();
+            }
+        }
+
+        private void LoadFeatureSet()
+        {
+            Stream featureSetStream = GetFeatureSetStream();
+            da = new DoubleArrayTrieSearch();
+            da.Load(featureSetStream);
+        }
+
+        private void LoadFeatureWeights()
+        {
+            //feature weight array
+            alpha_ = new double[maxid_ + 1];
+
+            using (Stream featureWeightStream = GetFeatureWeightStream())
+            {
+                //Load all features alpha data
+                var sr_alpha = new StreamReader(featureWeightStream);
+                var br_alpha = new BinaryReader(sr_alpha.BaseStream);
+
+                //Get VQ Size
+                int vqSize = br_alpha.ReadInt32();
+
+                if (vqSize > 0)
+                {
+                    //This is a VQ model, we need to get code book at first
+                    List<double> vqCodeBook = new List<double>();
+                    for (int i = 0; i < vqSize; i++)
+                    {
+                        vqCodeBook.Add(br_alpha.ReadDouble());
+                    }
+
+                    //Load weights
+                    for (long i = 0; i < maxid_; i++)
+                    {
+                        int vqIdx = br_alpha.ReadByte();
+                        alpha_[i] = vqCodeBook[vqIdx];
+                    }
+                }
+                else
+                {
+                    //This is a normal model
+                    for (long i = 0; i < maxid_; i++)
+                    {
+                        alpha_[i] = br_alpha.ReadSingle();
+                    }
+                }
+
+                br_alpha.Close();
+            }
+        }
+
+    }
+}

BotSharp.MachineLearning/CRFLite/Decoder/ModelReaderExtensions.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Decoder
+{
+    internal static class ModelReaderExtensions
+    {
+        private static readonly string featureFileNameExtension = ".feature";
+        private static readonly string weightFileNameExtension = ".alpha";
+
+        internal static string ToMetadataModelName(this string modelName)
+        {
+            return modelName;
+        }
+
+        internal static string ToFeatureSetFileName(this string modelName)
+        {
+            return String.Concat(modelName, featureFileNameExtension);
+        }
+
+        internal static string ToFeatureWeightFileName(this string modelName)
+        {
+            return String.Concat(modelName, weightFileNameExtension);
+        }
+
+        internal static void ThrowIfNotExists(this string fileName)
+        {
+            if (String.IsNullOrWhiteSpace(fileName))
+                throw new ArgumentNullException("fileName",
+                    "Please specify a valid model path");
+
+            if (!File.Exists(fileName))
+                throw new FileNotFoundException("fileName",
+                    "Please specify a valid model path");
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/CRFEncoderThread.cs

@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public class CRFEncoderThread
+    {
+        public EncoderTagger[] x;
+        public int start_i;
+        public int thread_num;
+        public int zeroone;
+        public int err;
+        public double obj;
+        public Node[,] node_;
+        short[] result_;
+        public short max_xsize_;
+        public LBFGS lbfgs;
+        public int[,] merr;
+
+        public void Init()
+        {
+            if (x.Length == 0)
+            {
+                return;
+            }
+
+            var ysize_ = x[0].ysize_;
+            max_xsize_ = 0;
+            for (var i = start_i; i < x.Length; i += thread_num)
+            {
+                if (max_xsize_ < x[i].word_num)
+                {
+                    max_xsize_ = x[i].word_num;
+                }
+            }
+
+            result_ = new short[max_xsize_];
+            node_ = new Node[max_xsize_, ysize_];
+            for (var i = 0; i < max_xsize_; i++)
+            {
+                for (var j = 0; j < ysize_; j++)
+                {
+                    node_[i, j] = new Node();
+                    node_[i, j].x = (short)i;
+                    node_[i, j].y = (short)j;
+                    node_[i, j].lpathList = new List<Path>(ysize_);
+                    node_[i, j].rpathList = new List<Path>(ysize_);
+                }
+            }
+
+            for (short cur = 1; cur < max_xsize_; ++cur)
+            {
+                for (short j = 0; j < ysize_; ++j)
+                {
+                    for (short i = 0; i < ysize_; ++i)
+                    {
+                        var path = new Path();
+                        path.fid = -1;
+                        path.cost = 0.0;
+                        path.add(node_[cur - 1, j], node_[cur, i]);
+                    }
+                }
+            }
+
+            merr = new int[ysize_, ysize_];
+        }
+
+        public void Run()
+        {
+            //Initialize thread self data structure
+            obj = 0.0f;
+            err = zeroone = 0;
+            //expected.Clear();
+            Array.Clear(merr, 0, merr.Length);
+            for (var i = start_i; i < x.Length; i += thread_num)
+            {
+                x[i].Init(result_, node_);
+                obj += x[i].gradient(lbfgs.expected);
+                var error_num = x[i].eval(merr);
+                err += error_num;
+                if (error_num > 0)
+                {
+                    ++zeroone;
+                }
+            }
+        }
+
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/DefaultFeatureLexicalDict.cs

@@ -0,0 +1,125 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public class DefaultFeatureLexicalDict : IFeatureLexicalDict
+    {
+        CRFLite.Utils.BTreeDictionary<string, FeatureIdPair> featureset_dict_;
+        long maxid_;
+        Object thisLock = new object();
+        ParallelOptions parallelOption;
+
+        public DefaultFeatureLexicalDict(int thread_num)
+        {
+            featureset_dict_ = new CRFLite.Utils.BTreeDictionary<string, FeatureIdPair>(StringComparer.Ordinal, 128);
+            maxid_ = 0;
+            parallelOption = new ParallelOptions();
+            parallelOption.MaxDegreeOfParallelism = thread_num;
+        }
+
+        public void Clear()
+        {
+            featureset_dict_.Clear();
+            featureset_dict_ = null;
+        }
+
+        public long Size
+        {
+            get
+            {
+                return featureset_dict_.Count;
+            }
+        }
+
+        public void Shrink(int freq)
+        {
+            var i = 0;
+            while (i < featureset_dict_.Count)
+            {
+                if (featureset_dict_.ValueList[i].Value < freq)
+                {
+                    //If the feature's frequency is less than specific frequency, drop the feature.
+                    featureset_dict_.RemoveAt(i);
+                }
+                else
+                {
+                    i++;
+                }
+            }
+        }
+
+        public void GenerateLexicalIdList(out IList<string> keyList, out IList<int> valList)
+        {
+            keyList = featureset_dict_.KeyList;
+            var fixArrayValue = new int[Size];
+            valList = fixArrayValue;
+
+            Parallel.For(0, featureset_dict_.ValueList.Count, parallelOption, i =>
+            {
+                fixArrayValue[i] = (int)featureset_dict_.ValueList[i].Key;
+            });
+
+        }
+
+        public long RegenerateFeatureId(CRFLite.Utils.BTreeDictionary<long, long> old2new, long ysize)
+        {
+            long new_maxid = 0;
+            //Regenerate new feature id and create feature ids mapping
+            foreach (var it in featureset_dict_)
+            {
+                var strFeature = it.Key;
+                //Regenerate new feature id
+                old2new.Add(it.Value.Key, new_maxid);
+                it.Value.Key = new_maxid;
+
+                var addValue = (strFeature[0] == 'U' ? ysize : ysize * ysize);
+                new_maxid += addValue;
+            }
+
+            return new_maxid;
+        }
+
+        //Get feature id from feature set by feature string
+        //If feature string is not existed in the set, generate a new id and return it
+        private long GetId(string key)
+        {
+            FeatureIdPair pair;
+            if (featureset_dict_.TryGetValue(key, out pair) == true)
+            {
+                return pair.Key;
+            }
+
+            return BaseUtils.ERROR_INVALIDATED_FEATURE;
+        }
+
+        public long GetOrAddId(string key)
+        {
+            FeatureIdPair pair;
+            if (featureset_dict_.TryGetValue(key, out pair) == true && pair != null)
+            {
+                //Find its feature id
+                System.Threading.Interlocked.Increment(ref pair.Value);
+            }
+            else
+            {
+                lock (thisLock)
+                {
+                    if (featureset_dict_.TryGetValue(key, out pair) == true)
+                    {
+                        System.Threading.Interlocked.Increment(ref pair.Value);
+                    }
+                    else
+                    {
+                        var oldValue = Interlocked.Increment(ref maxid_) - 1;
+                        pair = new FeatureIdPair(oldValue, 1);
+                        featureset_dict_.Add(key, pair);
+                    }
+                }
+            }
+            return pair.Key;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/EncoderOptions.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public class EncoderOptions
+    {
+        /// <summary>
+        /// Maximum iteration
+        /// </summary>
+        public int MaxIteration { get; set; }
+
+        /// <summary>
+        /// Minimum feature frequency, if one feature's frequency is less than this value, the feature will be dropped.
+        /// </summary>
+        public int MinFeatureFreq = 2;
+
+        /// <summary>
+        /// Minimum diff value, when diff less than the value consecutive 3 times, the process will be ended.
+        /// </summary>
+        public double MinDifference;
+
+        /// <summary>
+        /// The maximum slot usage rate threshold when building feature set.
+        /// </summary>
+        public double SlotUsageRateThreshold { get; set; }
+
+        /// <summary>
+        /// The amount of threads used to train model.
+        /// </summary>
+        public int ThreadsNum { get; set; }
+
+        /// <summary>
+        /// Regularization type
+        /// </summary>
+        public CRFEncoder.REG_TYPE RegType { get; set; }
+
+        /// <summary>
+        /// Template file name
+        /// </summary>
+        [Required]
+        public string TemplateFileName { get; set; }
+
+        /// <summary>
+        /// Training corpus file name
+        /// </summary>
+        [Required]
+        public string TrainingCorpusFileName { get; set; }
+
+        /// <summary>
+        /// Encoded model file name
+        /// </summary>
+        [Required]
+        public string ModelFileName { get; set; }
+
+        /// <summary>
+        /// The model file name for re-training
+        /// </summary>
+        public string RetrainModelFileName { get; set; }
+
+        /// <summary>
+        /// Debug level
+        /// </summary>
+        public int DebugLevel { get; set; }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public uint HugeLexMemLoad = 0;
+
+        /// <summary>
+        /// cost factor, too big or small value may lead encoded model over tune or under tune
+        /// </summary>
+        public double CostFactor { get; set; }
+
+        /// <summary>
+        /// If we build vector quantization model for feature weights
+        /// </summary>
+        public bool BVQ { get; set; }
+
+        public EncoderOptions()
+        {
+            MaxIteration = 100;
+            MinFeatureFreq = 2;
+            MinDifference = 0.0001;
+            SlotUsageRateThreshold = 0.95;
+            ThreadsNum = 1;
+            RegType = CRFEncoder.REG_TYPE.L2;
+            CostFactor = 1.0;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/EncoderTagger.cs

@@ -0,0 +1,198 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Security.Cryptography;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public class EncoderTagger : Tagger
+    {
+        public ModelWriter feature_index_;
+        public short[] answer_;
+
+        public int eval(int[,] merr)
+        {
+            var err = 0;
+            for (var i = 0; i < word_num; ++i)
+            {
+                if (answer_[i] != result_[i])
+                {
+                    ++err;
+                    merr[answer_[i], result_[i]]++;
+                }
+            }
+            return err;
+        }
+
+        public EncoderTagger(ModelWriter modelWriter)
+        {
+            feature_index_ = modelWriter;
+            ysize_ = (short)feature_index_.ysize();
+        }
+
+        public bool GenerateFeature(List<List<string>> recordList)
+        {
+            word_num = (short)recordList.Count;
+            if (word_num == 0)
+            {
+                return false;
+            }
+
+            //Try to find each record's answer tag
+            var x_num = 0;
+            var xsize = (int)feature_index_.xsize_;
+            answer_ = new short[word_num];
+            for (int index = 0; index < recordList.Count; index++)
+            {
+                var record = recordList[index];
+//get result tag's index and fill answer
+                for (short k = 0; k < ysize_; ++k)
+                {
+                    if (feature_index_.y(k) == record[xsize])
+                    {
+                        answer_[x_num] = k;
+                        break;
+                    }
+                }
+                x_num++;
+            }
+
+            //Build record feature set
+            x_ = recordList;
+            Z_ = 0.0;
+            feature_cache_ = new List<long[]>();
+            feature_index_.BuildFeatures(this);
+            x_ = null;
+
+            return true;
+        }
+
+        private void LockFreeAdd(double[] expected, long exp_offset, double addValue)
+        {
+            double initialValue;
+            double newValue;
+            do
+            {
+                initialValue = expected[exp_offset]; // read current value
+                newValue = initialValue + addValue;  //calculate new value
+            }
+            while (initialValue != Interlocked.CompareExchange(ref expected[exp_offset], newValue, initialValue));
+        }
+
+        private void calcExpectation(int x, int y, double[] expected)
+        {
+            var n = node_[x, y];
+            var c = Math.Exp(n.alpha + n.beta - n.cost - Z_);
+            var offset = y + 1; //since expected array is based on 1
+            for (int index = 0; index < feature_cache_[n.fid].Length; index++)
+            {
+                var item = feature_cache_[n.fid][index];
+                LockFreeAdd(expected, item + offset, c);
+            }
+
+            for (int index = 0; index < n.lpathList.Count; index++)
+            {
+                var p = n.lpathList[index];
+                c = Math.Exp(p.lnode.alpha + p.cost + p.rnode.beta - Z_);
+                offset = p.lnode.y * ysize_ + p.rnode.y + 1; //since expected array is based on 1
+                for (int i = 0; i < feature_cache_[p.fid].Length; i++)
+                {
+                    var item = feature_cache_[p.fid][i];
+                    LockFreeAdd(expected, item + offset, c);
+                }
+            }
+        }
+
+        public double gradient(double[] expected)
+        {
+            buildLattice();
+            forwardbackward();
+            var s = 0.0;
+
+            for (var i = 0; i < word_num; ++i)
+            {
+                for (var j = 0; j < ysize_; ++j)
+                {
+                    calcExpectation(i, j, expected);
+                }
+            }
+
+            for (var i = 0; i < word_num; ++i)
+            {
+                var answer_val = answer_[i];
+                var answer_Node = node_[i, answer_val];
+                var offset = answer_val + 1; //since expected array is based on 1
+                for (int index = 0; index < feature_cache_[answer_Node.fid].Length; index++)
+                {
+                    var fid = feature_cache_[answer_Node.fid][index];
+                    LockFreeAdd(expected, fid + offset, -1.0f);
+                }
+                s += answer_Node.cost;  // UNIGRAM cost
+
+
+                for (int index = 0; index < answer_Node.lpathList.Count; index++)
+                {
+                    var lpath = answer_Node.lpathList[index];
+                    if (lpath.lnode.y == answer_[lpath.lnode.x])
+                    {
+                        offset = lpath.lnode.y * ysize_ + lpath.rnode.y + 1;
+                        for (int index1 = 0; index1 < feature_cache_[lpath.fid].Length; index1++)
+                        {
+                            var fid = feature_cache_[lpath.fid][index1];
+                            LockFreeAdd(expected, fid + offset, -1.0f);
+                        }
+
+                        s += lpath.cost; // BIGRAM COST
+                        break;
+                    }
+                }
+            }
+
+            viterbi();  // call for eval()
+            return Z_ - s;
+        }
+
+        public void Init(short[] result, Node[,] node)
+        {
+            result_ = result;
+            node_ = node;
+        }
+
+
+
+
+        public void buildLattice()
+        {
+            RebuildFeatures();
+            for (var i = 0; i < word_num; ++i)
+            {
+                for (var j = 0; j < ysize_; ++j)
+                {
+                    var node_i_j = node_[i, j];
+                    node_i_j.cost = calcCost(node_i_j.fid, j);
+                    for (int index = 0; index < node_i_j.lpathList.Count; index++)
+                    {
+                        var p = node_i_j.lpathList[index];
+                        var offset = p.lnode.y * ysize_ + p.rnode.y;
+                        p.cost = calcCost(p.fid, offset);
+                    }
+                }
+            }
+        }
+
+        public double calcCost(int featureListIdx, int offset)
+        {
+            double c = 0.0f;
+            offset++; //since alpha_ array is based on 1
+            for (int index = 0; index < feature_cache_[featureListIdx].Length; index++)
+            {
+                var fid = feature_cache_[featureListIdx][index];
+                c += feature_index_.alpha_[fid + offset];
+            }
+            return feature_index_.cost_factor_ * c;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/FeatureIdPair.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public sealed class FeatureIdPair
+    {
+        public long Key;
+        public int Value;
+
+        public FeatureIdPair(long key, int value)
+        {
+            Key = key;
+            Value = value;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/FeatureItem.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public sealed class FeatureItem : IComparable<FeatureItem>
+    {
+        public string strFeature;
+        public FeatureIdPair feaIdPair;
+
+        public FeatureItem(string s, FeatureIdPair item)
+        {
+            strFeature = s;
+            feaIdPair = item;
+        }
+
+        public int CompareTo(FeatureItem fi)
+        {
+            return StringComparer.Ordinal.Compare(strFeature, fi.strFeature);
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/HugeFeatureLexicalDict.cs

@@ -0,0 +1,305 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
+    public class MEMORYSTATUSEX
+    {
+        public uint dwLength;
+        public uint dwMemoryLoad;
+        public ulong ullTotalPhys;
+        public ulong ullAvailPhys;
+        public ulong ullTotalPageFile;
+        public ulong ullAvailPageFile;
+        public ulong ullTotalVirtual;
+        public ulong ullAvailVirtual;
+        public ulong ullAvailExtendedVirtual;
+        public MEMORYSTATUSEX()
+        {
+            this.dwLength = (uint)Marshal.SizeOf(typeof(MEMORYSTATUSEX));
+        }
+    }
+
+    public sealed class FeatureFreq : IComparable<FeatureFreq>
+    {
+        public string strFeature;
+        public long value;
+
+        public int CompareTo(FeatureFreq fi)
+        {
+            return StringComparer.Ordinal.Compare(strFeature, fi.strFeature);
+        }
+    }
+
+    public sealed class HugeFeatureLexicalDict : IFeatureLexicalDict
+    {
+        CRFLite.Utils.VarBigArray<FeatureFreq> arrayFeatureFreq;
+        long arrayFeatureFreqSize;
+        uint SHRINK_AVALI_MEM_LOAD;
+        CRFLite.Utils.MD5 md5;
+        ParallelOptions parallelOption;
+
+        [return: MarshalAs(UnmanagedType.Bool)]
+        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
+        static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer);
+
+        public HugeFeatureLexicalDict(int thread_num, uint shrinkMemLoad)
+        {
+            SHRINK_AVALI_MEM_LOAD = shrinkMemLoad;
+            arrayFeatureFreq = new CRFLite.Utils.VarBigArray<FeatureFreq>(1024 * 1024);
+            arrayFeatureFreqSize = 0;
+            md5 = new CRFLite.Utils.MD5();
+            parallelOption = new ParallelOptions();
+            parallelOption.MaxDegreeOfParallelism = thread_num;
+        }
+
+        public void Clear()
+        {
+            arrayFeatureFreq.Clear();
+            arrayFeatureFreq = null;
+        }
+
+
+        public CRFLite.Utils.VarBigArray<FeatureFreq> featureFreq
+        {
+            get
+            {
+                return arrayFeatureFreq;
+            }
+        }
+
+        public long Size
+        {
+            get
+            {
+                return arrayFeatureFreqSize;
+            }
+        }
+
+        private long ParallelMerge(long startIndex, long endIndex, int freq)
+        {
+            var sizePerThread = (endIndex - startIndex + 1) / parallelOption.MaxDegreeOfParallelism;
+            //Fistly, merge items in each block by parallel
+            Parallel.For(0, parallelOption.MaxDegreeOfParallelism, parallelOption, i =>
+            {
+                Merge(startIndex + i * sizePerThread, startIndex + (i + 1) * sizePerThread - 1, 0);
+            });
+
+            //Secondly, merge all items
+            return Merge(startIndex, endIndex, freq);
+        }
+
+        private void ForceCollectMemory()
+        {
+            GC.Collect();
+            GC.WaitForPendingFinalizers();
+            GC.Collect();
+        }
+
+        //Merge same items in sorted list
+        private long Merge(long startIndex, long endIndex, int freq)
+        {
+            var newEndIndex = startIndex;
+
+            //Try to find first not null item
+            while ((arrayFeatureFreq[startIndex] == null) &&
+                startIndex <= endIndex)
+            {
+                startIndex++;
+            }
+            arrayFeatureFreq[newEndIndex] = arrayFeatureFreq[startIndex];
+            for (var i = startIndex + 1; i <= endIndex; i++)
+            {
+                if (arrayFeatureFreq[i] == null)
+                {
+                    continue;
+                }
+
+                if (arrayFeatureFreq[newEndIndex].strFeature == arrayFeatureFreq[i].strFeature)
+                {
+                    //two same items, sum their value up
+                    arrayFeatureFreq[newEndIndex].value += arrayFeatureFreq[i].value;
+                    arrayFeatureFreq[i] = null;
+                }
+                else
+                {
+                    //two different items
+                    if (arrayFeatureFreq[newEndIndex].value >= freq)
+                    {
+                        newEndIndex++;
+                    }
+
+                    arrayFeatureFreq[newEndIndex] = arrayFeatureFreq[i];
+                    if (newEndIndex < i)
+                    {
+                        arrayFeatureFreq[i] = null;
+                    }
+                }
+            }
+
+            return newEndIndex;
+        }
+
+        //Generate feature string and its id list
+        public void GenerateLexicalIdList(out IList<string> keyList, out IList<int> valList)
+        {
+            var fixArrayKey = new CRFLite.Utils.FixedBigArray<string>(Size, 0);
+            keyList = fixArrayKey;
+
+            var fixArrayValue = new CRFLite.Utils.FixedBigArray<int>(Size, 0);
+            valList = fixArrayValue;
+            Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
+            {
+                fixArrayKey[i] = arrayFeatureFreq[i].strFeature;
+                fixArrayValue[i] = (int)(arrayFeatureFreq[i].value);
+            });
+        }
+
+        Object thisLock = new object();
+        //Generate feature id by NGram rules
+        public long RegenerateFeatureId(CRFLite.Utils.BTreeDictionary<long, long> old2new, long ysize)
+        {
+            long maxid_ = 0;
+            Parallel.For(0, arrayFeatureFreqSize, parallelOption, i =>
+            {
+                //Generate new feature id
+                var addValue = (arrayFeatureFreq[i].strFeature[0] == 'U' ? ysize : ysize * ysize);
+                var oldValue = maxid_;
+                while (System.Threading.Interlocked.CompareExchange(ref maxid_, oldValue + addValue, oldValue) != oldValue)
+                {
+                    oldValue = maxid_;
+                }
+
+                //Create existed and new feature ids mapping
+                lock (thisLock)
+                {
+                    old2new.Add(
+                        GetId(arrayFeatureFreq[i].strFeature),
+                        oldValue);
+                }
+
+                arrayFeatureFreq[i].value = oldValue;
+            });
+            return maxid_;
+        }
+
+        //Shrink entire list
+        public void Shrink(int freq)
+        {
+            var newEndIndex = Shrink(0, arrayFeatureFreqSize - 1, freq);
+            arrayFeatureFreqSize = newEndIndex + 1;
+        }
+
+        //Shrink item list
+        private long Shrink(long startIndex, long endIndex, int freq)
+        {
+            Console.Write("Sorting...");
+            arrayFeatureFreq.Sort(startIndex, endIndex - startIndex + 1, parallelOption.MaxDegreeOfParallelism);
+            Console.Write("Merging...");
+
+            var newEndIndex = ParallelMerge(startIndex, endIndex, freq);
+            sortedEndIndex = newEndIndex;
+
+            Console.WriteLine("Done!");
+            ForceCollectMemory();
+
+            return newEndIndex;
+        }
+
+        //Get feature string id
+        private long GetId(string strFeature)
+        {
+            var rawbytes = Encoding.UTF8.GetBytes(strFeature);
+            
+            lock (thisLock)
+            {
+                return md5.Compute64BitHash(rawbytes);
+            }
+        }
+
+        private long sortedEndIndex = 0;
+        private int ShrinkingLock = 0;
+        private int AddLock = 0;
+        //Add the feature string into list and get feature string id
+        public long GetOrAddId(string strFeature)
+        {
+            while (ShrinkingLock == 1) { Thread.Sleep(5000); }
+
+            //add item-adding lock
+            Interlocked.Increment(ref AddLock);
+
+            var newFFItem = new FeatureFreq();
+            newFFItem.strFeature = strFeature;
+            newFFItem.value = 1;
+            if (sortedEndIndex > 0)
+            {
+                var ff = arrayFeatureFreq.BinarySearch(0, sortedEndIndex, newFFItem);
+                if (ff != null)
+                {
+                    Interlocked.Increment(ref ff.value);
+                    //free item-adding lock
+                    Interlocked.Decrement(ref AddLock);
+                    return GetId(strFeature);
+                }
+            }
+
+            var oldValue = Interlocked.Increment(ref arrayFeatureFreqSize) - 1;
+            arrayFeatureFreq[oldValue] = newFFItem;
+
+            //free item-adding lock
+            Interlocked.Decrement(ref AddLock);
+
+            //Check whether shrink process should be started
+            uint memoryLoad = 0;
+            if (oldValue % 10000000 == 0)
+            {
+                var msex = new MEMORYSTATUSEX();
+                GlobalMemoryStatusEx(msex);
+                memoryLoad = msex.dwMemoryLoad;
+            }
+
+            if (memoryLoad >= SHRINK_AVALI_MEM_LOAD)
+            {
+                if (Interlocked.CompareExchange(ref ShrinkingLock, 1, 0) == 0)
+                {
+                    //Double check whether shrink should be started
+                    var msex = new MEMORYSTATUSEX();
+                    GlobalMemoryStatusEx(msex);
+                    if (msex.dwMemoryLoad >= SHRINK_AVALI_MEM_LOAD)
+                    {
+                        while (AddLock != 0) { Thread.Sleep(1000); }
+
+                        var startDT = DateTime.Now;
+                        Console.WriteLine("Begin to shrink [Feature Size: {0}]...", arrayFeatureFreqSize);
+                        var newArrayFeatureFreqSize = Shrink(0, arrayFeatureFreqSize - 1, 0) + 1;
+
+                        GlobalMemoryStatusEx(msex);
+                        if (msex.dwMemoryLoad >= SHRINK_AVALI_MEM_LOAD - 1)
+                        {
+                            //Still have enough available memory, raise shrink threshold
+                            SHRINK_AVALI_MEM_LOAD = msex.dwMemoryLoad + 1;
+                            if (SHRINK_AVALI_MEM_LOAD >= 100)
+                            {
+                                //if use more than 100% memory, the performance will extremely reduce
+                                SHRINK_AVALI_MEM_LOAD = 100;
+                            }
+                        }
+
+                        arrayFeatureFreqSize = newArrayFeatureFreqSize;
+                        var ts = DateTime.Now - startDT;
+                        Console.WriteLine("Shrink has been done!");
+                        Console.WriteLine("[Feature Size:{0}, TimeSpan:{1}, Next Shrink Rate:{2}%]", arrayFeatureFreqSize, ts, SHRINK_AVALI_MEM_LOAD);
+                    }
+
+                    Interlocked.Decrement(ref ShrinkingLock);
+                }
+            }
+            return GetId(strFeature);
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/IFeatureLexicalDict.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public interface IFeatureLexicalDict
+    {
+        void Shrink(int freq);
+        long GetOrAddId(string strFeature);
+        long RegenerateFeatureId(CRFLite.Utils.BTreeDictionary<long, long> old2new, long ysize);
+        void GenerateLexicalIdList(out IList<string> fea, out IList<int> val);
+        void Clear();
+
+        long Size
+        {
+            get;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/LBFGS.cs

@@ -0,0 +1,349 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public class LBFGS
+    {
+        double [] diag;
+        CRFLite.Utils.FixedBigArray<double> w;
+        Mcsrch mcsrch_;
+        long nfev, point, npt, iter, info, ispt, iypt;
+        int iflag_;
+        double stp;
+        public int zeroone;
+        public int err;
+        public double obj;
+
+        public double[] expected;
+        public double[] v;
+        public double[] xi;
+
+        private ParallelOptions parallelOption;
+
+        public LBFGS(int thread_num)
+        {
+            iflag_ = 0; nfev = 0;
+            point = 0; npt = 0; iter = 0; info = 0;
+            ispt = 0; iypt = 0;
+            stp = 0.0;
+            mcsrch_ = new Mcsrch(thread_num);
+
+            parallelOption = new ParallelOptions();
+            parallelOption.MaxDegreeOfParallelism = thread_num;
+        }
+
+        private double ddot_(long size, CRFLite.Utils.FixedBigArray<double> dx, long dx_idx, CRFLite.Utils.FixedBigArray<double> dy, long dy_idx)
+        {
+            double ret = 0.0f;
+            Parallel.For<double>(0, size, parallelOption, () => 0, (i, loop, subtotal) =>
+            {
+                subtotal += dx[i + dx_idx] * dy[i + dy_idx];
+                return subtotal;
+            },
+            (subtotal) => // lock free accumulator
+            {
+                double initialValue;
+                double newValue;
+                do
+                {
+                    initialValue = ret; // read current value
+                    newValue = initialValue + subtotal;  //calculate new value
+                }
+                while (initialValue != Interlocked.CompareExchange(ref ret, newValue, initialValue));
+            });
+            return ret;
+        }
+
+
+        private double ddot_(long size, double[] dx, long dx_idx, double[] dy, long dy_idx)
+        {
+            double ret = 0.0f;
+            Parallel.For<double>(0, size, parallelOption, () => 0, (i, loop, subtotal) =>
+            {
+                subtotal += dx[i + dx_idx] * dy[i + dy_idx];
+                return subtotal;
+            },
+            (subtotal) => // lock free accumulator
+            {
+                double initialValue;
+                double newValue;
+                do
+                {
+                    initialValue = ret; // read current value
+                    newValue = initialValue + subtotal;  //calculate new value
+                }
+                while (initialValue != Interlocked.CompareExchange(ref ret, newValue, initialValue));
+            });
+            return ret;
+        }
+        void pseudo_gradient(double[] x, double C)
+        {
+            var size = expected.LongLength - 1;
+            Parallel.For(1, size + 1, parallelOption, i =>
+            {
+                if (x[i] == 0)
+                {
+                    if (expected[i] + C < 0)
+                    {
+                        v[i] = (expected[i] + C);
+                    }
+                    else if (expected[i] - C > 0)
+                    {
+                        v[i] = (expected[i] - C);
+                    }
+                    else
+                    {
+                        v[i] = 0;
+                    }
+                }
+                else
+                {
+                    v[i] = (expected[i] + C * sigma(x[i]));
+                }
+            });
+        }
+        double sigma(double x)
+        {
+            if (x > 0) return 1.0;
+            else if (x < 0) return -1.0;
+            return 0.0;
+        }
+
+        public int optimize(double[] x, double C, bool orthant)
+        {
+            const long msize = 5;
+            var size = x.LongLength - 1;
+            if (w == null || w.LongLength == 0)
+            {
+                iflag_ = 0;
+                w = new CRFLite.Utils.FixedBigArray<double>(size * (2 * msize + 1) + 2 * msize, 1);
+                diag = new double[size + 1];
+                if (orthant == true)
+                {
+                    xi = new double[size + 1];
+                    v = new double[size + 1];
+                }
+            }
+
+            if (orthant == true)
+            {
+                pseudo_gradient(x, C);
+            }
+            else
+            {
+                v = expected;
+            }
+
+            lbfgs_optimize(msize, x, orthant, C);
+            if (iflag_ < 0)
+            {
+                Console.WriteLine("routine stops with unexpected error");
+                return -1;
+            }
+
+            return iflag_;
+        }
+
+        void lbfgs_optimize(long msize, double[] x, bool orthant, double C)
+        {
+            var size = x.LongLength - 1;
+            var yy = 0.0;
+            var ys = 0.0;
+            long bound = 0;
+            long cp = 0;
+            var bExit = false;
+
+            // initialization
+            if (iflag_ == 0)
+            {
+                point = 0;
+                ispt = size + (msize << 1);
+                iypt = ispt + size * msize;
+
+                Parallel.For(1, size + 1, parallelOption, i =>
+                {
+                    diag[i] = 1.0f;
+                    w[ispt + i] = -v[i];
+                    w[i] = expected[i];
+                });
+
+                if (orthant == true)
+                {
+                    Parallel.For(1, size + 1, parallelOption, i =>
+                    {
+                        xi[i] = (x[i] != 0 ? sigma(x[i]) : sigma(-v[i]));
+                    });
+                }
+
+                //第一次试探步长
+                stp = 1.0f / Math.Sqrt(ddot_(size, v, 1, v, 1));
+
+                ++iter;
+                info = 0;
+                nfev = 0;
+            }
+
+            // MAIN ITERATION LOOP
+            bExit = LineSearchAndUpdateStepGradient(msize, x, orthant);
+            while (bExit == false)
+            {
+                ++iter;
+                info = 0;
+
+                if (orthant == true)
+                {
+                    Parallel.For(1, size + 1, parallelOption, i =>
+                    {
+                        xi[i] = (x[i] != 0 ? sigma(x[i]) : sigma(-v[i]));
+                    });
+                }
+
+                if (iter > size)
+                {
+                    bound = size;
+                }
+
+                // COMPUTE -H*G USING THE FORMULA GIVEN IN: Nocedal, J. 1980,
+                // "Updating quasi-Newton matrices with limited storage",
+                // Mathematics of Computation, Vol.24, No.151, pp. 773-782.
+                ys = ddot_(size, w, iypt + npt + 1, w, ispt + npt + 1);
+                yy = ddot_(size, w, iypt + npt + 1, w, iypt + npt + 1);
+
+                var r_ys_yy = ys / yy;
+                Parallel.For(1, size + 1, parallelOption, i =>
+                {
+                    diag[i] = r_ys_yy;
+                    w[i] = -v[i];
+                });
+
+                cp = point;
+                if (point == 0)
+                {
+                    cp = msize;
+                }
+
+                w[size + cp] = (1.0 / ys);
+                //回退次数
+                bound = Math.Min(iter - 1, msize);
+                cp = point;
+                for (var i = 1; i <= bound; ++i)
+                {
+                    --cp;
+                    if (cp == -1) cp = msize - 1;
+                    var sq = ddot_(size, w, ispt + cp * size + 1, w, 1);
+                    var inmc = size + msize + cp + 1;
+                    var iycn = iypt + cp * size;
+                    w[inmc] = (w[size + cp + 1] * sq);
+                    var d = -w[inmc];
+
+                    Parallel.For(1, size + 1, parallelOption, j =>
+                        {
+                            w[j] = (w[j] + d * w[iycn + j]);
+                        });
+                }
+
+                Parallel.For(1, size + 1, parallelOption, i =>
+                {
+                    w[i] = (diag[i] * w[i]);
+                });
+
+                for (var i = 1; i <= bound; ++i)
+                {
+                    var yr = ddot_(size, w, iypt + cp * size + 1, w, 1);
+                    var beta = w[size + cp + 1] * yr;
+                    var inmc = size + msize + cp + 1;
+                    beta = w[inmc] - beta;
+                    var iscn = ispt + cp * size;
+
+                    Parallel.For(1, size + 1, parallelOption, j =>
+                        {
+                            w[j] = (w[j] + beta * w[iscn + j]);
+                        });
+
+                    ++cp;
+                    if (cp == msize)
+                    {
+                        cp = 0;
+                    }
+                }
+
+                if (orthant == true)
+                {
+                    Parallel.For(1, size + 1, parallelOption, i =>
+                    {
+                        w[i] = (sigma(w[i]) == sigma(-v[i]) ? w[i] : 0);
+                    });
+                }
+
+
+                // STORE THE NEW SEARCH DIRECTION
+                var offset = ispt + point * size;
+                Parallel.For(1, size + 1, parallelOption, i =>
+                {
+                    w[offset + i] = w[i];
+                    w[i] = expected[i];
+                });
+
+                stp = 1.0f;
+                nfev = 0;
+                bExit = LineSearchAndUpdateStepGradient(msize, x, orthant);
+            }
+        }
+
+        private bool LineSearchAndUpdateStepGradient(long msize, double[] x, bool orthant)
+        {
+            var size = x.LongLength - 1;
+            var bExit = false;
+            mcsrch_.mcsrch(x, obj, v, w, ispt + point * size,
+                            ref stp, ref info, ref nfev, diag);
+            if (info == -1)
+            {
+                if (orthant == true)
+                {
+                    Parallel.For(1, size + 1, parallelOption, i =>
+                    {
+                        x[i] = (sigma(x[i]) == sigma(xi[i]) ? x[i] : 0);
+                    });
+                }
+
+
+                iflag_ = 1;  // next value
+                bExit = true;
+            }
+            else if (info != 1)
+            {
+                //MCSRCH error, please see error code in info
+                iflag_ = -1;
+                bExit = true;
+            }
+            else
+            {
+                // COMPUTE THE NEW STEP AND GRADIENT CHANGE
+                npt = point * size;
+                Parallel.For(1, size + 1, parallelOption, i =>
+                {
+                    w[ispt + npt + i] = (stp * w[ispt + npt + i]);
+                    w[iypt + npt + i] = expected[i] - w[i];
+                });
+
+                ++point;
+                if (point == msize)
+                {
+                    point = 0;
+                }
+
+                var gnorm = Math.Sqrt(ddot_(size, v, 1, v, 1));
+                var xnorm = Math.Max(1.0, Math.Sqrt(ddot_(size, x, 1, x, 1)));
+                if (gnorm / xnorm <= BaseUtils.eps)
+                {
+                    iflag_ = 0;  // OK terminated
+                    bExit = true;
+                }
+            }
+
+            return bExit;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/Mcsrch.cs

@@ -0,0 +1,461 @@
+using BotSharp.MachineLearning.CRFLite.Utils;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    class Mcsrch
+    {
+        private int infoc;
+        private bool stage1, brackt;
+        private double dginit;
+        private double width, width1;
+        private double fx, dgx, fy, dgy;
+        private double finit;
+        private double dgtest;
+        private double stx, sty;
+        private double stmin, stmax;
+
+        private ParallelOptions parallelOption;
+
+        public Mcsrch(int thread_num)
+        {
+            infoc = 0;
+            stage1 = false;
+            brackt = false;
+            finit = 0.0;
+            dginit = 0.0;
+            dgtest = 0.0;
+            width = 0.0;
+            width1 = 0.0;
+            stx = 0.0;
+            fx = 0.0;
+            dgx = 0.0;
+            sty = 0.0;
+            fy = 0.0;
+            dgy = 0.0;
+            stmin = 0.0;
+            stmax = 0.0;
+
+            parallelOption = new ParallelOptions();
+            parallelOption.MaxDegreeOfParallelism = thread_num;
+        }
+
+
+
+        void mcstep(ref double stx, ref double fx, ref double dx,
+            ref double sty, ref double fy, ref double dy,
+            ref double stp, double fp, double dp,
+            ref bool brackt,
+            double stpmin, double stpmax,
+            ref int info)
+        {
+            var bound = true;
+            double p, q, d3, r, stpq, stpc, stpf;
+            double gamma;
+            double s;
+            double d1, d2;
+            double theta;
+            info = 0;
+
+            if (brackt == true && ((stp <= Math.Min(stx, sty) || stp >= Math.Max(stx, sty)) ||
+                            dx * (stp - stx) >= 0.0 || stpmax < stpmin))
+            {
+                return;
+            }
+
+            var sgnd = dp * (dx / Math.Abs(dx));
+            if (fp > fx)
+            {
+                info = 1;
+                bound = true;
+                theta = (fx - fp) * 3 / (stp - stx) + dx + dp;
+                d1 = Math.Abs(theta);
+                d2 = Math.Abs(dx);
+                d1 = Math.Max(d1, d2);
+                d2 = Math.Abs(dp);
+                s = Math.Max(d1, d2);
+                d1 = theta / s;
+                gamma = s * Math.Sqrt(d1 * d1 - dx / s * (dp / s));
+                if (stp < stx)
+                {
+                    gamma = -gamma;
+                }
+                p = gamma - dx + theta;
+                q = gamma - dx + gamma + dp;
+                r = p / q;
+                stpc = stx + r * (stp - stx);
+                stpq = stx + dx / ((fx - fp) /
+                                     (stp - stx) + dx) / 2 * (stp - stx);
+                d1 = stpc - stx;
+                d2 = stpq - stx;
+                if (Math.Abs(d1) < Math.Abs(d2))
+                {
+                    stpf = stpc;
+                }
+                else
+                {
+                    stpf = stpc + (stpq - stpc) / 2;
+                }
+                brackt = true;
+            }
+            else if (sgnd < 0.0)
+            {
+                info = 2;
+                bound = false;
+                theta = (fx - fp) * 3 / (stp - stx) + dx + dp;
+                d1 = Math.Abs(theta);
+                d2 = Math.Abs(dx);
+                d1 = Math.Max(d1, d2);
+                d2 = Math.Abs(dp);
+                s = Math.Max(d1, d2);
+                d1 = theta / s;
+                gamma = s * Math.Sqrt(d1 * d1 - dx / s * (dp / s));
+                if (stp > stx)
+                {
+                    gamma = -gamma;
+                }
+                p = gamma - dp + theta;
+                q = gamma - dp + gamma + dx;
+                r = p / q;
+                stpc = stp + r * (stx - stp);
+                stpq = stp + dp / (dp - dx) * (stx - stp);
+
+                d1 = stpc - stp;
+                d2 = stpq - stp;
+                if (Math.Abs(d1) > Math.Abs(d2))
+                {
+                    stpf = stpc;
+                }
+                else
+                {
+                    stpf = stpq;
+                }
+                brackt = true;
+            }
+            else if (Math.Abs(dp) < Math.Abs(dx))
+            {
+                info = 3;
+                bound = true;
+                theta = (fx - fp) * 3 / (stp - stx) + dx + dp;
+                d1 = Math.Abs(theta);
+                d2 = Math.Abs(dx);
+                d1 = Math.Max(d1, d2);
+                d2 = Math.Abs(dp);
+                s = Math.Max(d1, d2);
+                d3 = theta / s;
+                d1 = 0.0f;
+                d2 = d3 * d3 - dx / s * (dp / s);
+                gamma = s * Math.Sqrt((Math.Max(d1, d2)));
+                if (stp > stx)
+                {
+                    gamma = -gamma;
+                }
+                p = gamma - dp + theta;
+                q = gamma + (dx - dp) + gamma;
+                r = p / q;
+                if (r < 0.0 && gamma != 0.0)
+                {
+                    stpc = stp + r * (stx - stp);
+                }
+                else if (stp > stx)
+                {
+                    stpc = stpmax;
+                }
+                else
+                {
+                    stpc = stpmin;
+                }
+                stpq = stp + dp / (dp - dx) * (stx - stp);
+                if (brackt == true)
+                {
+                    d1 = stp - stpc;
+                    d2 = stp - stpq;
+                    if (Math.Abs(d1) < Math.Abs(d2))
+                    {
+                        stpf = stpc;
+                    }
+                    else
+                    {
+                        stpf = stpq;
+                    }
+                }
+                else
+                {
+                    d1 = stp - stpc;
+                    d2 = stp - stpq;
+                    if (Math.Abs(d1) > Math.Abs(d2))
+                    {
+                        stpf = stpc;
+                    }
+                    else
+                    {
+                        stpf = stpq;
+                    }
+                }
+            }
+            else
+            {
+                info = 4;
+                bound = false;
+                if (brackt == true)
+                {
+                    theta = (fp - fy) * 3 / (sty - stp) + dy + dp;
+                    d1 = Math.Abs(theta);
+                    d2 = Math.Abs(dy);
+                    d1 = Math.Max(d1, d2);
+                    d2 = Math.Abs(dp);
+                    s = Math.Max(d1, d2);
+                    d1 = theta / s;
+                    gamma = s * Math.Sqrt(d1 * d1 - dy / s * (dp / s));
+                    if (stp > sty)
+                    {
+                        gamma = -gamma;
+                    }
+                    p = gamma - dp + theta;
+                    q = gamma - dp + gamma + dy;
+                    r = p / q;
+                    stpc = stp + r * (sty - stp);
+                    stpf = stpc;
+                }
+                else if (stp > stx)
+                {
+                    stpf = stpmax;
+                }
+                else
+                {
+                    stpf = stpmin;
+                }
+            }
+
+            if (fp > fx)
+            {
+                sty = stp;
+                fy = fp;
+                dy = dp;
+            }
+            else
+            {
+                if (sgnd < 0.0)
+                {
+                    sty = stx;
+                    fy = fx;
+                    dy = dx;
+                }
+                stx = stp;
+                fx = fp;
+                dx = dp;
+            }
+
+            stpf = Math.Min(stpmax, stpf);
+            stpf = Math.Max(stpmin, stpf);
+            stp = stpf;
+            if (brackt == true && bound)
+            {
+                if (sty > stx)
+                {
+                    d1 = stx + (sty - stx) * 0.66;
+                    stp = Math.Min(d1, stp);
+                }
+                else
+                {
+                    d1 = stx + (sty - stx) * 0.66;
+                    stp = Math.Max(d1, stp);
+                }
+            }
+
+            return;
+        }
+
+
+
+        const double lb3_1_gtol = 0.9;
+        const double xtol = 1e-16;
+        const double lb3_1_stpmin = 1e-20;
+        const double lb3_1_stpmax = 1e20;
+        const double ftol = 1e-4;
+        const double p5 = 0.5;
+        const double p66 = 0.66;
+        const double xtrapf = 4.0;
+        const int maxfev = 20;
+
+        private double ddot_(long size, double[] dx, long dx_idx, FixedBigArray<double> dy, long dy_idx)
+        {
+            double ret = 0.0f;
+            Parallel.For<double>(0, size, parallelOption, () => 0, (i, loop, subtotal) =>
+            {
+                subtotal += dx[i + dx_idx] * dy[i + dy_idx];
+                return subtotal;
+            },
+            (subtotal) => // lock free accumulator
+            {
+                double initialValue;
+                double newValue;
+                do
+                {
+                    initialValue = ret; // read current value
+                    newValue = initialValue + subtotal;  //calculate new value
+                }
+                while (initialValue != Interlocked.CompareExchange(ref ret, newValue, initialValue));
+            });
+            return ret;
+        }
+
+        public void mcsrch(double[] x, double f, double[] g, FixedBigArray<double> s, long s_idx,
+            ref double stp, ref long info, ref long nfev, double[] wa)
+        {
+            var size = x.LongLength - 1;
+            /* Parameter adjustments */
+            if (info == -1)
+            {
+                info = 0;
+                nfev++;
+
+                var dg = ddot_(size, g, 1, s, s_idx + 1);
+                var ftest1 = finit + stp * dgtest;
+
+                if (brackt && ((stp <= stmin || stp >= stmax) || infoc == 0))
+                {
+                    info = 6;
+                    Console.WriteLine("MCSRCH warning: Rounding errors prevent further progress.There may not be a step which satisfies the sufficient decrease and curvature conditions. Tolerances may be too small.");
+                    Console.WriteLine("bracket: {0}, stp:{1}, stmin:{2}, stmax:{3}, infoc:{4}", brackt, stp, stmin, stmax, infoc);
+                }
+                if (stp == lb3_1_stpmax && f <= ftest1 && dg <= dgtest)
+                {
+                    info = 5;
+                    Console.WriteLine("MCSRCH warning: The step is too large.");
+                }
+                if (stp == lb3_1_stpmin && (f > ftest1 || dg >= dgtest))
+                {
+                    info = 4;
+                    Console.WriteLine("MCSRCH warning: The step is too small.");
+                    Console.WriteLine("stp:{0}, lb3_1_stpmin:{1}, f:{2}, ftest1:{3}, dg:{4}, dgtest:{5}", stp, lb3_1_stpmin, f, ftest1, dg, dgtest);
+                }
+                if (nfev >= maxfev)
+                {
+                    info = 3;
+                    Console.WriteLine("MCSRCH warning: More than {0} function evaluations were required at the present iteration.", maxfev);
+                }
+                if (brackt && stmax - stmin <= xtol * stmax)
+                {
+                    info = 2;
+                    Console.WriteLine("MCSRCH warning: Relative width of the interval of uncertainty is at most xtol.");
+                }
+                if (f <= ftest1 && Math.Abs(dg) <= lb3_1_gtol * (-dginit))
+                {
+                    info = 1;
+                }
+
+                if (info != 0)
+                {
+                    return;
+                }
+
+                if (stage1 && f <= ftest1 && dg >= Math.Min(ftol, lb3_1_gtol) * dginit)
+                {
+                    stage1 = false;
+                }
+
+                if (stage1 && f <= fx && f > ftest1)
+                {
+                    var fm = f - stp * dgtest;
+                    var fxm = fx - stx * dgtest;
+                    var fym = fy - sty * dgtest;
+                    var dgm = dg - dgtest;
+                    var dgxm = dgx - dgtest;
+                    var dgym = dgy - dgtest;
+                    mcstep(ref stx, ref fxm, ref dgxm, ref sty, ref fym, ref dgym, ref stp, fm, dgm, ref brackt,
+                           stmin, stmax, ref infoc);
+                    fx = fxm + stx * dgtest;
+                    fy = fym + sty * dgtest;
+                    dgx = dgxm + dgtest;
+                    dgy = dgym + dgtest;
+                }
+                else
+                {
+                    mcstep(ref stx, ref fx, ref dgx, ref sty, ref fy, ref dgy, ref stp, f, dg, ref brackt,
+                           stmin, stmax, ref infoc);
+                }
+
+                if (brackt)
+                {
+                    var d1 = 0.0;
+                    d1 = sty - stx;
+                    if (Math.Abs(d1) >= p66 * width1)
+                    {
+                        stp = stx + p5 * (sty - stx);
+                    }
+                    width1 = width;
+                    d1 = sty - stx;
+                    width = Math.Abs(d1);
+                }
+            }
+            else
+            {
+                infoc = 1;
+                if (size <= 0 || stp <= 0.0)
+                {
+                    return;
+                }
+
+                dginit = ddot_(size, g, 1, s, s_idx + 1);
+                if (dginit >= 0.0)
+                {
+                    return;
+                }
+
+                brackt = false;
+                stage1 = true;
+                nfev = 0;
+                finit = f;
+                dgtest = ftol * dginit;
+                width = lb3_1_stpmax - lb3_1_stpmin;
+                width1 = width / p5;
+
+                Parallel.For(1, size + 1, parallelOption, i =>
+                {
+                    wa[i] = x[i];
+                }
+                );
+
+                stx = 0.0;
+                fx = finit;
+                dgx = dginit;
+                sty = 0.0;
+                fy = finit;
+                dgy = dginit;
+            }
+
+            if (brackt)
+            {
+                stmin = Math.Min(stx, sty);
+                stmax = Math.Max(stx, sty);
+            }
+            else
+            {
+                stmin = stx;
+                stmax = stp + xtrapf * (stp - stx);
+            }
+
+            stp = Math.Max(stp, lb3_1_stpmin);
+            stp = Math.Min(stp, lb3_1_stpmax);
+
+            if ((brackt && ((stp <= stmin || stp >= stmax) ||
+                            nfev >= maxfev - 1 || infoc == 0)) ||
+                (brackt && (stmax - stmin <= xtol * stmax)))
+            {
+                stp = stx;
+            }
+
+            var stp_t = stp;
+            Parallel.For(1, size + 1, parallelOption, i =>
+            {
+                x[i] = (wa[i] + stp_t * s[s_idx + i]);
+            });
+
+            info = -1;
+        }
+
+    }
+}

BotSharp.MachineLearning/CRFLite/Encoder/ModelWriter.cs

@@ -0,0 +1,405 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.MachineLearning.CRFLite.Decoder;
+
+namespace BotSharp.MachineLearning.CRFLite.Encoder
+{
+    public class ModelWriter : BaseModel
+    {
+        private readonly string modelFileName;
+
+        private readonly Pool<StringBuilder> _buildersPool =
+            new Pool<StringBuilder>(p => new StringBuilder(100), b => b.Clear());
+
+
+        int thread_num_;
+        public IFeatureLexicalDict featureLexicalDict;
+        List<List<List<string>>> trainCorpusList;
+        ParallelOptions parallelOption = new ParallelOptions();
+
+        public ModelWriter(int thread_num, double cost_factor,
+            uint hugeLexShrinkMemLoad, string modelFileName)
+        {
+            cost_factor_ = cost_factor;
+            maxid_ = 0;
+            thread_num_ = thread_num;
+            this.modelFileName = modelFileName;
+            parallelOption.MaxDegreeOfParallelism = thread_num;
+
+            if (hugeLexShrinkMemLoad > 0)
+            {
+                featureLexicalDict = new HugeFeatureLexicalDict(thread_num_, hugeLexShrinkMemLoad);
+            }
+            else
+            {
+                featureLexicalDict = new DefaultFeatureLexicalDict(thread_num_);
+            }
+        }
+
+        //Regenerate feature id and shrink features with lower frequency
+        public void Shrink(EncoderTagger[] xList, int freq)
+        {
+            var old2new = new CRFLite.Utils.BTreeDictionary<long, long>();
+            featureLexicalDict.Shrink(freq);
+            maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
+            var feature_count = xList.Length;
+
+            //Update feature ids
+            Parallel.For(0, feature_count, parallelOption, i =>
+            {
+                for (var j = 0; j < xList[i].feature_cache_.Count; j++)
+                {
+                    var newfs = new List<long>();
+                    long rstValue = 0;
+                    for (int index = 0; index < xList[i].feature_cache_[j].Length; index++)
+                    {
+                        var v = xList[i].feature_cache_[j][index];
+                        if (old2new.TryGetValue(v, out rstValue) == true)
+                        {
+                            newfs.Add(rstValue);
+                        }
+                    }
+                    xList[i].feature_cache_[j] = newfs.ToArray();
+                }
+            });
+        }
+
+        // Load all records and generate features
+        public EncoderTagger[] ReadAllRecords()
+        {
+            var arrayEncoderTagger = new EncoderTagger[trainCorpusList.Count];
+            var arrayEncoderTaggerSize = 0;
+
+            //Generate each record features
+            Parallel.For(0, trainCorpusList.Count, parallelOption, i =>
+            {
+                var _x = new EncoderTagger(this);
+                if (_x.GenerateFeature(trainCorpusList[i]) == false)
+                {
+                }
+                else
+                {
+                    var oldValue = Interlocked.Increment(ref arrayEncoderTaggerSize) - 1;
+                    arrayEncoderTagger[oldValue] = _x;
+
+                    if (oldValue % 10000 == 0)
+                    {
+                        //Show current progress on console
+                        Console.Write("{0}...", oldValue);
+                    }
+                }
+            });
+
+            trainCorpusList.Clear();
+            trainCorpusList = null;
+
+            Console.WriteLine();
+            return arrayEncoderTagger;
+        }
+
+        //Open and check training and template file
+        public bool Open(string strTemplateFileName, string strTrainCorpusFileName)
+        {
+            return OpenTemplateFile(strTemplateFileName) && OpenTrainCorpusFile(strTrainCorpusFileName);
+        }
+
+        //Build feature set into indexed data
+        public bool BuildFeatureSetIntoIndex(string filename, double max_slot_usage_rate_threshold, int debugLevel)
+        {
+            IList<string> keyList;
+            IList<int> valList;
+            featureLexicalDict.GenerateLexicalIdList(out keyList, out valList);
+
+            if (debugLevel > 0)
+            {
+                var filename_featureset_raw_format = filename + ".feature.raw_text";
+                var sw = new StreamWriter(filename_featureset_raw_format);
+                // save feature and its id into lists in raw format
+                for (var i = 0; i < keyList.Count; i++)
+                {
+                    sw.WriteLine("{0}\t{1}", keyList[i], valList[i]);
+                }
+                sw.Close();
+            }
+
+            //Build feature index
+            var filename_featureset = filename + ".feature";
+            var da = new CRFLite.Utils.DoubleArrayTrieBuilder(thread_num_);
+            if (da.build(keyList, valList, max_slot_usage_rate_threshold) == false)
+            {
+                return false;
+            }
+            //Save indexed feature set into file
+            da.save(filename_featureset);
+
+            if (string.IsNullOrWhiteSpace(modelFileName))
+            {
+                //Clean up all data
+                featureLexicalDict.Clear();
+                featureLexicalDict = null;
+                keyList = null;
+                valList = null;
+
+                GC.Collect();
+
+                //Create weight matrix
+                alpha_ = new double[feature_size() + 1];
+            }
+            else
+            {
+                //Create weight matrix
+                alpha_ = new double[feature_size() + 1];
+                var modelReader = new ModelReader(this.modelFileName);
+                modelReader.LoadModel();
+
+                if (modelReader.y_.Count == y_.Count)
+                {
+                    for (var i = 0; i < keyList.Count; i++)
+                    {
+                        var index = modelReader.get_id(keyList[i]);
+                        if (index < 0)
+                        {
+                            continue;
+                        }
+                        var size = (keyList[i][0] == 'U' ? y_.Count : y_.Count * y_.Count);
+                        for (var j = 0; j < size; j++)
+                        {
+                            alpha_[valList[i] + j + 1] = modelReader.GetAlpha(index + j);
+                        }
+                    }
+                }
+                else
+                {
+                }
+
+                //Clean up all data
+                featureLexicalDict.Clear();
+                featureLexicalDict = null;
+                keyList = null;
+                valList = null;
+
+                GC.Collect();
+            }
+
+            return true;
+        }
+
+        //Save model meta data into file
+        public bool SaveModelMetaData(string filename)
+        {
+            var tofs = new StreamWriter(filename);
+
+            // header
+            tofs.WriteLine("version: " + BaseUtils.MODEL_TYPE_NORM);
+            tofs.WriteLine("cost-factor: " + cost_factor_);
+            tofs.WriteLine("maxid: " + maxid_);
+            tofs.WriteLine("xsize: " + xsize_);
+
+            tofs.WriteLine();
+
+            // y
+            for (var i = 0; i < y_.Count; ++i)
+            {
+                tofs.WriteLine(y_[i]);
+            }
+            tofs.WriteLine();
+
+            // template
+            for (var i = 0; i < unigram_templs_.Count; ++i)
+            {
+                tofs.WriteLine(unigram_templs_[i]);
+            }
+            for (var i = 0; i < bigram_templs_.Count; ++i)
+            {
+                tofs.WriteLine(bigram_templs_[i]);
+            }
+
+            tofs.Close();
+
+            return true;
+        }
+
+        /// <summary>
+        /// Save feature weights into file
+        /// </summary>
+        /// <param name="filename"></param>
+        /// <param name="bVQ"></param>
+        /// <returns></returns>
+        public void SaveFeatureWeight(string filename, bool bVQ)
+        {
+            var filename_alpha = filename + ".alpha";
+            var tofs = new StreamWriter(filename_alpha, false);
+            var bw = new BinaryWriter(tofs.BaseStream);
+
+            if (bVQ == true)
+            {
+                //Build code book
+                CRFLite.Utils.VectorQuantization vq = new CRFLite.Utils.VectorQuantization();
+                for (long i = 1; i <= maxid_; i++)
+                {
+                    vq.Add(alpha_[i]);
+                }
+
+                int vqSize = 256;
+                double distortion = vq.BuildCodebook(vqSize);
+
+                //VQ size
+                bw.Write(vqSize);
+
+                //Save VQ codebook into file
+                for (int j = 0; j < vqSize; j++)
+                {
+                    bw.Write(vq.CodeBook[j]);
+                }
+
+                //Save weights
+                for (long i = 1; i <= maxid_; ++i)
+                {
+                    bw.Write((byte)vq.ComputeVQ(alpha_[i]));
+                }
+            }
+            else
+            {
+                bw.Write(0);
+                //Save weights
+                for (long i = 1; i <= maxid_; ++i)
+                {
+                    bw.Write((float)alpha_[i]);
+                }
+            }
+
+            bw.Close();
+        }
+
+        bool OpenTemplateFile(string filename)
+        {
+            var ifs = new StreamReader(filename);
+            unigram_templs_ = new List<string>();
+            bigram_templs_ = new List<string>();
+            while (ifs.EndOfStream == false)
+            {
+                var line = ifs.ReadLine();
+                if (line.Length == 0 || line[0] == '#')
+                {
+                    continue;
+                }
+                if (line[0] == 'U')
+                {
+                    unigram_templs_.Add(line);
+                }
+                else if (line[0] == 'B')
+                {
+                    bigram_templs_.Add(line);
+                }
+                else
+                {
+                }
+            }
+            ifs.Close();
+            return true;
+        }
+
+        bool OpenTrainCorpusFile(string strTrainingCorpusFileName)
+        {
+            var ifs = new StreamReader(strTrainingCorpusFileName);
+            y_ = new List<string>();
+            trainCorpusList = new List<List<List<string>>>();
+            var hashCand = new HashSet<string>();
+            var recordList = new List<List<string>>();
+
+            var last_xsize = -1;
+            while (ifs.EndOfStream == false)
+            {
+                var line = ifs.ReadLine();
+                if (line.Length == 0 || line[0] == ' ' || line[0] == '\t')
+                {
+                    //Current record is finished, save it into the list
+                    if (recordList.Count > 0)
+                    {
+                        trainCorpusList.Add(recordList);
+                        recordList = new List<List<string>>();
+                    }
+                    continue;
+                }
+
+                var items = line.Split('\t');
+                var size = items.Length;
+                if (last_xsize >= 0 && last_xsize != size)
+                {
+                    return false;
+                }
+                last_xsize = size;
+                xsize_ = (uint)(size - 1);
+                recordList.Add(new List<string>(items));
+
+                if (hashCand.Contains(items[items.Length - 1]) == false)
+                {
+                    hashCand.Add(items[items.Length - 1]);
+                    y_.Add(items[items.Length - 1]);
+                }
+            }
+            ifs.Close();
+
+            return true;
+        }
+
+        //Get feature id from feature set by feature string
+        //If feature string is not existed in the set, generate a new id and return it
+        public bool BuildFeatures(EncoderTagger tagger)
+        {
+            var feature = new List<long>();
+            using (var v = _buildersPool.GetOrCreate())
+            {
+                var localBuilder = v.Item;
+                //tagger.feature_id_ = tagger.feature_cache_.Count;
+                for (var cur = 0; cur < tagger.word_num; ++cur)
+                {
+                    for (int index = 0; index < unigram_templs_.Count; index++)
+                    {
+                        var it = unigram_templs_[index];
+                        var strFeature = apply_rule(it, cur, localBuilder, tagger);
+                        if (strFeature == null)
+                        {
+                        }
+                        else
+                        {
+                            var id = featureLexicalDict.GetOrAddId(strFeature.ToString());
+                            feature.Add(id);
+                        }
+                    }
+                    tagger.feature_cache_.Add(feature.ToArray());
+                    feature.Clear();
+                }
+
+                for (var cur = 1; cur < tagger.word_num; ++cur)
+                {
+                    for (int index = 0; index < bigram_templs_.Count; index++)
+                    {
+                        var it = bigram_templs_[index];
+                        var strFeature = apply_rule(it, cur, localBuilder, tagger);
+                        if (strFeature == null)
+                        {
+                        }
+                        else
+                        {
+                            var id = featureLexicalDict.GetOrAddId(strFeature.ToString());
+                            feature.Add(id);
+                        }
+                    }
+
+                    tagger.feature_cache_.Add(feature.ToArray());
+                    feature.Clear();
+
+                }
+
+            }
+
+            return true;
+        }
+
+    }
+}

BotSharp.MachineLearning/CRFLite/IGenerateFeature.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public interface IGenerateFeature
+    {
+        bool Initialize();
+        List<List<string>> GenerateFeature(string strText);
+    }
+}

BotSharp.MachineLearning/CRFLite/Node.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class Node
+    {
+        public int fid;
+        public short x;
+        public short y;
+        public double alpha;
+        public double beta;
+        public double cost;
+        public double bestCost;
+        public Node prev;
+
+        public List<Path> lpathList;
+        public List<Path> rpathList;
+    }
+}

BotSharp.MachineLearning/CRFLite/Path.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class Path
+    {
+        public int fid;
+        public Node rnode;
+        public Node lnode;
+        public double cost;
+
+        public Path()
+        {
+            rnode = null;
+            lnode = null;
+            cost = 0;
+        }
+
+        public void add(Node _lnode, Node _rnode)
+        {
+            lnode = _lnode;
+            rnode = _rnode;
+
+            lnode.rpathList.Add(this);
+            rnode.lpathList.Add(this);
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Pool.cs

@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    /// <summary>
+    /// Represents general purpose pool that has no restrictions (e.g. grows if it's required)
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    internal sealed class Pool<T>
+    {
+        private int _totalCount;
+        private readonly ConcurrentStack<T> _container = new ConcurrentStack<T>();
+        private readonly Func<Pool<T>, T> _creator;
+        private readonly Action<T> _cleaner;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="T:System.Object"/> class.
+        /// </summary>
+        public Pool(Func<Pool<T>, T> creator, Action<T> cleaner = null)
+        {
+            _creator = creator;
+            _cleaner = cleaner;
+        }
+
+        /// <summary>
+        /// Gets item from pool or creates a new item
+        /// </summary>
+        /// <returns></returns>
+        public PoolItem<T> GetOrCreate()
+        {
+            T item;
+            if (_container.TryPop(out item))
+            {
+                return new PoolItem<T>(item, _cleaner, this);
+            }
+            var newItem = _creator(this);
+            if (newItem == null)
+            {
+                throw new ApplicationException("Unable to create new pool item");
+            }
+            Interlocked.Increment(ref _totalCount);
+            return new PoolItem<T>(newItem, _cleaner, this);
+        }
+
+        /// <summary>
+        /// Returns amount of free items in the bag
+        /// </summary>
+        public int FreeCount { get { return _container.Count; } }
+
+        /// <summary>
+        /// Returns amount items created by pool
+        /// </summary>
+        public int TotalCount { get { return _totalCount; } }
+
+        private void Return(T item)
+        {
+            _container.Push(item);
+        }
+
+        /// <summary>
+        /// Pool item that is return when pool request is processed
+        /// </summary>
+        /// <typeparam name="T1"></typeparam>
+        internal struct PoolItem<T1> : IDisposable
+        {
+            /// <summary>
+            /// Pooled item
+            /// </summary>
+            public readonly T1 Item;
+            private readonly Pool<T1> _owner;
+            private readonly Action<T1> _cleaner;
+
+            /// <summary>
+            /// Creates a new pool item
+            /// </summary>
+            /// <param name="item"></param>
+            /// <param name="cleaner"></param>
+            /// <param name="owner"></param>
+            internal PoolItem(T1 item, Action<T1> cleaner, Pool<T1> owner)
+            {
+                Item = item;
+                _cleaner = cleaner;
+                _owner = owner;
+            }
+
+            /// <summary>
+            /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+            /// </summary>
+            public void Dispose()
+            {
+                _cleaner?.Invoke(Item);
+                _owner.Return(Item);
+            }
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/SegDecoderTagger.cs

@@ -0,0 +1,131 @@
+using BotSharp.MachineLearning.CRFLite.Decoder;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class SegDecoderTagger : DecoderTagger
+    {
+        public SegDecoderTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
+            : base(nbest, this_crf_max_word_num)
+        {
+            crf_max_word_num = this_crf_max_word_num;
+        }
+
+        int seg_termbuf_build(crf_seg_out term_buf)
+        {
+            term_buf.Clear();
+
+            //build raw result at first
+            var iRet = termbuf_build(term_buf);
+            if (iRet != BaseUtils.ERROR_SUCCESS)
+            {
+                return iRet;
+            }
+
+            //Then build token result
+            var term_len = 0;
+            var weight = 0.0;
+            var num = 0;
+            for (var i = 0; i < x_.Count; i++)
+            {
+                //Adding the length of current token
+                var strTag = term_buf.result_[i];
+                term_len += x_[i][0].Length;
+                weight += term_buf.weight_[i];
+                num++;
+
+                //Check if current term is the end of a token
+                if ((strTag.StartsWith("B_") == false &&
+                    strTag.StartsWith("M_") == false) ||
+                    i == x_.Count - 1)
+                {
+                    var tkn = new SegToken();
+                    tkn.length = term_len;
+                    tkn.offset = term_buf.termTotalLength;
+
+                    var spos = strTag.IndexOf('_');
+                    if (spos < 0)
+                    {
+                        if (strTag == "NOR")
+                        {
+                            tkn.strTag = "";
+                        }
+                        else
+                        {
+                            tkn.strTag = strTag;
+                        }
+                    }
+                    else
+                    {
+                        tkn.strTag = strTag.Substring(spos + 1);
+                    }
+
+                    term_buf.termTotalLength += term_len;
+                    //Calculate each token's weight
+                    switch (vlevel_)
+                    {
+                        case 0:
+                            tkn.fWeight = 0.0;
+                            break;
+                        case 2:
+                            tkn.fWeight = weight / num;
+                            weight = 0.0;
+                            num = 0;
+                            break;
+                    }
+
+                    term_buf.tokenList.Add(tkn);
+                    term_len = 0;
+                }
+            }
+
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+
+        public int output(crf_seg_out[] pout)
+        {
+            var n = 0;
+            var ret = 0;
+
+            if (nbest_ == 1)
+            {
+                //If only best result and no need probability, "next" is not to be used
+                ret = seg_termbuf_build(pout[0]);
+                if (ret < 0)
+                {
+                    return ret;
+                }
+            }
+            else
+            {
+                //Fill the n best result
+                var iNBest = nbest_;
+                if (pout.Length < iNBest)
+                {
+                    iNBest = pout.Length;
+                }
+
+                for (n = 0; n < iNBest; ++n)
+                {
+                    ret = next();
+                    if (ret < 0)
+                    {
+                        break;
+                    }
+
+                    ret = seg_termbuf_build(pout[n]);
+                    if (ret < 0)
+                    {
+                        return ret;
+                    }
+                }
+            }
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Tagger.cs

@@ -0,0 +1,154 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class Tagger
+    {
+        public List<List<string>> x_;
+        public Node[,] node_; //Node matrix
+        public short ysize_;
+        public short word_num; //the number of tokens need to be labeled
+        public double Z_;  //概率值
+        public double cost_;  //The path cost
+        public short[] result_;
+        public List<long[]> feature_cache_;
+
+        //Calculate the cost of each path. It's used for finding the best or N-best result
+        public int viterbi()
+        {
+            var bestc = double.MinValue;
+            Node bestNode = null;
+
+            for (var i = 0; i < word_num; ++i)
+            {
+                for (var j = 0; j < ysize_; ++j)
+                {
+                    bestc = double.MinValue;
+                    bestNode = null;
+
+                    var node_i_j = node_[i, j];
+
+                    for (int index = 0; index < node_i_j.lpathList.Count; ++index)
+                    {
+                        var p = node_i_j.lpathList[index];
+                        var cost = p.lnode.bestCost + p.cost + node_i_j.cost;
+                        if (cost > bestc)
+                        {
+                            bestc = cost;
+                            bestNode = p.lnode;
+                        }
+                    }
+
+                    node_i_j.prev = bestNode;
+                    node_i_j.bestCost = bestNode != null ? bestc : node_i_j.cost;
+                }
+            }
+
+            bestc = double.MinValue;
+            bestNode = null;
+
+            var s = (short)(word_num - 1);
+            for (short j = 0; j < ysize_; ++j)
+            {
+                if (bestc < node_[s, j].bestCost)
+                {
+                    bestNode = node_[s, j];
+                    bestc = node_[s, j].bestCost;
+                }
+            }
+
+            var n = bestNode;
+            while (n != null)
+            {
+                result_[n.x] = n.y;
+                n = n.prev;
+            }
+
+            cost_ = -node_[s, result_[s]].bestCost;
+
+            return BaseUtils.ERROR_SUCCESS;
+        }
+
+        private void calcAlpha(int m, int n)
+        {
+            var nd = node_[m, n];
+            nd.alpha = 0.0;
+
+            var i = 0;
+            for (int index = 0; index < nd.lpathList.Count; index++)
+            {
+                var p = nd.lpathList[index];
+                nd.alpha = BaseUtils.logsumexp(nd.alpha, p.cost + p.lnode.alpha, (i == 0));
+                i++;
+            }
+            nd.alpha += nd.cost;
+        }
+
+        private void calcBeta(int m, int n)
+        {
+            var nd = node_[m, n];
+            nd.beta = 0.0f;
+            if (m + 1 < word_num)
+            {
+                var i = 0;
+                for (int index = 0; index < nd.rpathList.Count; index++)
+                {
+                    var p = nd.rpathList[index];
+                    nd.beta = BaseUtils.logsumexp(nd.beta, p.cost + p.rnode.beta, (i == 0));
+                    i++;
+                }
+            }
+            nd.beta += nd.cost;
+        }
+
+        public void forwardbackward()
+        {
+            for (int i = 0, k = word_num - 1; i < word_num; ++i, --k)
+            {
+                for (var j = 0; j < ysize_; ++j)
+                {
+                    calcAlpha(i, j);
+                    calcBeta(k, j);
+                }
+            }
+
+            Z_ = 0.0;
+            for (var j = 0; j < ysize_; ++j)
+            {
+                Z_ = BaseUtils.logsumexp(Z_, node_[0, j].beta, j == 0);
+            }
+        }
+
+
+        //Assign feature ids to node and path
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int RebuildFeatures()
+        {
+            var fid = 0;
+            for (short cur = 0; cur < word_num; ++cur)
+            {
+                for (short i = 0; i < ysize_; ++i)
+                {
+                    node_[cur, i].fid = fid;
+                    if (cur > 0)
+                    {
+                        Node previousNode = node_[cur - 1, i];
+                        for (int index = 0; index < previousNode.rpathList.Count; ++index)
+                        {
+                            Path path = previousNode.rpathList[index];
+                            path.fid = fid + word_num - 1;
+                        }
+                    }
+                }
+
+                ++fid;
+            }
+
+            return 0;
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils.cs

@@ -0,0 +1,160 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite
+{
+    public class QueueElement
+    {
+        public Node node;
+        public QueueElement next;
+        public double fx;
+        public double gx;
+    };
+
+    public class Heap
+    {
+        public int capacity;
+        public int elem_size; //size of elem_list
+        public int size;	// size of elem_ptr_list
+        public List<QueueElement> elem_ptr_list;
+        public List<QueueElement> elem_list;
+    };
+
+    public class BaseUtils
+    {
+        public const double eps = 1e-7;
+
+
+        public const int MINUS_LOG_EPSILON = 13;
+        public const int DEFAULT_CRF_MAX_WORD_NUM = 100;
+
+        public const int MODEL_TYPE_NORM = 100;
+
+
+        public const int ERROR_INVALIDATED_FEATURE = -8;
+        public const int ERROR_HEAP_SIZE_TOO_BIG = -7;
+        public const int ERROR_INSERT_HEAP_FAILED = -6;
+        public const int ERROR_EMPTY_FEATURE = -5;
+        public const int ERROR_INVALIDATED_PARAMETER = -4;
+        public const int ERROR_WRONG_STATUS = -3;
+        public const int ERROR_TOO_LONG_WORD = -2;
+        public const int ERROR_UNKNOWN = -1;
+        public const int ERROR_SUCCESS = 0;
+
+        public static Heap heap_init(int max_size)
+        {
+            Heap H;
+
+            H = new Heap();
+            H.capacity = max_size;
+            H.size = 0;
+            H.elem_size = 0;
+
+            H.elem_ptr_list = new List<QueueElement>(max_size + 1);
+            H.elem_list = new List<QueueElement>(max_size + 1);
+
+            for (var z = 0; z < max_size; z++)
+            {
+                H.elem_list.Add(new QueueElement());
+                H.elem_ptr_list.Add(null);
+            }
+            H.elem_list[0].fx = double.MinValue;
+            H.elem_ptr_list.Add(H.elem_list[0]);
+
+            return H;
+        }
+
+        public static QueueElement allc_from_heap(Heap H)
+        {
+            if (H.elem_size >= H.capacity)
+            {
+                return null;
+            }
+            else
+            {
+                return H.elem_list[++H.elem_size];
+            }
+        }
+
+        public static int heap_insert(QueueElement qe, Heap H)
+        {
+            if (H.size >= H.capacity)
+            {
+                return BaseUtils.ERROR_HEAP_SIZE_TOO_BIG;
+            }
+            var i = ++H.size;
+            while (i != 1 && H.elem_ptr_list[i / 2].fx > qe.fx)
+            {
+                H.elem_ptr_list[i] = H.elem_ptr_list[i / 2];  //此时i还没有进行i/2操作		
+                i /= 2;
+            }
+            H.elem_ptr_list[i] = qe;
+            return 0;
+        }
+
+        public static QueueElement heap_delete_min(Heap H)
+        {
+            var min_elem = H.elem_ptr_list[1];  //堆是从第1号元素开始的
+            var last_elem = H.elem_ptr_list[H.size--];
+            int i = 1, ci = 2;
+            while (ci <= H.size)
+            {
+                if (ci < H.size && H.elem_ptr_list[ci].fx > H.elem_ptr_list[ci + 1].fx)
+                {
+                    ci++;
+                }
+                if (last_elem.fx <= H.elem_ptr_list[ci].fx)
+                {
+                    break;
+                }
+                H.elem_ptr_list[i] = H.elem_ptr_list[ci];
+                i = ci;
+                ci *= 2;
+            }
+            H.elem_ptr_list[i] = last_elem;
+            return min_elem;
+        }
+
+        public static bool is_heap_empty(Heap H)
+        {
+            return H.size == 0;
+        }
+
+        public static void heap_reset(Heap H)
+        {
+            if (H != null)
+            {
+                H.size = 0;
+                H.elem_size = 0;
+            }
+        }
+
+        public static double logsumexp(double x, double y, bool flg)
+        {
+            if (flg)
+            {
+                return y;  // init mode
+            }
+            double vmin;
+            double vmax;
+            if (x > y)
+            {
+                vmin = y;
+                vmax = x;
+            }
+            else
+            {
+                vmin = x;
+                vmax = y;
+            }
+
+            if (vmax > vmin + MINUS_LOG_EPSILON)
+            {
+                return vmax;
+            }
+            return vmax + Math.Log(Math.Exp(vmin - vmax) + 1.0);
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils/BigArray.cs

@@ -0,0 +1,433 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+using System.Threading.Tasks;
+#endif
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    abstract public class BigArray<T> : IList<T> where T : IComparable<T>
+    {
+        public const long sizePerBlock = 1024 * 1024 * 64; //(<<26bits)
+        public const int moveBit = 26;
+        public long size_;
+        public List<T[]> arrList;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+        private ParallelOptions parallelOption;
+        private LimitedConcurrencyLevelTaskScheduler lcts;
+#endif
+        public BigArray()
+        {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+            parallelOption = new ParallelOptions();
+#endif
+        }
+
+        public int IndexOf(T item)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Insert(int index, T item)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void RemoveAt(int index)
+        {
+            throw new NotImplementedException();
+        }
+
+        public T this[int i]
+        {
+            get
+            {
+                return this[(long)i];
+            }
+            set
+            {
+                this[(long)i] = value;
+            }
+        }
+
+        public abstract T this[long i]
+        {
+            get;
+            set;
+        }
+
+        public void Add(T item)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Clear()
+        {
+            foreach (T[] item in arrList)
+            {
+                Array.Clear(item, 0, item.Length);
+            }
+        }
+
+        public bool Contains(T item)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void CopyTo(T[] array, int arrayIndex)
+        {
+            throw new NotImplementedException();
+        }
+
+        public int Count
+        {
+            get
+            {
+                return (int)LongLength;
+            }
+        }
+
+        public long LongLength
+        {
+            get
+            {
+                return size_;
+            }
+        }
+
+        public bool IsReadOnly
+        {
+            get { throw new NotImplementedException(); }
+        }
+
+        public bool Remove(T item)
+        {
+            throw new NotImplementedException();
+        }
+
+        public IEnumerator<T> GetEnumerator()
+        {
+            throw new NotImplementedException();
+        }
+
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+        {
+            throw new NotImplementedException();
+        }
+
+
+        void swap(long pos1, long pos2)
+        {
+            int nBlock1 = (int)(pos1 >> moveBit);
+            int offset1 = (int)(pos1 & (sizePerBlock - 1));
+
+            int nBlock2 = (int)(pos2 >> moveBit);
+            int offset2 = (int)(pos2 & (sizePerBlock - 1));
+
+            T tmp = arrList[nBlock1][offset1];
+            arrList[nBlock1][offset1] = arrList[nBlock2][offset2];
+            arrList[nBlock2][offset2] = tmp;
+        }
+
+
+        private long med3(long a, long b, long c)
+        {
+            return this[a].CompareTo(this[b]) < 0 ? (this[b].CompareTo(this[c]) < 0 ? b : this[a].CompareTo(this[c]) < 0 ? c : a) : this[b].CompareTo(this[c]) > 0 ? b : this[a].CompareTo(this[c]) > 0 ? c : a;
+        }
+
+        private void vecswap(long a, long b, long n)
+        {
+#if NO_SUPPORT_PARALLEL_LIB
+            for (long i = 0;i < n;i++)
+#else
+            Parallel.For(0, n, parallelOption, i =>
+#endif
+            {
+                int nBlock1 = (int)((a + i) >> moveBit);
+                int offset1 = (int)((a + i) & (sizePerBlock - 1));
+
+                int nBlock2 = (int)((b + i) >> moveBit);
+                int offset2 = (int)((b + i) & (sizePerBlock - 1));
+
+                T tmp = arrList[nBlock1][offset1];
+                arrList[nBlock1][offset1] = arrList[nBlock2][offset2];
+                arrList[nBlock2][offset2] = tmp;
+            }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+            );
+#endif
+        }
+
+        const int INSERT_SORT_THRESHOLD = 7;
+        public void QuickSort(long left, long right)
+        {
+            if (left >= right)
+            {
+                return;
+            }
+
+            //use insert sort to handle small data
+            long len = right - left + 1;
+            if (len < INSERT_SORT_THRESHOLD)
+            {
+                for (long i = left; i <= right; i++)
+                {
+                    T t = this[i];
+                    long j = i;
+                    for (; j > left && this[j - 1].CompareTo(t) > 0; j--)
+                    {
+                        this[j] = this[j - 1];
+                    }
+                    this[j] = t;
+                }
+                return;
+            }
+
+            //Choose the pivot value
+            long mid = left + (len >> 1);
+            if (len > INSERT_SORT_THRESHOLD)
+            {
+                //Split the list into three parts, and find middle value in each part,
+                //and finally, use middle value in above three middle values as pivot.
+                long leftMid = left;
+                long rightMid = right;
+                if (len > 40)
+                {
+                    long size = len / 8;
+                    leftMid = med3(leftMid, leftMid + size, leftMid + 2 * size);
+                    mid = med3(mid - size, mid, mid + size);
+                    rightMid = med3(right - 2 * size, right - size, right);
+                }
+                mid = med3(leftMid, mid, rightMid);
+            }
+
+            T v = this[mid];
+
+            //Scan the list from two directions
+            long pivotLeftSide = left, leftScanIndex = pivotLeftSide;
+            long rightScanIndex = right, pivotRightSide = rightScanIndex;
+
+
+            int leftScanIndexBlock = (int)(leftScanIndex >> moveBit);
+            int leftScanIndexOffset = (int)(leftScanIndex & (sizePerBlock - 1));
+            T[] arrayLeft = arrList[leftScanIndexBlock];
+
+            int rightScanIndexBlock = (int)(rightScanIndex >> moveBit);
+            int rightScanIndexOffset = (int)(rightScanIndex & (sizePerBlock - 1));
+            T[] arrayRight = arrList[rightScanIndexBlock];
+            while (true)
+            {
+                //Try to find item which is bigger than pivot
+                while (leftScanIndex <= rightScanIndex)
+                {
+                    int cmpRst = arrayLeft[leftScanIndexOffset].CompareTo(v);
+                    if (cmpRst > 0)
+                    {
+                        //Found one.
+                        break;
+                    }
+                    else if (cmpRst == 0)
+                    {
+                        //If the item is equal to pivot, exchange it with the item in left-side.
+                        swap(pivotLeftSide++, leftScanIndex);
+                    }
+                    leftScanIndex++;
+
+                    leftScanIndexOffset++;
+                    if (leftScanIndexOffset == sizePerBlock)
+                    {
+                        leftScanIndexOffset = 0;
+                        leftScanIndexBlock++;
+                        if (leftScanIndexBlock == arrList.Count)
+                        {
+                            break;
+                        }
+                        arrayLeft = arrList[leftScanIndexBlock];
+                    }
+                }
+
+                //Try to find item which is smaller than pivot
+                while (rightScanIndex >= leftScanIndex)
+                {
+                    int cmpRst = arrayRight[rightScanIndexOffset].CompareTo(v);
+                    if (cmpRst < 0)
+                    {
+                        //Found one.
+                        break;
+                    }
+                    else if (cmpRst == 0)
+                    {
+                        //If the item is equal to pivot, exchange it with the item in left-side.
+                        swap(rightScanIndex, pivotRightSide--);
+                    }
+                    rightScanIndex--;
+
+                    rightScanIndexOffset--;
+                    if (rightScanIndexOffset < 0)
+                    {
+                        rightScanIndexOffset = (int)(sizePerBlock - 1);
+                        rightScanIndexBlock--;
+                        if (rightScanIndexBlock < 0)
+                        {
+                            break;
+                        }
+                        arrayRight = arrList[rightScanIndexBlock];
+                    }
+                }
+
+                if (leftScanIndex > rightScanIndex)
+                {
+                    //Scan finished
+                    break;
+                }
+
+                //Exchange two found items between pivot
+                T temp = arrayLeft[(int)leftScanIndexOffset];
+                arrayLeft[(int)leftScanIndexOffset] = arrayRight[(int)rightScanIndexOffset];
+                arrayRight[(int)rightScanIndexOffset] = temp;
+
+                leftScanIndex++;
+                rightScanIndex--;
+
+                leftScanIndexOffset++;
+                if (leftScanIndexOffset == sizePerBlock)
+                {
+                    leftScanIndexOffset = 0;
+                    leftScanIndexBlock++;
+                    if (leftScanIndexBlock == arrList.Count)
+                    {
+                        break;
+                    }
+                    arrayLeft = arrList[leftScanIndexBlock];
+                }
+
+                rightScanIndexOffset--;
+                if (rightScanIndexOffset < 0)
+                {
+                    rightScanIndexOffset = (int)(sizePerBlock - 1);
+                    rightScanIndexBlock--;
+                    if (rightScanIndexBlock < 0)
+                    {
+                        break;
+                    }
+                    arrayRight = arrList[rightScanIndexBlock];
+                }
+            }
+
+            //Continue to sort two sub-sections
+            long splitIndexLeft = leftScanIndex - pivotLeftSide;
+            long splitIndexRight = pivotRightSide - rightScanIndex;
+            if (splitIndexLeft > 1 && splitIndexRight > 1)
+            {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+                Parallel.Invoke(parallelOption,
+                    () =>
+#endif
+                    {
+                        //exchange items with same value into middle of the list
+                        long size = Math.Min(pivotLeftSide - left, leftScanIndex - pivotLeftSide);
+                        vecswap(left, leftScanIndex - size, size);
+                        QuickSort(left, splitIndexLeft + left - 1);
+                    }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+                    ,
+                    () =>
+#endif
+                    {
+                        //exchange items with same value into middle of the list
+                        long size = Math.Min(pivotRightSide - rightScanIndex, right - pivotRightSide);
+                        vecswap(leftScanIndex, right - size + 1, size);
+                        QuickSort(right - splitIndexRight + 1, right);
+                    }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+                );
+#endif
+            }
+            else
+            {
+                //exchange items with same value into middle of the list
+                long size = Math.Min(pivotLeftSide - left, leftScanIndex - pivotLeftSide);
+                vecswap(left, leftScanIndex - size, size);
+
+                size = Math.Min(pivotRightSide - rightScanIndex, right - pivotRightSide);
+                vecswap(leftScanIndex, right - size + 1, size);
+
+                if (splitIndexLeft > 1)
+                {
+                    QuickSort(left, splitIndexLeft + left - 1);
+                }
+
+                if (splitIndexRight > 1)
+                {
+                    QuickSort(right - splitIndexRight + 1, right);
+                }
+            }
+        }
+
+
+
+        public void Sort(long startIndex, long size, int threadnum = -1)
+        {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+            parallelOption.MaxDegreeOfParallelism = threadnum;
+            if (threadnum > 0)
+            {
+                lcts = new LimitedConcurrencyLevelTaskScheduler(threadnum * 2);
+                parallelOption.TaskScheduler = lcts;
+            }
+#endif
+            QuickSort(startIndex, startIndex + size - 1);
+        }
+
+        public void Sort(int threadnum = -1)
+        {
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+            parallelOption.MaxDegreeOfParallelism = threadnum;
+            if (threadnum > 0)
+            {
+                lcts = new LimitedConcurrencyLevelTaskScheduler(threadnum * 2);
+                parallelOption.TaskScheduler = lcts;
+            }
+#endif
+            QuickSort(0, Count - 1);
+        }
+
+
+        public T BinarySearch(long low, long high, T goal)
+        {
+            long mid = 0;
+
+            while (low <= high)
+            {
+                mid = (high + low) / 2;
+                if (this[mid].CompareTo(goal) == 0)
+                {
+                    return this[mid];
+                }
+                else if (this[mid].CompareTo(goal) > 0)
+                {
+                    high = mid - 1;
+                }
+                else
+                {
+                    low = mid + 1;
+                }
+            }
+            return default(T);
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils/DoubleArrayTrie.cs

@@ -0,0 +1,631 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using System.Threading;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+using System.Threading.Tasks;
+#endif
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    public class unit_t : IComparable<unit_t>
+    {
+        public int base1;
+        public int check;
+
+        public int CompareTo(unit_t obj)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public struct sunit_t : IComparable<sunit_t>
+    {
+        public int base1;
+        public int check;
+
+        public sunit_t(int b, int c)
+        {
+            base1 = b;
+            check = c;
+        }
+
+        public int CompareTo(sunit_t obj)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public class DoubleArrayTrieSearch
+    {
+#if NO_SUPPORT_VERY_BIG_OBJECT
+        private VarBigArray<sunit_t> array;
+#else
+        private sunit_t[] array;
+#endif
+
+        /// <summary>
+        /// Loads ArrayTrie from file
+        /// </summary>
+        /// <param name="fileName">path to file</param>
+        /// <param name="numberOfElementsInChunk">
+        /// Number of elements (2 int32) in read buffer.
+        /// Default is 2048 (16K buffer size)
+        /// </param>
+        public void Load(string fileName, int numberOfElementsInChunk = 2048)
+        {
+            if(!File.Exists(fileName))
+                throw new FileNotFoundException(
+                    "Please check that the specified file exists", fileName);
+            using (var stream = File.OpenRead(fileName))
+                Load(stream, numberOfElementsInChunk);
+        }
+
+        /// <summary>
+        /// Loads ArrayTrie from an arbitrary <see cref="Stream"/>.
+        /// <paramref name="sourceStream"/> is closed and
+        /// disposed once this method completes.
+        /// </summary>
+        /// <param name="sourceStream">
+        /// A <see cref="Stream"/> containing the model.
+        /// </param>
+        /// <param name="numberOfElementsInChunk">
+        /// Number of elements (2 int32) in read buffer. 
+        /// Default is 2048 (16K buffer size)</param>
+        public void Load(Stream sourceStream, int numberOfElementsInChunk = 2048)
+        {
+            const int int32Size = sizeof(int);
+            const int elementSize = int32Size * 2;
+            var fileSizeInBytes = sourceStream.Length;
+            var numberOfElements = fileSizeInBytes / elementSize;
+#if NO_SUPPORT_VERY_BIG_OBJECT
+            array = new VarBigArray<sunit_t>(numberOfElements);
+#else
+            array = new sunit_t[numberOfElements];
+#endif
+            using (var sr = new StreamReader(sourceStream))
+            using (var br = new BinaryReader(sr.BaseStream))
+            {
+                var buffersize = elementSize * numberOfElementsInChunk;
+                var buffer = new byte[elementSize * numberOfElementsInChunk];
+                var index = 0;
+                for (long j = 0; j <= numberOfElements / numberOfElementsInChunk; j++)
+                {
+                    var numberOfReadBytes = br.Read(buffer, 0, buffersize);
+                    if (numberOfReadBytes == buffersize)
+                    {
+                        for (int i = 0; i < numberOfElementsInChunk; i++, index++)
+                        {
+                            var base1 = BitConverter.ToInt32(buffer, elementSize * i);
+                            var check = BitConverter.ToInt32(buffer, (elementSize * i) + int32Size);
+                            array[index] = new sunit_t(base1, check);
+                        }
+                    }
+                    else
+                    {
+                        for (int i = 0; i < numberOfReadBytes / elementSize; i++)
+                        {
+                            var base1 = BitConverter.ToInt32(buffer, elementSize * i);
+                            var check = BitConverter.ToInt32(buffer, (elementSize * i) + int32Size);
+                            array[index++] = new sunit_t(base1, check);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+
+        //Match indexed key which is perfect matched with given string
+        public int SearchByPerfectMatch(string key)
+        {
+            int b = array[0].base1;
+            int p;
+            for (int index = 0; index < key.Length; index++)
+            {
+                char ch = key[index];
+                p = b + ch + 1;
+                if (p >= array.Length)
+                {
+                    return -1;
+                }
+
+                if (b == array[p].check)
+                {
+                    b = array[p].base1;
+                }
+                else
+                {
+                    return -1;
+                }
+            }
+
+            if (b >= array.Length)
+            {
+                return -1;
+            }
+
+            int n = array[b].base1;
+            if (b == array[b].check && n < 0)
+            {
+                return -n - 1;
+            }
+            return -1;
+        }
+
+        /// <summary>
+        /// Match all indexed keys which is prefix of the given string.
+        /// </summary>
+        public int SearchAsKeyPrefix(string key, List<int> result)
+        {
+            int len = key.Length;
+            int b = array[0].base1;
+            int n, p;
+            result.Clear();
+
+            for (int i = 0; i < len; i++)
+            {
+                p = b;
+                if (p >= array.Length)
+                {
+                    return result.Count;
+                }
+
+                n = array[p].base1;
+
+                if (b == array[p].check && n < 0)
+                {
+                    result.Add(-n - 1);
+                }
+
+                p = b + (int)key[i] + 1;
+                if (p >= array.Length)
+                {
+                    return result.Count;
+                }
+
+                if (b == array[p].check)
+                {
+                    b = array[p].base1;
+                }
+                else
+                {
+                    return result.Count;
+                }
+            }
+
+            p = b;
+            if (p >= array.Length)
+            {
+                return result.Count;
+            }
+            n = array[p].base1;
+
+            if (b == array[p].check && n < 0)
+            {
+                result.Add(-n - 1);
+            }
+
+            return result.Count;
+        }
+
+
+        /// <summary>
+        /// Search keys by their prefix string
+        /// </summary>
+        public int SearchByPrefix(string strKeyPrefx, List<int> result)
+        {
+            int b = array[0].base1;
+            int p;
+
+            result.Clear();
+            for (int index = 0; index < strKeyPrefx.Length; index++)
+            {
+                var ch = strKeyPrefx[index];
+                p = b + (int)ch + 1;
+                if (p >= array.Length)
+                {
+                    //The given string isn't existed in the DART
+                    return -1;
+                }
+
+                if (b == array[p].check)
+                {
+                    b = array[p].base1;
+                }
+                else
+                {
+                    return -1;
+                }
+            }
+
+            Queue<int> queue = new Queue<int>();
+            queue.Enqueue(b);
+
+            while (queue.Count > 0)
+            {
+                b = queue.Dequeue();
+                if (b >= array.Length)
+                {
+                    //Invalidated base, skip it
+                    continue;
+                }
+
+                if (b == array[b].check && array[b].base1 < 0)
+                {
+                    result.Add(-array[b].base1 - 1);
+                }
+
+                for (int i = 0; i <= 65535; i++)
+                {
+                    p = b + i + 1;
+                    if (p >= array.Length)
+                    {
+                        //Out of the size of array, skip current search
+                        break;
+                    }
+
+                    if (b == array[p].check)
+                    {
+                        queue.Enqueue(array[p].base1);
+                    }
+                }
+            }
+
+            return result.Count;
+        }
+    }
+
+    public class DoubleArrayTrieBuilder
+    {
+        private VarBigArray<unit_t> array;
+        private VarBigArray<int> used;
+        private IList<string> key_;
+        private IList<int> val_;
+        private int next_chk_pos_;
+        private int progress_;
+        private int thread_num_;
+        private static double MAX_SLOT_USAGE_RATE_THRESHOLD = 0.95;
+        private static double MIN_SLOT_USAGE_RATE_THRESHOLD = 0.05;
+        private double slot_usage_rate_threshold_ = MAX_SLOT_USAGE_RATE_THRESHOLD;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+        private ParallelOptions parallelOption;
+        private LimitedConcurrencyLevelTaskScheduler lcts;
+#endif
+
+        private DateTime startDT;
+        private double lastQPS;
+        private double lastQPSDelta;
+
+        public class Node
+        {
+            public int code;
+            public int depth;
+            public int left;
+            public int right;
+        };
+
+        public DoubleArrayTrieBuilder(int thread_num)
+        {
+            array = null;
+            thread_num_ = thread_num;
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+            lcts = new LimitedConcurrencyLevelTaskScheduler(thread_num_ * 2);
+            parallelOption = new ParallelOptions();
+            parallelOption.TaskScheduler = lcts;
+#endif
+            lastQPS = 0.0;
+            lastQPSDelta = 0.0;
+        }
+
+        int fetch(Node parent, List<Node> siblings)
+        {
+            int prev = 0;
+
+            int i = parent.left;
+            for (int j = parent.left; j < parent.right; j++)
+            {
+                string key = key_[j];
+                if (key.Length < parent.depth)
+                    continue;
+                int cur = 0;
+                if (key.Length != parent.depth)
+                {
+                    cur = ((int)key[parent.depth]) + 1;
+                }
+                if (prev > cur)
+                {
+                    throw new Exception("Fatal: given strings are not sorted.\n");
+                }
+                if (cur != prev || siblings.Count == 0)
+                {
+                    Node tmp_node = new Node();
+                    tmp_node.depth = parent.depth + 1;
+                    tmp_node.code = cur;
+                    tmp_node.left = i;
+                    if (siblings.Count != 0)
+                        siblings[siblings.Count - 1].right = i;
+                    siblings.Add(tmp_node);
+                }
+                prev = cur;
+                i++;
+            }
+            if (siblings.Count != 0)
+                siblings[siblings.Count - 1].right = parent.right;
+            return siblings.Count;
+        }
+
+        int insert(List<Node> siblings)
+        {
+            Random rnd = new Random(DateTime.Now.Millisecond + Thread.CurrentThread.ManagedThreadId);
+            int begin = 0;
+            bool cont = true;
+            int nonzeronum = 0;
+
+            while (used[next_chk_pos_] == 1)
+            {
+                Interlocked.Increment(ref next_chk_pos_);
+            }
+
+            int pos = next_chk_pos_;
+            int startpos = pos;
+
+            //search begin position
+            pos--;
+            while (cont == true)
+            {
+                pos++;
+                if (used[pos] == 0)
+                {
+                    //Check whether slots are available, if not go on to search,
+                    cont = false;
+                    foreach (Node n in siblings)
+                    {
+                        if (used[pos + n.code] == 1 || array[pos + n.code] != null)
+                        {
+                            cont = true;
+                            break;
+                        }
+                    }
+                }
+                else
+                {
+                    nonzeronum++;
+                }
+            }
+            begin = pos;
+
+            //check average slot usage rate. If the rate is no less than the threshold, update next_chk_pos_ to 
+            //pos whose slot range has much less conflict.
+            //note that, the higher rate threshold, the higher slot space usage rate, however, the timing-cost for tri-tree build
+            //will also become more higher.
+            if ((double)nonzeronum / (double)(pos - startpos + 1) >= slot_usage_rate_threshold_ &&
+                pos > next_chk_pos_)
+            {
+                System.Threading.Interlocked.Exchange(ref next_chk_pos_, pos);
+            }
+
+            //double check whether slots are available
+            //the reason why double check is because:
+            //1. in entire slots space, conflict rate is different. the conflict rate of array's tail
+            //   is much lower than that of its header and body
+            //2. roll back cost is heavy. So in high conflict rate range, we just check conflict and no other action (first check)
+            //   once we find a availabe range without conflict, we try to allocate memory on this range and double check conflict
+            bool bAllNull;
+            bool bZeroCode = false;
+            foreach (Node n in siblings)
+            {
+                if (n.code == 0)
+                {
+                    bZeroCode = true;
+                    break;
+                }
+            }
+
+            if (bZeroCode == false)
+            {
+                Node sNode = new Node();
+                sNode.code = 0;
+                siblings.Add(sNode);
+            }
+
+            do
+            {
+                bAllNull = true;
+                //Test conflict in multi-threads
+                int cnt = 0;
+                foreach (Node n in siblings)
+                {
+                    int nBlock = (begin + n.code) >> VarBigArray<int>.moveBit;
+                    long offset = (begin + n.code) & (VarBigArray<int>.sizePerBlock - 1);
+
+                    if (used[begin + n.code] == 1 ||
+                        System.Threading.Interlocked.CompareExchange(ref used.arrList[nBlock][offset], 1, 0) != 0)
+                    {
+                        bAllNull = false;
+                        foreach (Node revertNode in siblings.GetRange(0, cnt))
+                        {
+                            used[begin + revertNode.code] = 0;
+                        }
+                        begin += rnd.Next(thread_num_) + 1;
+                        break;
+                    }
+                    cnt++;
+                }
+            } while (bAllNull == false);
+
+            if (bZeroCode == false)
+            {
+                siblings.RemoveAt(siblings.Count - 1);
+            }
+
+#if NO_SUPPORT_PARALLEL_LIB
+            for (int i = 0;i < siblings.Count;i++)
+#else
+            Parallel.For(0, siblings.Count, parallelOption, i =>
+#endif
+            {
+                List<Node> new_siblings = new List<Node>();
+                Node sibling = siblings[i];
+                int offset = begin + sibling.code;
+
+                array[offset] = new unit_t();
+                array[offset].check = begin;
+                if (fetch(sibling, new_siblings) == 0)
+                {
+                    array[offset].base1 = -val_[sibling.left] - 1;
+                    if (Interlocked.Increment(ref progress_) % 10000 == 0)
+                    {
+                        //Try to adjust slot usage rate in order to keep high performance
+                        TimeSpan ts = DateTime.Now - startDT;
+                        double currQPS = progress_ / (ts.TotalSeconds + 1);
+                        double currQPSDelta = currQPS - lastQPS;
+
+                        if (currQPS < lastQPS && currQPSDelta < lastQPSDelta)
+                        {
+                            //Average QPS becomes slow down, need to reduce slot usage rate
+                            slot_usage_rate_threshold_ -= 0.1;
+                            if (slot_usage_rate_threshold_ < MIN_SLOT_USAGE_RATE_THRESHOLD)
+                            {
+                                slot_usage_rate_threshold_ = MIN_SLOT_USAGE_RATE_THRESHOLD;
+                            }
+                        }
+                        else
+                        {
+                            //Average QPS becomes fast, need to add slot usage rate
+                            slot_usage_rate_threshold_ += 0.1;
+                            if (slot_usage_rate_threshold_ > MAX_SLOT_USAGE_RATE_THRESHOLD)
+                            {
+                                slot_usage_rate_threshold_ = MAX_SLOT_USAGE_RATE_THRESHOLD;
+                            }
+                        }
+
+                        lastQPSDelta = currQPSDelta;
+                        lastQPS = currQPS;
+
+                        if (progress_ % 100000 == 0)
+                        {
+                            //Show current progress on console
+                            Console.Write("{0}...", progress_);
+                        }
+                    }
+                }
+                else
+                {
+                    int b = insert(new_siblings);
+                    array[offset].base1 = b;
+                }
+            }
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+);
+#endif
+
+            return begin;
+        }
+
+        void clear()
+        {
+            array = null;
+        }
+
+        public bool build(IDictionary<string, int> keyvalueList, double max_slot_usage_rate_threshold = 0.95)
+        {
+            FixedBigArray<string> keyList = new FixedBigArray<string>(keyvalueList.Count, 0);
+            FixedBigArray<int> valList = new FixedBigArray<int>(keyvalueList.Count, 0);
+            long index = 0;
+            foreach (KeyValuePair<string, int> pair in keyvalueList)
+            {
+                keyList[index] = pair.Key;
+                valList[index] = pair.Value;
+                index++;
+            }
+
+            return build(keyList, valList, max_slot_usage_rate_threshold);
+        }
+
+        public bool build(IList<string> keyList, IList<int> valList, double max_slot_usage_rate_threshold = 0.95)
+        {
+            if (keyList == null)
+            {
+                Console.WriteLine("Key list is empty");
+                return false;
+            }
+            if (valList == null)
+            {
+                Console.WriteLine("Value list is empty");
+                return false;
+            }
+
+            if (keyList.Count != valList.Count)
+            {
+                Console.WriteLine("The size of key list and value list is not equal");
+                return false;
+            }
+
+            for (int i = 0; i < valList.Count; i++)
+            {
+                if (valList[i] <= -1)
+                {
+                    Console.WriteLine("Invalidated value {0} at index {1}", valList[i], i);
+                    return false;
+                }
+            }
+
+            MAX_SLOT_USAGE_RATE_THRESHOLD = max_slot_usage_rate_threshold;
+            slot_usage_rate_threshold_ = max_slot_usage_rate_threshold;
+            progress_ = 0;
+            key_ = keyList;
+            val_ = valList;
+
+            startDT = DateTime.Now;
+            array = new VarBigArray<unit_t>(key_.Count * 5);
+            used = new VarBigArray<int>(key_.Count * 5);
+            array[0] = new unit_t();
+            array[0].base1 = 1;
+            used[0] = 1;
+            next_chk_pos_ = 0;
+            Node root_node = new Node();
+            root_node.left = 0;
+            root_node.right = key_.Count;
+            root_node.depth = 0;
+            List<Node> siblings = new List<Node>();
+            fetch(root_node, siblings);
+            insert(siblings);
+
+            return true;
+        }
+
+        public void save(string file)
+        {
+            StreamWriter sw = new StreamWriter(file);
+            BinaryWriter bw = new BinaryWriter(sw.BaseStream);
+
+            long r_length = array.LongLength;
+            while (array[r_length - 1] == null)
+            {
+                r_length--;
+            }
+
+            for (long i = 0; i < r_length; i++)
+            {
+                if (array[i] == null)
+                {
+                    bw.Write(0);
+                    bw.Write(0);
+                }
+                else
+                {
+                    bw.Write(array[i].base1);
+                    bw.Write(array[i].check);
+                }
+            }
+            bw.Close();
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils/FixedBigArray.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    public sealed class FixedBigArray<T> : BigArray<T> where T : IComparable<T>
+    {
+        public int lowBounding_;
+
+        public override T this[long i]
+        {
+            get
+            {
+                long offset = (i - lowBounding_);
+                int nBlock = (int)(offset >> moveBit);
+                return arrList[nBlock][offset & (sizePerBlock - 1)];
+            }
+            set
+            {
+                long offset = (i - lowBounding_);
+                int nBlock = (int)(offset >> moveBit);
+                arrList[nBlock][offset & (sizePerBlock - 1)] = value;
+            }
+        }
+
+        
+
+        //construct big array
+        //size is array's default length
+        //lowBounding is the lowest bounding of the array
+        public FixedBigArray(long size, int lowBounding)
+        {
+            size_ = size;
+            lowBounding_ = lowBounding;
+            arrList = new List<T[]>();
+
+            for (long i = 0; i < size_; i += sizePerBlock)
+            {
+                if (i + sizePerBlock < size_)
+                {
+                    arrList.Add(new T[sizePerBlock]);
+                }
+                else
+                {
+                    arrList.Add(new T[size_ - i]);
+                }
+            }
+        }
+
+        
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils/LimitedConcurrencyLevelTaskScheduler.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+
+#if NO_SUPPORT_PARALLEL_LIB
+#else
+using System.Threading.Tasks;
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    /// <summary>
+    /// Provides a task scheduler that ensures a maximum concurrency level while
+    /// running on top of the ThreadPool.
+    /// </summary>
+    public class LimitedConcurrencyLevelTaskScheduler : TaskScheduler
+    {
+        /// <summary>Whether the current thread is processing work items.</summary>
+        [ThreadStatic]
+        private static bool _currentThreadIsProcessingItems;
+        /// <summary>The list of tasks to be executed.</summary>
+        private readonly LinkedList<Task> _tasks = new LinkedList<Task>(); // protected by lock(_tasks)
+        /// <summary>The maximum concurrency level allowed by this scheduler.</summary>
+        private readonly int _maxDegreeOfParallelism;
+        /// <summary>Whether the scheduler is currently processing work items.</summary>
+        private int _delegatesQueuedOrRunning = 0; // protected by lock(_tasks)
+
+        /// <summary>
+        /// Initializes an instance of the LimitedConcurrencyLevelTaskScheduler class with the
+        /// specified degree of parallelism.
+        /// </summary>
+        /// <param name="maxDegreeOfParallelism">The maximum degree of parallelism provided by this scheduler.</param>
+        public LimitedConcurrencyLevelTaskScheduler(int maxDegreeOfParallelism)
+        {
+            if (maxDegreeOfParallelism < 1) throw new ArgumentOutOfRangeException("maxDegreeOfParallelism");
+            _maxDegreeOfParallelism = maxDegreeOfParallelism;
+        }
+
+        /// <summary>Queues a task to the scheduler.</summary>
+        /// <param name="task">The task to be queued.</param>
+        protected sealed override void QueueTask(Task task)
+        {
+            // Add the task to the list of tasks to be processed.  If there aren't enough
+            // delegates currently queued or running to process tasks, schedule another.
+            lock (_tasks)
+            {
+                _tasks.AddLast(task);
+                if (_delegatesQueuedOrRunning < _maxDegreeOfParallelism)
+                {
+                    ++_delegatesQueuedOrRunning;
+                    NotifyThreadPoolOfPendingWork();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Informs the ThreadPool that there's work to be executed for this scheduler.
+        /// </summary>
+        private void NotifyThreadPoolOfPendingWork()
+        {
+            ThreadPool.UnsafeQueueUserWorkItem(_ =>
+            {
+                // Note that the current thread is now processing work items.
+                // This is necessary to enable inlining of tasks into this thread.
+                _currentThreadIsProcessingItems = true;
+                try
+                {
+                    // Process all available items in the queue.
+                    while (true)
+                    {
+                        Task item;
+                        lock (_tasks)
+                        {
+                            // When there are no more items to be processed,
+                            // note that we're done processing, and get out.
+                            if (_tasks.Count == 0)
+                            {
+                                --_delegatesQueuedOrRunning;
+                                break;
+                            }
+
+                            // Get the next item from the queue
+                            item = _tasks.First.Value;
+                            _tasks.RemoveFirst();
+                        }
+
+                        // Execute the task we pulled out of the queue
+                        base.TryExecuteTask(item);
+                    }
+                }
+                // We're done processing items on the current thread
+                finally { _currentThreadIsProcessingItems = false; }
+            }, null);
+        }
+
+        /// <summary>Attempts to execute the specified task on the current thread.</summary>
+        /// <param name="task">The task to be executed.</param>
+        /// <param name="taskWasPreviouslyQueued"></param>
+        /// <returns>Whether the task could be executed on the current thread.</returns>
+        protected sealed override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued)
+        {
+            // If this thread isn't already processing a task, we don't support inlining
+            if (!_currentThreadIsProcessingItems) return false;
+
+            // If the task was previously queued, remove it from the queue
+            if (taskWasPreviouslyQueued) TryDequeue(task);
+
+            // Try to run the task.
+            return base.TryExecuteTask(task);
+        }
+
+        /// <summary>Attempts to remove a previously scheduled task from the scheduler.</summary>
+        /// <param name="task">The task to be removed.</param>
+        /// <returns>Whether the task could be found and removed.</returns>
+        protected sealed override bool TryDequeue(Task task)
+        {
+            lock (_tasks) return _tasks.Remove(task);
+        }
+
+        /// <summary>Gets the maximum concurrency level supported by this scheduler.</summary>
+        public sealed override int MaximumConcurrencyLevel { get { return _maxDegreeOfParallelism; } }
+
+        /// <summary>Gets an enumerable of the tasks currently scheduled on this scheduler.</summary>
+        /// <returns>An enumerable of the tasks currently scheduled.</returns>
+        protected sealed override IEnumerable<Task> GetScheduledTasks()
+        {
+            bool lockTaken = false;
+            try
+            {
+                Monitor.TryEnter(_tasks, ref lockTaken);
+                if (lockTaken) return _tasks.ToArray();
+                else throw new NotSupportedException();
+            }
+            finally
+            {
+                if (lockTaken) Monitor.Exit(_tasks);
+            }
+        }
+    }
+}
+#endif

BotSharp.MachineLearning/CRFLite/Utils/MD5.cs

@@ -0,0 +1,554 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    public sealed class MD5
+    {
+        private const int BLOCK_SIZE_BYTES = 64;
+        private const int HASH_SIZE_BYTES = 16;
+
+        private uint _state0;
+        private uint _state1;
+        private uint _state2;
+        private uint _state3;
+
+        private uint _decodeBuf0;
+        private uint _decodeBuf1;
+        private uint _decodeBuf2;
+        private uint _decodeBuf3;
+        private uint _decodeBuf4;
+        private uint _decodeBuf5;
+        private uint _decodeBuf6;
+        private uint _decodeBuf7;
+        private uint _decodeBuf8;
+        private uint _decodeBuf9;
+        private uint _decodeBuf10;
+        private uint _decodeBuf11;
+        private uint _decodeBuf12;
+        private uint _decodeBuf13;
+        private uint _decodeBuf14;
+        private uint _decodeBuf15;
+
+        private ulong count;
+        private byte[] _ProcessingBuffer;   // Used to start data when passed less than a block worth.
+        private int _ProcessingBufferCount; // Counts how much data we have stored that still needs processed.
+        private byte[] hash;
+        private byte[] fooBuffer;
+
+        public MD5()
+        {
+            fooBuffer = new byte[BLOCK_SIZE_BYTES * 4096];
+            hash = new byte[16];
+            _ProcessingBuffer = new byte[BLOCK_SIZE_BYTES];
+
+            Initialize();
+        }
+
+        ~MD5()
+        {
+            Dispose(false);
+        }
+
+        private void Dispose(bool disposing)
+        {
+            if (_ProcessingBuffer != null)
+            {
+                Array.Clear(_ProcessingBuffer, 0, _ProcessingBuffer.Length);
+                _ProcessingBuffer = null;
+            }
+        }
+
+        private void HashCore(byte[] rgb, int start, int size)
+        {
+            int i;
+            if (_ProcessingBufferCount != 0)
+            {
+                if (size < (BLOCK_SIZE_BYTES - _ProcessingBufferCount))
+                {
+                    System.Buffer.BlockCopy(rgb, start, _ProcessingBuffer, _ProcessingBufferCount, size);
+                    _ProcessingBufferCount += size;
+                    return;
+                }
+                else
+                {
+                    i = (BLOCK_SIZE_BYTES - _ProcessingBufferCount);
+                    System.Buffer.BlockCopy(rgb, start, _ProcessingBuffer, _ProcessingBufferCount, i);
+                    ProcessBlock(_ProcessingBuffer, 0);
+                    _ProcessingBufferCount = 0;
+                    start += i;
+                    size -= i;
+                }
+            }
+
+            for (i = 0; i < size - size % BLOCK_SIZE_BYTES; i += BLOCK_SIZE_BYTES)
+            {
+                ProcessBlock(rgb, start + i);
+            }
+
+            if (size % BLOCK_SIZE_BYTES != 0)
+            {
+                System.Buffer.BlockCopy(rgb, size - size % BLOCK_SIZE_BYTES + start, _ProcessingBuffer, 0, size % BLOCK_SIZE_BYTES);
+                _ProcessingBufferCount = size % BLOCK_SIZE_BYTES;
+            }
+        }
+
+        public long Compute64BitHash(byte[] buffer)
+        {
+            HashCore(buffer, 0, buffer.Length);
+
+            ProcessFinalBlock(_ProcessingBuffer, 0, _ProcessingBufferCount);
+            long longRst = (((long)_state1 << 32) | (long)_state0);
+            this.Initialize();
+
+            return longRst;
+        }
+
+        public byte[] ComputeHash(byte [] buffer)
+        {
+            HashCore(buffer, 0, buffer.Length);
+
+            ProcessFinalBlock(_ProcessingBuffer, 0, _ProcessingBufferCount);
+
+            hash[0] = (byte)(_state0);
+            hash[1] = (byte)(_state0 >> 8);
+            hash[2] = (byte)(_state0 >> 16);
+            hash[3] = (byte)(_state0 >> 24);
+            hash[4] = (byte)(_state1);
+            hash[5] = (byte)(_state1 >> 8);
+            hash[6] = (byte)(_state1 >> 16);
+            hash[7] = (byte)(_state1 >> 24);
+            hash[8] = (byte)(_state2);
+            hash[9] = (byte)(_state2 >> 8);
+            hash[10] = (byte)(_state2 >> 16);
+            hash[11] = (byte)(_state2 >> 24);
+            hash[12] = (byte)(_state3);
+            hash[13] = (byte)(_state3 >> 8);
+            hash[14] = (byte)(_state3 >> 16);
+            hash[15] = (byte)(_state3 >> 24);
+
+            return hash;
+        }
+
+        public void Initialize()
+        {
+            count = 0;
+            _ProcessingBufferCount = 0;
+
+            _state0 = 0x67452301;
+            _state1 = 0xefcdab89;
+            _state2 = 0x98badcfe;
+            _state3 = 0x10325476;
+        }
+
+        private void ProcessBlock(byte[] inputBuffer, int inputOffset)
+        {
+            uint a, b, c, d;
+
+            count += BLOCK_SIZE_BYTES;
+
+            _decodeBuf0 = ((uint)(inputBuffer[inputOffset] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 1] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 2] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 3]) << 24);
+
+            _decodeBuf1 = ((uint)(inputBuffer[inputOffset + 4] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 5] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 6] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 7]) << 24);
+
+            _decodeBuf2 = ((uint)(inputBuffer[inputOffset + 8] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 9] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 10] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 11]) << 24);
+
+            _decodeBuf3 = ((uint)(inputBuffer[inputOffset + 12] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 13] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 14] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 15]) << 24);
+
+            _decodeBuf4 = ((uint)(inputBuffer[inputOffset + 16] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 17] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 18] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 19]) << 24);
+
+            _decodeBuf5 = ((uint)(inputBuffer[inputOffset + 20] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 21] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 22] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 23]) << 24);
+
+            _decodeBuf6 = ((uint)(inputBuffer[inputOffset + 24] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 25] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 26] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 27]) << 24);
+
+            _decodeBuf7 = ((uint)(inputBuffer[inputOffset + 28] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 29] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 30] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 31]) << 24);
+
+            _decodeBuf8 = ((uint)(inputBuffer[inputOffset + 32] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 33] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 34] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 35]) << 24);
+
+            _decodeBuf9 = ((uint)(inputBuffer[inputOffset + 36] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 37] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 38] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 39]) << 24);
+
+            _decodeBuf10 = ((uint)(inputBuffer[inputOffset + 40] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 41] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 42] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 43]) << 24);
+
+            _decodeBuf11 = ((uint)(inputBuffer[inputOffset + 44] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 45] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 46] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 47]) << 24);
+
+            _decodeBuf12 = ((uint)(inputBuffer[inputOffset + 48] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 49] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 50] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 51]) << 24);
+
+            _decodeBuf13 = ((uint)(inputBuffer[inputOffset + 52] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 53] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 54] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 55]) << 24);
+
+            _decodeBuf14 = ((uint)(inputBuffer[inputOffset + 56] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 57] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 58] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 59]) << 24);
+
+            _decodeBuf15 = ((uint)(inputBuffer[inputOffset + 60] & 0xff)) |
+              (((uint)(inputBuffer[inputOffset + 61] & 0xff)) << 8) |
+              (((uint)(inputBuffer[inputOffset + 62] & 0xff)) << 16) |
+              (((uint)inputBuffer[inputOffset + 63]) << 24);
+
+            a = _state0;
+            b = _state1;
+            c = _state2;
+            d = _state3;
+
+            // ---- Round 1 --------
+
+            // ---- Round 1 --------
+
+            a += (((c ^ d) & b) ^ d) + (uint)0xd76aa478 + _decodeBuf0;
+            a = (a << 7) | (a >> 25);
+            a += b;
+
+            d += (((b ^ c) & a) ^ c) + (uint)0xe8c7b756 + _decodeBuf1;
+            d = (d << 12) | (d >> 20);
+            d += a;
+
+            c += (((a ^ b) & d) ^ b) + (uint)0x242070db + _decodeBuf2;
+            c = (c << 17) | (c >> 15);
+            c += d;
+
+            b += (((d ^ a) & c) ^ a) + (uint)0xc1bdceee + _decodeBuf3;
+            b = (b << 22) | (b >> 10);
+            b += c;
+
+            a += (((c ^ d) & b) ^ d) + (uint)0xf57c0faf + _decodeBuf4;
+            a = (a << 7) | (a >> 25);
+            a += b;
+
+            d += (((b ^ c) & a) ^ c) + (uint)0x4787c62a + _decodeBuf5;
+            d = (d << 12) | (d >> 20);
+            d += a;
+
+            c += (((a ^ b) & d) ^ b) + (uint)0xa8304613 + _decodeBuf6;
+            c = (c << 17) | (c >> 15);
+            c += d;
+
+            b += (((d ^ a) & c) ^ a) + (uint)0xfd469501 + _decodeBuf7;
+            b = (b << 22) | (b >> 10);
+            b += c;
+
+            a += (((c ^ d) & b) ^ d) + (uint)0x698098d8 + _decodeBuf8;
+            a = (a << 7) | (a >> 25);
+            a += b;
+
+            d += (((b ^ c) & a) ^ c) + (uint)0x8b44f7af + _decodeBuf9;
+            d = (d << 12) | (d >> 20);
+            d += a;
+
+            c += (((a ^ b) & d) ^ b) + (uint)0xffff5bb1 + _decodeBuf10;
+            c = (c << 17) | (c >> 15);
+            c += d;
+
+            b += (((d ^ a) & c) ^ a) + (uint)0x895cd7be + _decodeBuf11;
+            b = (b << 22) | (b >> 10);
+            b += c;
+
+            a += (((c ^ d) & b) ^ d) + (uint)0x6b901122 + _decodeBuf12;
+            a = (a << 7) | (a >> 25);
+            a += b;
+
+            d += (((b ^ c) & a) ^ c) + (uint)0xfd987193 + _decodeBuf13;
+            d = (d << 12) | (d >> 20);
+            d += a;
+
+            c += (((a ^ b) & d) ^ b) + (uint)0xa679438e + _decodeBuf14;
+            c = (c << 17) | (c >> 15);
+            c += d;
+
+            b += (((d ^ a) & c) ^ a) + (uint)0x49b40821 + _decodeBuf15;
+            b = (b << 22) | (b >> 10);
+            b += c;
+
+
+            // ---- Round 2 --------
+
+            a += ((b & d) | (c & ~d)) + (uint)0xf61e2562 + _decodeBuf1;
+            a = (a << 5) | (a >> 27);
+            a += b;
+
+            d += ((a & c) | (b & ~c)) + (uint)0xc040b340 + _decodeBuf6;
+            d = (d << 9) | (d >> 23);
+            d += a;
+
+            c += ((d & b) | (a & ~b)) + (uint)0x265e5a51 + _decodeBuf11;
+            c = (c << 14) | (c >> 18);
+            c += d;
+
+            b += ((c & a) | (d & ~a)) + (uint)0xe9b6c7aa + _decodeBuf0;
+            b = (b << 20) | (b >> 12);
+            b += c;
+
+            a += ((b & d) | (c & ~d)) + (uint)0xd62f105d + _decodeBuf5;
+            a = (a << 5) | (a >> 27);
+            a += b;
+
+            d += ((a & c) | (b & ~c)) + (uint)0x02441453 + _decodeBuf10;
+            d = (d << 9) | (d >> 23);
+            d += a;
+
+            c += ((d & b) | (a & ~b)) + (uint)0xd8a1e681 + _decodeBuf15;
+            c = (c << 14) | (c >> 18);
+            c += d;
+
+            b += ((c & a) | (d & ~a)) + (uint)0xe7d3fbc8 + _decodeBuf4;
+            b = (b << 20) | (b >> 12);
+            b += c;
+
+            a += ((b & d) | (c & ~d)) + (uint)0x21e1cde6 + _decodeBuf9;
+            a = (a << 5) | (a >> 27);
+            a += b;
+
+            d += ((a & c) | (b & ~c)) + (uint)0xc33707d6 + _decodeBuf14;
+            d = (d << 9) | (d >> 23);
+            d += a;
+
+            c += ((d & b) | (a & ~b)) + (uint)0xf4d50d87 + _decodeBuf3;
+            c = (c << 14) | (c >> 18);
+            c += d;
+
+            b += ((c & a) | (d & ~a)) + (uint)0x455a14ed + _decodeBuf8;
+            b = (b << 20) | (b >> 12);
+            b += c;
+
+            a += ((b & d) | (c & ~d)) + (uint)0xa9e3e905 + _decodeBuf13;
+            a = (a << 5) | (a >> 27);
+            a += b;
+
+            d += ((a & c) | (b & ~c)) + (uint)0xfcefa3f8 + _decodeBuf2;
+            d = (d << 9) | (d >> 23);
+            d += a;
+
+            c += ((d & b) | (a & ~b)) + (uint)0x676f02d9 + _decodeBuf7;
+            c = (c << 14) | (c >> 18);
+            c += d;
+
+            b += ((c & a) | (d & ~a)) + (uint)0x8d2a4c8a + _decodeBuf12;
+            b = (b << 20) | (b >> 12);
+            b += c;
+
+
+            // ---- Round 3 --------
+
+            a += (b ^ c ^ d) + (uint)0xfffa3942 + _decodeBuf5;
+            a = (a << 4) | (a >> 28);
+            a += b;
+
+            d += (a ^ b ^ c) + (uint)0x8771f681 + _decodeBuf8;
+            d = (d << 11) | (d >> 21);
+            d += a;
+
+            c += (d ^ a ^ b) + (uint)0x6d9d6122 + _decodeBuf11;
+            c = (c << 16) | (c >> 16);
+            c += d;
+
+            b += (c ^ d ^ a) + (uint)0xfde5380c + _decodeBuf14;
+            b = (b << 23) | (b >> 9);
+            b += c;
+
+            a += (b ^ c ^ d) + (uint)0xa4beea44 + _decodeBuf1;
+            a = (a << 4) | (a >> 28);
+            a += b;
+
+            d += (a ^ b ^ c) + (uint)0x4bdecfa9 + _decodeBuf4;
+            d = (d << 11) | (d >> 21);
+            d += a;
+
+            c += (d ^ a ^ b) + (uint)0xf6bb4b60 + _decodeBuf7;
+            c = (c << 16) | (c >> 16);
+            c += d;
+
+            b += (c ^ d ^ a) + (uint)0xbebfbc70 + _decodeBuf10;
+            b = (b << 23) | (b >> 9);
+            b += c;
+
+            a += (b ^ c ^ d) + (uint)0x289b7ec6 + _decodeBuf13;
+            a = (a << 4) | (a >> 28);
+            a += b;
+
+            d += (a ^ b ^ c) + (uint)0xeaa127fa + _decodeBuf0;
+            d = (d << 11) | (d >> 21);
+            d += a;
+
+            c += (d ^ a ^ b) + (uint)0xd4ef3085 + _decodeBuf3;
+            c = (c << 16) | (c >> 16);
+            c += d;
+
+            b += (c ^ d ^ a) + (uint)0x04881d05 + _decodeBuf6;
+            b = (b << 23) | (b >> 9);
+            b += c;
+
+            a += (b ^ c ^ d) + (uint)0xd9d4d039 + _decodeBuf9;
+            a = (a << 4) | (a >> 28);
+            a += b;
+
+            d += (a ^ b ^ c) + (uint)0xe6db99e5 + _decodeBuf12;
+            d = (d << 11) | (d >> 21);
+            d += a;
+
+            c += (d ^ a ^ b) + (uint)0x1fa27cf8 + _decodeBuf15;
+            c = (c << 16) | (c >> 16);
+            c += d;
+
+            b += (c ^ d ^ a) + (uint)0xc4ac5665 + _decodeBuf2;
+            b = (b << 23) | (b >> 9);
+            b += c;
+
+
+            // ---- Round 4 --------
+
+            a += (((~d) | b) ^ c) + (uint)0xf4292244 + _decodeBuf0;
+            a = (a << 6) | (a >> 26);
+            a += b;
+
+            d += (((~c) | a) ^ b) + (uint)0x432aff97 + _decodeBuf7;
+            d = (d << 10) | (d >> 22);
+            d += a;
+
+            c += (((~b) | d) ^ a) + (uint)0xab9423a7 + _decodeBuf14;
+            c = (c << 15) | (c >> 17);
+            c += d;
+
+            b += (((~a) | c) ^ d) + (uint)0xfc93a039 + _decodeBuf5;
+            b = (b << 21) | (b >> 11);
+            b += c;
+
+            a += (((~d) | b) ^ c) + (uint)0x655b59c3 + _decodeBuf12;
+            a = (a << 6) | (a >> 26);
+            a += b;
+
+            d += (((~c) | a) ^ b) + (uint)0x8f0ccc92 + _decodeBuf3;
+            d = (d << 10) | (d >> 22);
+            d += a;
+
+            c += (((~b) | d) ^ a) + (uint)0xffeff47d + _decodeBuf10;
+            c = (c << 15) | (c >> 17);
+            c += d;
+
+            b += (((~a) | c) ^ d) + (uint)0x85845dd1 + _decodeBuf1;
+            b = (b << 21) | (b >> 11);
+            b += c;
+
+            a += (((~d) | b) ^ c) + (uint)0x6fa87e4f + _decodeBuf8;
+            a = (a << 6) | (a >> 26);
+            a += b;
+
+            d += (((~c) | a) ^ b) + (uint)0xfe2ce6e0 + _decodeBuf15;
+            d = (d << 10) | (d >> 22);
+            d += a;
+
+            c += (((~b) | d) ^ a) + (uint)0xa3014314 + _decodeBuf6;
+            c = (c << 15) | (c >> 17);
+            c += d;
+
+            b += (((~a) | c) ^ d) + (uint)0x4e0811a1 + _decodeBuf13;
+            b = (b << 21) | (b >> 11);
+            b += c;
+
+            a += (((~d) | b) ^ c) + (uint)0xf7537e82 + _decodeBuf4;
+            a = (a << 6) | (a >> 26);
+            a += b;
+
+            d += (((~c) | a) ^ b) + (uint)0xbd3af235 + _decodeBuf11;
+            d = (d << 10) | (d >> 22);
+            d += a;
+
+            c += (((~b) | d) ^ a) + (uint)0x2ad7d2bb + _decodeBuf2;
+            c = (c << 15) | (c >> 17);
+            c += d;
+
+            b += (((~a) | c) ^ d) + (uint)0xeb86d391 + _decodeBuf9;
+            b = (b << 21) | (b >> 11);
+            b += c;
+
+            _state0 += a;
+            _state1 += b;
+            _state2 += c;
+            _state3 += d;
+        }
+
+        private void ProcessFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount)
+        {
+            ulong total = count + (ulong)inputCount;
+            int paddingSize = (int)(56 - total % BLOCK_SIZE_BYTES);
+
+            if (paddingSize < 1)
+                paddingSize += BLOCK_SIZE_BYTES;
+
+            for (int i = 0; i < inputCount; i++)
+            {
+                fooBuffer[i] = inputBuffer[i + inputOffset];
+            }
+
+            fooBuffer[inputCount] = 0x80;
+            for (int i = inputCount + 1; i < inputCount + paddingSize; i++)
+            {
+                fooBuffer[i] = 0x00;
+            }
+
+            // I deal in bytes. The algorithm deals in bits.
+            ulong size = total << 3;
+            AddLength(size, fooBuffer, inputCount + paddingSize);
+            ProcessBlock(fooBuffer, 0);
+
+            if (inputCount + paddingSize + 8 == 128)
+            {
+                ProcessBlock(fooBuffer, 64);
+            }
+        }
+
+        internal void AddLength(ulong length, byte[] buffer, int position)
+        {
+            buffer[position++] = (byte)(length);
+            buffer[position++] = (byte)(length >> 8);
+            buffer[position++] = (byte)(length >> 16);
+            buffer[position++] = (byte)(length >> 24);
+            buffer[position++] = (byte)(length >> 32);
+            buffer[position++] = (byte)(length >> 40);
+            buffer[position++] = (byte)(length >> 48);
+            buffer[position] = (byte)(length >> 56);
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils/VarBigArray.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Text;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    public sealed class VarBigArray<T> : BigArray<T> where T : IComparable<T>
+    {
+        long blockSizeInTotal_;
+        private object ll = new object();
+
+        public override T this[long offset]
+        {
+            get
+            {
+                if (offset >= size_)
+                {
+                    //resize array size, it need to be synced,
+                    //for high performance, we use double check to avoid useless resize call and save memory
+                    lock (ll)
+                    {
+                        if (offset >= size_)
+                        {
+                            Resize(offset + 1);
+                        }
+                    }
+                }
+
+                long nBlock = offset >> moveBit;
+                return arrList[(int)nBlock][offset & (sizePerBlock-1)];
+            }
+            set
+            {
+                if (offset >= size_)
+                {
+                    //resize array size, it need to be synced,
+                    //for high performance, we use double check to avoid useless resize call and save memory
+                    lock (ll)
+                    {
+                        if (offset >= size_)
+                        {
+                            Resize(offset + 1);
+                        }
+                    }
+                }
+
+                long nBlock = offset >> moveBit;
+                arrList[(int)nBlock][offset & (sizePerBlock-1)] = value;
+            }
+        }
+
+
+
+        private void Resize(long new_size)
+        {
+            while (blockSizeInTotal_ <= new_size)
+            {
+                arrList.Add(new T[sizePerBlock]);
+                blockSizeInTotal_ += sizePerBlock;
+            }
+
+            size_ = new_size;
+        }
+
+        //construct variable size big array
+        //size is array's default length
+        //lowBounding is the lowest bounding of the array
+        //when accessing the position which is outer bounding, the big array will be extend automatically.
+        public VarBigArray(long size)
+        {
+            size_ = size;
+            arrList = new List<T[]>();
+
+            for (blockSizeInTotal_ = 0; blockSizeInTotal_ < size_; 
+                blockSizeInTotal_ += sizePerBlock)
+            {
+                arrList.Add(new T[sizePerBlock]);
+            }
+        }
+    }
+}

BotSharp.MachineLearning/CRFLite/Utils/VectorQuantization.cs

@@ -0,0 +1,377 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.IO;
+
+namespace BotSharp.MachineLearning.CRFLite.Utils
+{
+    public class VQCluster : IComparer<VQCluster>
+    {
+        public int iStart, iEnd;
+        public double variance, mean;
+
+        public VQCluster(double m, double v, int i, int j)
+        {
+            mean = m;
+            variance = v;
+            iStart = i;
+            iEnd = j;
+        }
+
+        public VQCluster() { }
+
+        public int Compare(VQCluster X, VQCluster Y)
+        {
+            if (X.mean > Y.mean) return 1;
+            if (X.mean < Y.mean) return -1;
+            return 0;
+        }
+    };
+
+
+    public class VectorQuantization
+    {
+        protected List<VQCluster> vqClusters;
+        protected double[] codebook;
+        protected VarBigArray<double> dataSet;
+        protected int dataSetSize;
+
+        public double[] CodeBook { get { return codebook; } }
+
+        public VectorQuantization()
+        {
+            dataSet = new VarBigArray<double>(1024 * 1024);
+            dataSetSize = 0;
+        }
+
+        /// <summary>
+        /// Add a set of data into data set
+        /// </summary>
+        /// <param name="values"></param>
+        public void Add(double[] values)
+        {
+            foreach (double value in values)
+            {
+                Add(value);
+            }
+        }
+
+        /// <summary>
+        /// Add a single data into data set
+        /// </summary>
+        /// <param name="value"></param>
+        public void Add(double value)
+        {
+            dataSet[dataSetSize] = value;
+            dataSetSize++;
+        }
+
+        public int ComputeVQ(double value)
+        {
+            return BinarySearch(value);
+        }
+
+        /// <summary>
+        /// Build codebook according given data set
+        /// </summary>
+        /// <param name="vqSize"></param>
+        /// <returns></returns>
+        public double BuildCodebook(int vqSize)
+        {
+            if (vqSize > dataSetSize)
+            {
+                return -1;
+            }
+
+            dataSet.Sort(0, dataSetSize);
+
+            //Set entire data as a single cluster, and then split it
+            double mean, var;
+            ComputeVariables(0, dataSetSize - 1, out mean, out var);
+            VQCluster c = new VQCluster(mean, var, 0, dataSetSize - 1);
+            vqClusters = new List<VQCluster>();
+            vqClusters.Add(c);
+
+            //Split clusters according its variance values
+            while (vqClusters.Count < vqSize)
+            {
+                int maxVarClusterId = MaxVarianceClusterId();
+                if (maxVarClusterId < 0) break; // no more to split
+
+                //Split the cluster into two and remove the orginal one
+                SplitCluster(vqClusters[maxVarClusterId].iStart, vqClusters[maxVarClusterId].iEnd, 0, 1);
+                vqClusters.RemoveAt(maxVarClusterId);
+            }
+
+            //Adjust clusters according their mean values
+            AdjustCluster();
+
+            //Final codebook
+            vqSize = vqClusters.Count;
+            codebook = new double[vqSize];
+            double distortion = 0;
+            for (int i = 0; i < vqSize; i++)
+            {
+                codebook[i] = vqClusters[i].mean;
+                for (int j = vqClusters[i].iStart; j <= vqClusters[i].iEnd; j++)
+                {
+                    double diff = dataSet[j] - codebook[i];
+                    distortion += diff * diff;
+                }
+            }
+
+            distortion = Math.Sqrt(distortion / dataSetSize);
+            return distortion;
+        }
+
+        public bool WriteCodebook(string filename)
+        {
+            using (StreamWriter sw = new StreamWriter(filename))
+            {
+                sw.WriteLine("Codeword\tMean\tCount");
+                for (int i = 0; i < codebook.Length; i++)
+                {
+                    int count = (int)(vqClusters[i].iEnd - vqClusters[i].iStart + 1);
+                    sw.WriteLine("{0,8} {1}\t{2}", i, codebook[i], count);
+                }
+            }
+
+            return true;
+        }
+
+        public void ReadCodebook(string filename)
+        {
+            using (StreamReader sr = new StreamReader(filename))
+            {
+                //Skip column title
+                sr.ReadLine();
+
+                //Read each line
+                string line = null;
+                List<double> cb = new List<double>();
+                while ((line = sr.ReadLine()) != null)
+                {
+                    string[] words = line.Split();
+                    double mean = 0;
+
+                    int n = int.Parse(words[0]);
+                    mean = double.Parse(words[1]);
+                    int count = int.Parse(words[2]);
+                    cb.Add(mean);
+                }
+                codebook = cb.ToArray<double>();
+            }
+        }
+
+        /// <summary>
+        /// Adjust cluster boundary according mean values
+        /// </summary>
+        void AdjustCluster()
+        {
+            int vqsize = vqClusters.Count;
+            bool updateCluster = true;
+            double mean, var;
+
+            vqClusters.Sort(new VQCluster());
+
+            for (int iter = 0; iter < 20 && updateCluster; iter++)
+            {
+                updateCluster = false;
+                for (int i = 1; i < vqsize; i++)
+                {
+                    int j = (int)vqClusters[i - 1].iEnd;
+                    while (true)
+                    {
+                        double d1 = dataSet[j] - vqClusters[i - 1].mean;
+                        if (d1 <= 0) break;
+
+                        double d2 = vqClusters[i].mean - dataSet[j];
+
+                        if (d1 <= d2) break;
+                        j--;
+                    }
+
+                    if (j < vqClusters[i - 1].iEnd)
+                    {
+                        ComputeVariables((int)vqClusters[i - 1].iStart, j, out mean, out var);
+                        UpdateCluster(i - 1, (int)vqClusters[i - 1].iStart, j, mean, var);
+
+                        ComputeVariables(j + 1, (int)vqClusters[i].iEnd, out mean, out var);
+                        UpdateCluster(i, j + 1, (int)vqClusters[i].iEnd, mean, var);
+
+                        updateCluster = true;
+                        continue;
+                    }
+
+                    j = (int)vqClusters[i].iStart;
+                    while (true)
+                    {
+                        double d1 = vqClusters[i].mean - dataSet[j];
+                        if (d1 <= 0) break;
+
+                        double d2 = dataSet[j] - vqClusters[i - 1].mean;
+
+                        if (d1 <= d2) break;
+                        j++;
+                    }
+                    if (j > vqClusters[i].iStart)
+                    {
+                        ComputeVariables((int)vqClusters[i - 1].iStart, j - 1, out mean, out var);
+                        UpdateCluster(i - 1, (int)vqClusters[i - 1].iStart, j - 1, mean, var);
+
+                        ComputeVariables(j, (int)vqClusters[i].iEnd, out mean, out var);
+                        UpdateCluster(i, j, (int)vqClusters[i].iEnd, mean, var);
+
+                        updateCluster = true;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Search codebook and get the index which value is the nearest to given value
+        /// </summary>
+        /// <param name="value"></param>
+        /// <returns></returns>
+        private int BinarySearch(double value)
+        {
+            int low = 0, high = codebook.Length, mid = 0;
+            while (low < high)
+            {
+                mid = (int)((high - low) / 2) + low;
+                if (value > codebook[mid])
+                    low = mid + 1;
+                else if (value < codebook[mid])
+                    high = mid;
+                else
+                    return mid;
+            }
+
+            int cw = mid;
+            double delta = Math.Abs(value - codebook[cw]);
+            if (mid + 1 < codebook.Length)
+            {
+                double d2 = Math.Abs(value - codebook[mid + 1]);
+                if (d2 < delta)
+                {
+                    cw = mid + 1;
+                    delta = d2;
+                }
+            }
+            if (mid - 1 >= 0)
+            {
+                double d2 = Math.Abs(value - codebook[mid - 1]);
+                if (d2 < delta)
+                {
+                    cw = mid - 1;
+                    delta = d2;
+                }
+            }
+            return cw;
+        }
+
+        /// <summary>
+        /// Return the cluster id which has the biggest variance value
+        /// </summary>
+        /// <returns></returns>
+        private int MaxVarianceClusterId()
+        {
+            double maxVar = -1;
+            int c = -1;
+            for (int i = 0; i < vqClusters.Count; i++)
+            {
+                if (vqClusters[i].variance > maxVar)
+                {
+                    maxVar = vqClusters[i].variance;
+                    c = i;
+                }
+            }
+            return c;
+        }
+
+        /// <summary>
+        /// Computing the mean and variance of given data set
+        /// </summary>
+        /// <param name="iStart"></param>
+        /// <param name="iEnd"></param>
+        /// <param name="mean"></param>
+        /// <param name="variance"></param>
+        private void ComputeVariables(int iStart, int iEnd, out double mean, out double variance)
+        {
+            double sum = 0;
+
+            mean = 0;
+            variance = 0;
+            for (int i = iStart; i <= iEnd; i++)
+                sum += dataSet[i];
+            mean = sum / (iEnd - iStart + 1);
+
+            sum = 0;
+            if (dataSet[iStart] < mean && dataSet[iEnd] > mean)
+            {
+                for (int i = iStart; i <= iEnd; i++)
+                {
+                    double diff = dataSet[i] - mean;
+                    sum += diff * diff;
+                }
+            }
+            variance = sum;
+        }
+
+        /// <summary>
+        /// Update given cluster's values
+        /// </summary>
+        /// <param name="index"></param>
+        /// <param name="iStart"></param>
+        /// <param name="iEnd"></param>
+        /// <param name="mean"></param>
+        /// <param name="variance"></param>
+        private void UpdateCluster(int index, int iStart, int iEnd, double mean, double variance)
+        {
+            vqClusters[index].iStart = iStart;
+            vqClusters[index].iEnd = iEnd;
+            vqClusters[index].mean = mean;
+            vqClusters[index].variance = variance;
+
+        }
+
+        /// <summary>
+        /// Split one cluster into two clusters according its mean
+        /// </summary>
+        /// <param name="iStart"></param>
+        /// <param name="iEnd"></param>
+        /// <param name="depth"></param>
+        /// <param name="maxDepth"></param>
+        private void SplitCluster(int iStart, int iEnd, int depth, int maxDepth)
+        {
+            if (iStart > iEnd)
+            {
+                return;
+            }
+
+            double mean, variance;
+            ComputeVariables(iStart, iEnd, out mean, out variance);
+            if (depth == maxDepth)
+            {
+                VQCluster c = new VQCluster(mean, variance, iStart, iEnd);
+                vqClusters.Add(c);
+            }
+            else
+            {
+                //Split the cluster into two clusters according mean value
+                int i;
+                for (i = iStart; i <= iEnd; i++)
+                {
+                    //The following data will be greater than mean value, so we split it here
+                    if (dataSet[i] > mean)
+                        break;
+                }
+
+                SplitCluster(iStart, i - 1, depth + 1, maxDepth);
+                SplitCluster(i, iEnd, depth + 1, maxDepth);
+            }
+        }
+    }
+}

BotSharp.sln

@@ -17,6 +17,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NL
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.MachineLearning.UnitTest", "BotSharp.MachineLearning.UnitTest\BotSharp.MachineLearning.UnitTest.csproj", "{B876F0E9-40F0-48B7-91D5-E09E0B442266}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -51,6 +53,10 @@ Global
 		{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B876F0E9-40F0-48B7-91D5-E09E0B442266}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE