using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; using BotSharp.Models.CRFLite.Utils; namespace BotSharp.Models.CRFLite.Decoder { public class ModelReader : BaseModel { private readonly Func modelLoader = null; public uint version; //模型版本号,读取模型时读入 private CRFLite.Utils.DoubleArrayTrieSearch da; //特征集合 /// /// Returns the model path. /// public string ModelPath { get; private set; } /// /// Creates a new /// that will load the model from the file system, /// using the given . /// /// /// Path to the model. /// public ModelReader(string modelPath) : this(GetStreamFromFileSystem, modelPath) { } /// /// Creates a new /// that will load the model from the file system, /// using the given . /// /// /// A delegate capable of resolving /// the given /// into a stream with the model file. /// /// /// Path to the model. /// public ModelReader(Func modelLoader, string modelPath) { this.modelLoader = modelLoader; this.ModelPath = modelPath; } /// /// Loads the model into memory. /// public void LoadModel() { //Load model meta data LoadMetadata(); //Load all feature set data LoadFeatureSet(); //Load all features alpha data LoadFeatureWeights(); } //get key feature id public virtual int get_id(string str) { return da.SearchByPerfectMatch(str); } public virtual double GetAlpha(long index) { return alpha_[index]; } /// /// The default model loading strategy - /// load files from the file system. /// /// /// Model file path. /// /// A stream containing the requested file. /// private static Stream GetStreamFromFileSystem(string path) { path.ThrowIfNotExists(); return File.OpenRead(path); } /// /// Provides access to the metadata stream. /// /// /// A instance /// that points to the model metadata file. /// private Stream GetMetadataStream() { string path = ModelPath.ToMetadataModelName(); return modelLoader(path); } /// /// Provides access to the feature set stream. /// /// /// A instance /// that allows accessing the model feature set file. /// private Stream GetFeatureSetStream() { string path = ModelPath.ToFeatureSetFileName(); return modelLoader(path); } /// /// Provides access to the feature set stream. /// /// /// A instance /// that allows accessing the model feature weight file. /// private Stream GetFeatureWeightStream() { string path = ModelPath.ToFeatureWeightFileName(); return modelLoader(path); } private void LoadMetadata() { using (Stream metadataStream = GetMetadataStream()) { var sr = new StreamReader(metadataStream); string strLine; strLine = sr.ReadLine(); version = uint.Parse(strLine.Split(':')[1].Trim()); strLine = sr.ReadLine(); cost_factor_ = double.Parse(strLine.Split(':')[1].Trim()); strLine = sr.ReadLine(); maxid_ = long.Parse(strLine.Split(':')[1].Trim()); strLine = sr.ReadLine(); xsize_ = uint.Parse(strLine.Split(':')[1].Trim()); strLine = sr.ReadLine(); y_ = new List(); while (true) { strLine = sr.ReadLine(); if (strLine.Length == 0) { break; } y_.Add(strLine); } // load unigram and bigram template unigram_templs_ = new List(); bigram_templs_ = new List(); while (sr.EndOfStream == false) { strLine = sr.ReadLine(); if (strLine.Length == 0) { break; } if (strLine[0] == 'U') { unigram_templs_.Add(strLine); } if (strLine[0] == 'B') { bigram_templs_.Add(strLine); } } sr.Close(); } } private void LoadFeatureSet() { Stream featureSetStream = GetFeatureSetStream(); da = new DoubleArrayTrieSearch(); da.Load(featureSetStream); } private void LoadFeatureWeights() { //feature weight array alpha_ = new double[maxid_ + 1]; using (Stream featureWeightStream = GetFeatureWeightStream()) { //Load all features alpha data var sr_alpha = new StreamReader(featureWeightStream); var br_alpha = new BinaryReader(sr_alpha.BaseStream); //Get VQ Size int vqSize = br_alpha.ReadInt32(); if (vqSize > 0) { //This is a VQ model, we need to get code book at first List vqCodeBook = new List(); for (int i = 0; i < vqSize; i++) { vqCodeBook.Add(br_alpha.ReadDouble()); } //Load weights for (long i = 0; i < maxid_; i++) { int vqIdx = br_alpha.ReadByte(); alpha_[i] = vqCodeBook[vqIdx]; } } else { //This is a normal model for (long i = 0; i < maxid_; i++) { alpha_[i] = br_alpha.ReadSingle(); } } br_alpha.Close(); } } } }