using System; using System.Collections.Generic; using System.Linq; using System.Text; using BotSharp.NLP.Tokenize; namespace BotSharp.NLP.Tag { /// /// N-Gramm taggers are based on a simple statistical algorithm: /// for each token, assign the tag that is most likely for that particular token. /// public class NGramTagger : ITagger { private List _contextMapping { get; set; } public void Tag(Sentence sentence, TagOptions options) { // need training to generate model if(_contextMapping == null) { Train(options.Corpus, options); } } public void Train(List sentences, TagOptions options) { _contextMapping = new List(); for (int idx = 0; idx < options.Corpus.Count; idx++) { var sent = options.Corpus[idx]; for (int ngram = 1; ngram < options.NGram; ngram++) { sent.Words.Insert(0, new Token { Text = "NIL", Pos = options.Tag, Start = (ngram - 1) * 3 }); } int pos = options.NGram - 1; for (pos = 1; pos < sent.Words.Count; pos++) { var freq = new NGramFreq { PrecedingTokens = new List { sent.Words[pos - 1] }, Token = sent.Words[pos], Count = 0 }; _contextMapping.Add(freq); } } /*var results = (from c in cache group c by c.Item1 into g select new { g.Key, Count = g.Count() }).ToList();*/ } private class NGramFreq { /// /// Tokens prior current token /// public List PrecedingTokens { get; set; } /// /// Current token tag /// public Token Token { get; set; } /// /// Occurence frequency /// public int Count { get; set; } public string Context { get { return $"{PrecedingTokens.First().Pos} {Token.Text} {Token.Pos}"; } } } } }