113 lines
3.4 KiB
C#
113 lines
3.4 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using BotSharp.NLP.Tokenize;
|
|
|
|
namespace BotSharp.NLP.Tag
|
|
{
|
|
/// <summary>
|
|
/// N-Gramm taggers are based on a simple statistical algorithm:
|
|
/// for each token, assign the tag that is most likely for that particular token.
|
|
/// </summary>
|
|
public class NGramTagger : ITagger
|
|
{
|
|
private List<NGramFreq> _contextMapping { get; set; }
|
|
|
|
public void Tag(Sentence sentence, TagOptions options)
|
|
{
|
|
// need training to generate model
|
|
if(_contextMapping == null)
|
|
{
|
|
Train(options.Corpus, options);
|
|
}
|
|
|
|
Fill(sentence, options);
|
|
|
|
for (int pos = options.NGram - 1; pos < sentence.Words.Count; pos++)
|
|
{
|
|
sentence.Words[pos].Pos = _contextMapping.FirstOrDefault(x => x.Context == GetContext(pos, sentence.Words, options))?.Tag;
|
|
|
|
// set default tag
|
|
if(sentence.Words[pos].Pos == null)
|
|
{
|
|
sentence.Words[pos].Pos = options.Tag;
|
|
}
|
|
}
|
|
|
|
for(int pos = 0; pos < options.NGram - 1; pos++)
|
|
{
|
|
sentence.Words.RemoveAt(0);
|
|
}
|
|
}
|
|
|
|
public void Train(List<Sentence> sentences, TagOptions options)
|
|
{
|
|
var cache = new List<NGramFreq>();
|
|
|
|
for (int idx = 0; idx < options.Corpus.Count; idx++)
|
|
{
|
|
var sent = options.Corpus[idx];
|
|
|
|
Fill(sent, options);
|
|
|
|
for (int pos = options.NGram - 1; pos < sent.Words.Count; pos++)
|
|
{
|
|
var freq = new NGramFreq
|
|
{
|
|
Context = GetContext(pos, sent.Words, options),
|
|
Tag = sent.Words[pos].Pos,
|
|
Count = 1
|
|
};
|
|
|
|
cache.Add(freq);
|
|
}
|
|
}
|
|
|
|
_contextMapping = (from c in cache
|
|
group c by new { c.Context, c.Tag } into g
|
|
select new NGramFreq
|
|
{
|
|
Context = g.Key.Context,
|
|
Tag = g.Key.Tag,
|
|
Count = g.Count()
|
|
}).OrderByDescending(x => x.Count)
|
|
.ToList();
|
|
}
|
|
|
|
private string GetContext(int pos, List<Token> words, TagOptions options)
|
|
{
|
|
string context = words[pos].Text;
|
|
for (int ngram = options.NGram - 1; ngram > 0; ngram--)
|
|
{
|
|
context = words[pos - ngram].Pos + " " + context;
|
|
}
|
|
|
|
return context;
|
|
}
|
|
|
|
private void Fill(Sentence sent, TagOptions options)
|
|
{
|
|
for (int ngram = 1; ngram < options.NGram; ngram++)
|
|
{
|
|
sent.Words.Insert(0, new Token { Text = "NIL", Pos = options.Tag, Start = (ngram - 1) * 3 });
|
|
}
|
|
}
|
|
|
|
private class NGramFreq
|
|
{
|
|
/// <summary>
|
|
/// Current token tag
|
|
/// </summary>
|
|
public string Tag { get; set; }
|
|
|
|
/// <summary>
|
|
/// Occurence frequency
|
|
/// </summary>
|
|
public int Count { get; set; }
|
|
|
|
public string Context { get; set; }
|
|
}
|
|
}
|
|
}
|