Merge branch 'master' of https://github.com/Oceania2018/BotSharp

BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs

@@ -207,7 +207,7 @@ namespace BotSharp.Core.Engines.NERs
                 entities.Add(new NlpEntity
                 {
                     Entity = entity,
-                    Start = doc.Sentences[0].Tokens[i].Offset,
+                    Start = doc.Sentences[0].Tokens[i].Start,
                     Value = doc.Sentences[0].Tokens[i].Text,
                     Confidence = probability
                 });

BotSharp.NLP.UnitTest/DefaultTaggerTest.cs

@@ -0,0 +1,28 @@
+using BotSharp.NLP.Corpus;
+using BotSharp.NLP.Tag;
+using BotSharp.NLP.Tokenize;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest
+{
+    [TestClass]
+    public class DefaultTaggerTest
+    {
+        [TestMethod]
+        public void TagInCoNLL2000()
+        {
+            var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
+            var tokens = tokenizer.Tokenize("How are you doing?");
+
+            var tagger = new TaggerFactory<DefaultTagger>(new TagOptions
+            {
+                Tag = "NN"
+            }, SupportedLanguage.English);
+
+            tagger.Tag(new Sentence { Words = tokens });
+        }
+    }
+}

BotSharp.NLP.UnitTest/NGramTaggerTest.cs

@@ -0,0 +1,46 @@
+using BotSharp.NLP.Corpus;
+using BotSharp.NLP.Tag;
+using BotSharp.NLP.Tokenize;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest
+{
+    [TestClass]
+    public class NGramTaggerTest
+    {
+        [TestMethod]
+        public void TagInCoNLL2000()
+        {
+            // tokenization
+            var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
+            {
+                Pattern = RegexTokenizer.WORD_PUNC
+            }, SupportedLanguage.English);
+
+            var tokens = tokenizer.Tokenize("How are you doing?");
+
+            // get training corpus
+            string corpusDir = Environment.GetEnvironmentVariable("BOTSHARP_CORPUS_PATH", EnvironmentVariableTarget.User);
+            var sentences = new CoNLLReader()
+                .Read(new ReaderOptions
+                {
+                    DataDir = Path.Combine(corpusDir, "CoNLL"),
+                    FileName = "conll2000_chunking_train.txt"
+                });
+
+            // start tag
+            var tagger = new TaggerFactory<NGramTagger>(new TagOptions
+            {
+                NGram = 2,
+                Tag = "NN",
+                Corpus = sentences
+            }, SupportedLanguage.English);
+
+            tagger.Tag(new Sentence { Words = tokens });
+        }
+    }
+}

BotSharp.NLP.UnitTest/RegexStemmerTest.cs

@@ -12,13 +12,12 @@ namespace BotSharp.NLP.UnitTest
         [TestMethod]
         public void StemInDefault()
         {
-            var stemmer = new StemmerFactory<RegexStemmer>();
+            var stemmer = new StemmerFactory<RegexStemmer>(new StemOptions
+            {
+                Pattern = RegexStemmer.DEFAULT
+            }, SupportedLanguage.English);
 
-            var stem = stemmer.Stem("doing",
-                new StemOptions
-                {
-                    Pattern = RegexStemmer.DEFAULT
-                });
+            var stem = stemmer.Stem("doing");
 
             Assert.IsTrue(stem == "do");
         }

BotSharp.NLP.UnitTest/RegexTokenizerTest.cs

@@ -9,91 +9,88 @@ namespace BotSharp.NLP.UnitTest
         [TestMethod]
         public void TokenizeInWhiteSpace()
         {
-            var tokenizer = new TokenizerFactory<RegexTokenizer>();
+            var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
+            {
+                Pattern = RegexTokenizer.WHITE_SPACE
+            }, SupportedLanguage.English);
 
-            var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?",
-                new TokenizationOptions
-                {
-                    Pattern = RegexTokenizer.WHITE_SPACE
-                });
+            var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?");
 
-            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Start == 0);
             Assert.IsTrue(tokens[0].Text == "Chop");
 
-            Assert.IsTrue(tokens[1].Offset == 5);
+            Assert.IsTrue(tokens[1].Start == 5);
             Assert.IsTrue(tokens[1].Text == "into");
 
-            Assert.IsTrue(tokens[2].Offset == 10);
+            Assert.IsTrue(tokens[2].Start == 10);
             Assert.IsTrue(tokens[2].Text == "pieces,");
 
-            Assert.IsTrue(tokens[3].Offset == 18);
+            Assert.IsTrue(tokens[3].Start == 18);
             Assert.IsTrue(tokens[3].Text == "isn't");
 
-            Assert.IsTrue(tokens[4].Offset == 24);
+            Assert.IsTrue(tokens[4].Start == 24);
             Assert.IsTrue(tokens[4].Text == "it?");
         }
 
         [TestMethod]
         public void TokenizeInWordPunctuation()
         {
-            var tokenizer = new TokenizerFactory<RegexTokenizer>();
+            var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
+            {
+                Pattern = RegexTokenizer.WORD_PUNC
+            }, SupportedLanguage.English);
 
-            var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?",
-                new TokenizationOptions
-                {
-                    Pattern = RegexTokenizer.WORD_PUNC
-                });
+            var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?");
 
-            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Start == 0);
             Assert.IsTrue(tokens[0].Text == "Chop");
 
-            Assert.IsTrue(tokens[1].Offset == 5);
+            Assert.IsTrue(tokens[1].Start == 5);
             Assert.IsTrue(tokens[1].Text == "into");
 
-            Assert.IsTrue(tokens[2].Offset == 10);
+            Assert.IsTrue(tokens[2].Start == 10);
             Assert.IsTrue(tokens[2].Text == "pieces");
 
-            Assert.IsTrue(tokens[3].Offset == 16);
+            Assert.IsTrue(tokens[3].Start == 16);
             Assert.IsTrue(tokens[3].Text == ",");
 
-            Assert.IsTrue(tokens[4].Offset == 18);
+            Assert.IsTrue(tokens[4].Start == 18);
             Assert.IsTrue(tokens[4].Text == "isn");
 
-            Assert.IsTrue(tokens[5].Offset == 21);
+            Assert.IsTrue(tokens[5].Start == 21);
             Assert.IsTrue(tokens[5].Text == "'");
 
-            Assert.IsTrue(tokens[6].Offset == 22);
+            Assert.IsTrue(tokens[6].Start == 22);
             Assert.IsTrue(tokens[6].Text == "t");
 
-            Assert.IsTrue(tokens[7].Offset == 24);
+            Assert.IsTrue(tokens[7].Start == 24);
             Assert.IsTrue(tokens[7].Text == "it");
 
-            Assert.IsTrue(tokens[8].Offset == 26);
+            Assert.IsTrue(tokens[8].Start == 26);
             Assert.IsTrue(tokens[8].Text == "?");
         }
 
         [TestMethod]
         public void TokenizeInBlankLine()
         {
-            var tokenizer = new TokenizerFactory<RegexTokenizer>();
+            var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
+            {
+                Pattern = RegexTokenizer.BLANK_LINE
+            }, SupportedLanguage.English);
 
             var tokens = tokenizer.Tokenize(@"Chop into pieces, 
 
 isn't
 
-it?",
-                new TokenizationOptions
-                {
-                    Pattern = RegexTokenizer.BLANK_LINE
-                });
+it?");
 
-            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Start == 0);
             Assert.IsTrue(tokens[0].Text == "Chop into pieces,");
 
-            Assert.IsTrue(tokens[1].Offset == 18);
+            Assert.IsTrue(tokens[1].Start == 18);
             Assert.IsTrue(tokens[1].Text == "isn't");
 
-            Assert.IsTrue(tokens[2].Offset == 28);
+            Assert.IsTrue(tokens[2].Start == 28);
             Assert.IsTrue(tokens[2].Text == "it?");
         }
     }

BotSharp.NLP/BotSharp.NLP.csproj

@@ -17,4 +17,14 @@
     <PackageTags>BotSharp, NLP, NLU, POS, CRS, NER, LSTM, CRF</PackageTags>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="DevExpress.Xpo" Version="18.1.4" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Update="NER\README.cs">
+      <SubType>Code</SubType>
+    </Compile>
+  </ItemGroup>
+
 </Project>

BotSharp.NLP/Corpus/CoNLL/README.md

@@ -0,0 +1,2 @@
+conll2000_chunking is downloaded from https://www.clips.uantwerpen.be/conll2000/chunking/
+The train and test data consist of three columns separated by spaces. Each word has been put on a separate line and there is an empty line after each sentence. The first column contains the current word, the second its part-of-speech tag as derived by the Brill tagger and the third its chunk tag as derived from the WSJ corpus. 

BotSharp.NLP/Corpus/ConllReader.cs

@@ -0,0 +1,53 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace BotSharp.NLP.Corpus
+{
+    /// <summary>
+    /// A corpus reader for CoNLL-style files.  These files consist of a
+    /// series of sentences, separated by blank lines.Each sentence is
+    /// encoded using a table(or "grid") of values, where each line
+    /// corresponds to a single word, and each column corresponds to an
+    /// annotation type.The set of columns used by CoNLL-style files can
+    /// vary from corpus to corpus;
+    /// </summary>
+    public class CoNLLReader
+    {
+        public List<Sentence> Read(ReaderOptions options)
+        {
+            var sentences = new List<Sentence>();
+            using(StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
+            {
+                string line = reader.ReadLine();
+                var sentence = new Sentence { Words = new List<Token> { } };
+
+                while (!reader.EndOfStream)
+                {
+                    if (String.IsNullOrEmpty(line))
+                    {
+                        sentences.Add(sentence);
+                        sentence = new Sentence { Words = new List<Token> { } };
+                    }
+                    else
+                    {
+                        var columns = line.Split(' ');
+
+                        sentence.Words.Add(new Token
+                        {
+                            Text = columns[0],
+                            Pos = columns[1]
+                        });
+                    }
+
+                    line = reader.ReadLine();
+                }
+                
+            }
+
+            return sentences;
+        }
+    }
+}

BotSharp.NLP/Corpus/ReaderOptions.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Corpus
+{
+    public class ReaderOptions
+    {
+        public string DataDir { get; set; }
+
+        public string FileName { get; set; }
+    }
+}

BotSharp.NLP/NER/README.md

@@ -0,0 +1,5 @@
+IOB tagging
+
+B-{CHUNK_TYPE} – for the word in the Beginning chunk
+I-{CHUNK_TYPE} – for words Inside the chunk
+O – Outside any chunk

BotSharp.NLP/Sentence.cs

@@ -0,0 +1,12 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP
+{
+    public class Sentence
+    {
+        public List<Token> Words { get; set; }
+    }
+}

BotSharp.NLP/Stem/IStemmer.cs

@@ -12,11 +12,6 @@ namespace BotSharp.NLP.Stem
     /// </summary>
     public interface IStemmer
     {
-        /// <summary>
-        /// Language
-        /// </summary>
-        SupportedLanguage Lang { get; set; }
-
         /// <summary>
         /// Strip affixes from the token and return the stem.
         /// </summary>

BotSharp.NLP/Stem/RegexStemmer.cs

@@ -15,8 +15,6 @@ namespace BotSharp.NLP.Stem
     {
         public const string DEFAULT = "ing$|s$|e$|able$";
 
-        public SupportedLanguage Lang { get; set; }
-
         private Regex _regex;
 
         public string Stem(string word, StemOptions options)

BotSharp.NLP/Stem/StemmerFactory.cs

@@ -14,16 +14,22 @@ namespace BotSharp.NLP.Stem
     /// <typeparam name="IStem"></typeparam>
     public class StemmerFactory<IStem> where IStem : IStemmer, new()
     {
+        private SupportedLanguage _lang { get; set; }
+
         private IStem _stemmer;
 
-        public StemmerFactory()
+        private StemOptions _options;
+
+        public StemmerFactory(StemOptions options, SupportedLanguage lang)
         {
+            _lang = lang;
+            _options = options;
             _stemmer = new IStem();
         }
 
-        public string Stem(string word, StemOptions options)
+        public string Stem(string word)
         {
-            return _stemmer.Stem(word, options);
+            return _stemmer.Stem(word, _options);
         }
     }
 }

BotSharp.NLP/Tag/DefaultTagger.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Tag
+{
+    /// <summary>
+    /// The simplest possible tagger assigns the same tag to each token. 
+    /// This may seem to be a rather banal step, but it establishes an important baseline for tagger performance. 
+    /// In order to get the best result, we tag each word with the most likely tag. 
+    /// </summary>
+    public class DefaultTagger : ITagger
+    {
+        public void Tag(Sentence sentence, TagOptions options)
+        {
+            
+        }
+
+        public void Train(List<Sentence> sentences, TagOptions options)
+        {
+            
+        }
+    }
+}

BotSharp.NLP/Tag/ITagger.cs

@@ -0,0 +1,24 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Tag
+{
+    /// <summary>
+    /// Part-Of-Speech tagging (or POS tagging, for short) is one of the main components of almost any NLP analysis. 
+    /// The task of POS-tagging simply implies labelling words with their appropriate Part-Of-Speech (Noun, Verb, Adjective, Adverb, Pronoun, …).
+    /// </summary>
+    public interface ITagger
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="sentences">A tagged corpus. Each item should be a list of tokens.</param>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        void Train(List<Sentence> sentences, TagOptions options);
+
+        void Tag(Sentence sentence, TagOptions options);
+    }
+}

BotSharp.NLP/Tag/NGramTagger.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Tag
+{
+    /// <summary>
+    /// N-Gramm taggers are based on a simple statistical algorithm: 
+    /// for each token, assign the tag that is most likely for that particular token.
+    /// </summary>
+    public class NGramTagger : ITagger
+    {
+        private List<NGramFreq> _contextMapping { get; set; }
+
+        public void Tag(Sentence sentence, TagOptions options)
+        {
+            // need training to generate model
+            if(_contextMapping == null)
+            {
+                Train(options.Corpus, options);
+            }
+        }
+
+        public void Train(List<Sentence> sentences, TagOptions options)
+        {
+            _contextMapping = new List<NGramFreq>();
+
+            for (int idx = 0; idx < options.Corpus.Count; idx++)
+            {
+                var sent = options.Corpus[idx];
+
+                for (int ngram = 1; ngram < options.NGram; ngram++)
+                {
+                    sent.Words.Insert(0, new Token { Text = "NIL", Pos = options.Tag, Start = (ngram - 1) * 3 });
+                }
+
+                int pos = options.NGram - 1;
+                for (pos = 1; pos < sent.Words.Count; pos++)
+                {
+                    var freq = new NGramFreq
+                    {
+                        PrecedingTokens = new List<Token> { sent.Words[pos - 1] },
+                        Token = sent.Words[pos],
+                        Count = 0
+                    };
+
+                    _contextMapping.Add(freq);
+                }
+            }
+
+            /*var results = (from c in cache
+                           group c by c.Item1 into g
+                           select new { g.Key, Count = g.Count() }).ToList();*/
+        }
+        
+        private class NGramFreq
+        {
+            /// <summary>
+            /// Tokens prior current token
+            /// </summary>
+            public List<Token> PrecedingTokens { get; set; }
+
+            /// <summary>
+            /// Current token tag
+            /// </summary>
+            public Token Token { get; set; }
+
+            /// <summary>
+            /// Occurence frequency
+            /// </summary>
+            public int Count { get; set; }
+
+            public string Context
+            {
+                get
+                {
+                    return $"{PrecedingTokens.First().Pos} {Token.Text} {Token.Pos}";
+                }
+            }
+        }
+    }
+}

BotSharp.NLP/Tag/TagOptions.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Tag
+{
+    public class TagOptions
+    {
+        /// <summary>
+        /// Display some stats, if requested.
+        /// </summary>
+        public bool Verbose { get; set; }
+
+        /// <summary>
+        /// Default Tag
+        /// Used in DefaultTagger
+        /// </summary>
+        public string Tag { get; set; }
+
+        /// <summary>
+        /// N-Gram number
+        /// </summary>
+        public int NGram { get; set; }
+
+        /// <summary>
+        /// Tagged corpus used for training a model
+        /// </summary>
+        public List<Sentence> Corpus { get; set; }
+    }
+}

BotSharp.NLP/Tag/TaggerFactory.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Tag
+{
+    public class TaggerFactory<ITag> where ITag : ITagger, new()
+    {
+        private SupportedLanguage _lang;
+
+        private ITag _tagger;
+
+        private TagOptions _options;
+
+        public TaggerFactory(TagOptions options, SupportedLanguage lang)
+        {
+            _lang = lang;
+            _options = options;
+            _tagger = new ITag();
+        }
+
+        public void Tag(Sentence sentence)
+        {
+            _tagger.Tag(sentence, _options);
+        }
+    }
+}

BotSharp.NLP/Tokenize/ITokenizer.cs

@@ -10,17 +10,12 @@ namespace BotSharp.NLP.Tokenize
     /// </summary>
     public interface ITokenizer
     {
-        /// <summary>
-        /// Language
-        /// </summary>
-        SupportedLanguage Lang { get; set; }
-
         /// <summary>
         /// Tokenize
         /// </summary>
-        /// <param name="text">input</param>
+        /// <param name="sentence">input sentence</param>
         /// <param name="options">Options such as: regex expression</param>
         /// <returns></returns>
-        Token[] Tokenize(string text, TokenizationOptions options);
+        List<Token> Tokenize(string sentence, TokenizationOptions options);
     }
 }

BotSharp.NLP/Tokenize/RegexTokenizer.cs

@@ -6,10 +6,11 @@ using System.Text.RegularExpressions;
 
 namespace BotSharp.NLP.Tokenize
 {
+    /// <summary>
+    /// Regular-Expression Tokenizers
+    /// </summary>
     public class RegexTokenizer : ITokenizer
     {
-        public SupportedLanguage Lang { get; set; }
-
         /// <summary>
         /// Tokenize a text into a sequence of alphabetic and non-alphabetic characters
         /// </summary>
@@ -31,11 +32,11 @@ namespace BotSharp.NLP.Tokenize
 
         private Regex _regex;
 
-        public Token[] Tokenize(string text, TokenizationOptions options)
+        public List<Token> Tokenize(string sentence, TokenizationOptions options)
         {
             _regex = new Regex(options.Pattern);
 
-            var matches = _regex.Matches(text).Cast<Match>().ToArray();
+            var matches = _regex.Matches(sentence).Cast<Match>().ToArray();
 
             options.IsGap = new string[] { WHITE_SPACE, BLANK_LINE }.Contains(options.Pattern);
 
@@ -48,8 +49,8 @@ namespace BotSharp.NLP.Tokenize
                 {
                     var token = new Token
                     {
-                        Text = (span == matches.Length) ? text.Substring(pos) : text.Substring(pos, matches[span].Index - pos),
-                        Offset = pos
+                        Text = (span == matches.Length) ? sentence.Substring(pos) : sentence.Substring(pos, matches[span].Index - pos),
+                        Start = pos
                     };
 
                     token.Text = token.Text.Trim();
@@ -62,15 +63,15 @@ namespace BotSharp.NLP.Tokenize
                     }
                 }
 
-                return tokens.ToArray();
+                return tokens.ToList();
             }
             else
             {
                 return matches.Select(x => new Token
                 {
                     Text = x.Value,
-                    Offset = x.Index
-                }).ToArray();
+                    Start = x.Index
+                }).ToList();
             }
         }
     }

BotSharp.NLP/Tokenize/Token.cs

@@ -6,17 +6,59 @@ namespace BotSharp.NLP.Tokenize
 {
     public class Token
     {
+        /// <summary>
+        /// The original word text.
+        /// </summary>
         public string Text { get; set; }
-        public int Offset { get; set; }
+
+        /// <summary>
+        /// The offset of word
+        /// </summary>
+        public int Start { get; set; }
+
+        /// <summary>
+        /// The simple part-of-speech tag.
+        /// Not widely used, Tag is more general.
+        /// </summary>
         public string Pos { get; set; }
+
+        /// <summary>
+        /// The detailed part-of-speech tag.
+        /// https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html
+        /// </summary>
         public string Tag { get; set; }
+
+        /// <summary>
+        /// The base form of the word.
+        /// </summary>
         public string Lemma { get; set; }
+
+        /// <summary>
+        /// The word shape – capitalisation, punctuation, digits.
+        /// </summary>
+        public string Shape { get; set; }
+
+        /// <summary>
+        /// Is the token an alpha character?
+        /// </summary>
+        public bool IsAlpha { get; set; }
+
+        /// <summary>
+        /// Is the token part of a stop list, i.e. the most common words of the language?
+        /// </summary>
+        public bool IsStop { get; set; }
+
         public int End
         {
             get
             {
-                return Offset + Text.Length - 1;
+                return Start + Text.Length - 1;
             }
         }
+
+        public override string ToString()
+        {
+            return $"{Text} {Start} {Pos}";
+        }
     }
 }

BotSharp.NLP/Tokenize/TokenizerFactory.cs

@@ -12,16 +12,22 @@ namespace BotSharp.NLP.Tokenize
     /// </summary>
     public class TokenizerFactory<ITokenize> where ITokenize : ITokenizer, new()
     {
+        private SupportedLanguage _lang;
+
         private ITokenize _tokenizer;
 
-        public TokenizerFactory()
+        private TokenizationOptions _options;
+
+        public TokenizerFactory(TokenizationOptions options, SupportedLanguage lang)
         {
+            _lang = lang;
+            _options = options;
             _tokenizer = new ITokenize();
         }
 
-        public Token[] Tokenize(string text, TokenizationOptions options)
+        public List<Token> Tokenize(string sentence)
         {
-            return _tokenizer.Tokenize(text, options);
+            return _tokenizer.Tokenize(sentence, _options);
         }
     }
 }