Add nlp tagger structure.

BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs

@@ -207,7 +207,7 @@ namespace BotSharp.Core.Engines.NERs
                 entities.Add(new NlpEntity
                 {
                     Entity = entity,
-                    Start = doc.Sentences[0].Tokens[i].Offset,
+                    Start = doc.Sentences[0].Tokens[i].Start,
                     Value = doc.Sentences[0].Tokens[i].Text,
                     Confidence = probability
                 });

BotSharp.NLP.UnitTest/DefaultTaggerTest.cs

@@ -0,0 +1,35 @@
+using BotSharp.NLP.Corpus;
+using BotSharp.NLP.Tag;
+using BotSharp.NLP.Tokenize;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest
+{
+    [TestClass]
+    public class DefaultTaggerTest
+    {
+        [TestMethod]
+        public void TagInCoNLL2000()
+        {
+            var sentences = new CoNLLReader()
+                .Read(new ReaderOptions
+                {
+                    DataDir = AppDomain.CurrentDomain.BaseDirectory,
+                    FileName = "conll2000_chunking_train"
+                });
+
+            var tagger = new TaggerFactory<DefaultTagger>();
+
+            // tokenize
+            
+            tagger.Tag(null,
+                new TagOptions
+                {
+                    
+                });
+        }
+    }
+}

BotSharp.NLP.UnitTest/RegexTokenizerTest.cs

@@ -17,19 +17,19 @@ namespace BotSharp.NLP.UnitTest
                     Pattern = RegexTokenizer.WHITE_SPACE
                 });
 
-            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Start == 0);
             Assert.IsTrue(tokens[0].Text == "Chop");
 
-            Assert.IsTrue(tokens[1].Offset == 5);
+            Assert.IsTrue(tokens[1].Start == 5);
             Assert.IsTrue(tokens[1].Text == "into");
 
-            Assert.IsTrue(tokens[2].Offset == 10);
+            Assert.IsTrue(tokens[2].Start == 10);
             Assert.IsTrue(tokens[2].Text == "pieces,");
 
-            Assert.IsTrue(tokens[3].Offset == 18);
+            Assert.IsTrue(tokens[3].Start == 18);
             Assert.IsTrue(tokens[3].Text == "isn't");
 
-            Assert.IsTrue(tokens[4].Offset == 24);
+            Assert.IsTrue(tokens[4].Start == 24);
             Assert.IsTrue(tokens[4].Text == "it?");
         }
 
@@ -44,31 +44,31 @@ namespace BotSharp.NLP.UnitTest
                     Pattern = RegexTokenizer.WORD_PUNC
                 });
 
-            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Start == 0);
             Assert.IsTrue(tokens[0].Text == "Chop");
 
-            Assert.IsTrue(tokens[1].Offset == 5);
+            Assert.IsTrue(tokens[1].Start == 5);
             Assert.IsTrue(tokens[1].Text == "into");
 
-            Assert.IsTrue(tokens[2].Offset == 10);
+            Assert.IsTrue(tokens[2].Start == 10);
             Assert.IsTrue(tokens[2].Text == "pieces");
 
-            Assert.IsTrue(tokens[3].Offset == 16);
+            Assert.IsTrue(tokens[3].Start == 16);
             Assert.IsTrue(tokens[3].Text == ",");
 
-            Assert.IsTrue(tokens[4].Offset == 18);
+            Assert.IsTrue(tokens[4].Start == 18);
             Assert.IsTrue(tokens[4].Text == "isn");
 
-            Assert.IsTrue(tokens[5].Offset == 21);
+            Assert.IsTrue(tokens[5].Start == 21);
             Assert.IsTrue(tokens[5].Text == "'");
 
-            Assert.IsTrue(tokens[6].Offset == 22);
+            Assert.IsTrue(tokens[6].Start == 22);
             Assert.IsTrue(tokens[6].Text == "t");
 
-            Assert.IsTrue(tokens[7].Offset == 24);
+            Assert.IsTrue(tokens[7].Start == 24);
             Assert.IsTrue(tokens[7].Text == "it");
 
-            Assert.IsTrue(tokens[8].Offset == 26);
+            Assert.IsTrue(tokens[8].Start == 26);
             Assert.IsTrue(tokens[8].Text == "?");
         }
 
@@ -87,13 +87,13 @@ it?",
                     Pattern = RegexTokenizer.BLANK_LINE
                 });
 
-            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Start == 0);
             Assert.IsTrue(tokens[0].Text == "Chop into pieces,");
 
-            Assert.IsTrue(tokens[1].Offset == 18);
+            Assert.IsTrue(tokens[1].Start == 18);
             Assert.IsTrue(tokens[1].Text == "isn't");
 
-            Assert.IsTrue(tokens[2].Offset == 28);
+            Assert.IsTrue(tokens[2].Start == 28);
             Assert.IsTrue(tokens[2].Text == "it?");
         }
     }

BotSharp.NLP/BotSharp.NLP.csproj

@@ -5,4 +5,14 @@
     <Platforms>AnyCPU;x64</Platforms>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="DevExpress.Xpo" Version="18.1.4" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Update="NER\README.cs">
+      <SubType>Code</SubType>
+    </Compile>
+  </ItemGroup>
+
 </Project>

BotSharp.NLP/Corpus/CoNLL/README.md

@@ -0,0 +1 @@
+conll2000_chunking is downloaded from https://www.clips.uantwerpen.be/conll2000/chunking/

BotSharp.NLP/Corpus/ConllReader.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace BotSharp.NLP.Corpus
+{
+    /// <summary>
+    /// A corpus reader for CoNLL-style files.  These files consist of a
+    /// series of sentences, separated by blank lines.Each sentence is
+    /// encoded using a table(or "grid") of values, where each line
+    /// corresponds to a single word, and each column corresponds to an
+    /// annotation type.The set of columns used by CoNLL-style files can
+    /// vary from corpus to corpus;
+    /// </summary>
+    public class CoNLLReader
+    {
+        public List<Sentence> Read(ReaderOptions options)
+        {
+            string corpus = File.ReadAllText(Path.Combine(options.DataDir, "conll2000_chunking_train.txt"));
+
+            return null;
+        }
+    }
+}

BotSharp.NLP/Corpus/ReaderOptions.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Corpus
+{
+    public class ReaderOptions
+    {
+        public string DataDir { get; set; }
+
+        public string FileName { get; set; }
+    }
+}

BotSharp.NLP/NER/README.md

@@ -0,0 +1,5 @@
+IOB tagging
+
+B-{CHUNK_TYPE} – for the word in the Beginning chunk
+I-{CHUNK_TYPE} – for words Inside the chunk
+O – Outside any chunk

BotSharp.NLP/Sentence.cs

@@ -0,0 +1,12 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP
+{
+    public class Sentence
+    {
+        public List<Token> Words { get; set; }
+    }
+}

BotSharp.NLP/Tag/DefaultTagger.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Tag
+{
+    /// <summary>
+    /// The simplest possible tagger assigns the same tag to each token. 
+    /// This may seem to be a rather banal step, but it establishes an important baseline for tagger performance. 
+    /// In order to get the best result, we tag each word with the most likely tag. 
+    /// </summary>
+    public class DefaultTagger : ITagger
+    {
+        public SupportedLanguage Lang { get; set; }
+
+        public void Tag(Sentence sentence, TagOptions options)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Train(List<Sentence> sentences, TagOptions options)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

BotSharp.NLP/Tag/ITagger.cs

@@ -0,0 +1,29 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Tag
+{
+    /// <summary>
+    /// Part-Of-Speech tagging (or POS tagging, for short) is one of the main components of almost any NLP analysis. 
+    /// The task of POS-tagging simply implies labelling words with their appropriate Part-Of-Speech (Noun, Verb, Adjective, Adverb, Pronoun, …).
+    /// </summary>
+    public interface ITagger
+    {
+        /// <summary>
+        /// Language
+        /// </summary>
+        SupportedLanguage Lang { get; set; }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="sentences">A tagged corpus. Each item should be a list of tokens.</param>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        void Train(List<Sentence> sentences, TagOptions options);
+
+        void Tag(Sentence sentence, TagOptions options);
+    }
+}

BotSharp.NLP/Tag/NGramTagger.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Tag
+{
+    /// <summary>
+    /// N-Gramm taggers are based on a simple statistical algorithm: 
+    /// for each token, assign the tag that is most likely for that particular token.
+    /// </summary>
+    public class NGramTagger : ITagger
+    {
+        public SupportedLanguage Lang { get; set; }
+
+        public void Tag(Sentence sentence, TagOptions options)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Train(List<Sentence> sentences, TagOptions options)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

BotSharp.NLP/Tag/TagOptions.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Tag
+{
+    public class TagOptions
+    {
+        /// <summary>
+        /// Display some stats, if requested.
+        /// </summary>
+        public bool Verbose { get; set; }
+
+        /// <summary>
+        /// Default Tag
+        /// Used in DefaultTagger
+        /// </summary>
+        public string Tag { get; set; }
+    }
+}

BotSharp.NLP/Tag/TaggerFactory.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Tag
+{
+    public class TaggerFactory<ITag> where ITag : ITagger, new()
+    {
+        private ITag _tagger;
+
+        public TaggerFactory()
+        {
+            _tagger = new ITag();
+        }
+
+        public void Tag(Sentence sentence, TagOptions options)
+        {
+            _tagger.Tag(sentence, options);
+        }
+    }
+}

BotSharp.NLP/Tokenize/ITokenizer.cs

@@ -18,9 +18,9 @@ namespace BotSharp.NLP.Tokenize
         /// <summary>
         /// Tokenize
         /// </summary>
-        /// <param name="text">input</param>
+        /// <param name="sentence">input sentence</param>
         /// <param name="options">Options such as: regex expression</param>
         /// <returns></returns>
-        Token[] Tokenize(string text, TokenizationOptions options);
+        Token[] Tokenize(string sentence, TokenizationOptions options);
     }
 }

BotSharp.NLP/Tokenize/RegexTokenizer.cs

@@ -6,6 +6,9 @@ using System.Text.RegularExpressions;
 
 namespace BotSharp.NLP.Tokenize
 {
+    /// <summary>
+    /// Regular-Expression Tokenizers
+    /// </summary>
     public class RegexTokenizer : ITokenizer
     {
         public SupportedLanguage Lang { get; set; }
@@ -31,11 +34,11 @@ namespace BotSharp.NLP.Tokenize
 
         private Regex _regex;
 
-        public Token[] Tokenize(string text, TokenizationOptions options)
+        public Token[] Tokenize(string sentence, TokenizationOptions options)
         {
             _regex = new Regex(options.Pattern);
 
-            var matches = _regex.Matches(text).Cast<Match>().ToArray();
+            var matches = _regex.Matches(sentence).Cast<Match>().ToArray();
 
             options.IsGap = new string[] { WHITE_SPACE, BLANK_LINE }.Contains(options.Pattern);
 
@@ -48,8 +51,8 @@ namespace BotSharp.NLP.Tokenize
                 {
                     var token = new Token
                     {
-                        Text = (span == matches.Length) ? text.Substring(pos) : text.Substring(pos, matches[span].Index - pos),
-                        Offset = pos
+                        Text = (span == matches.Length) ? sentence.Substring(pos) : sentence.Substring(pos, matches[span].Index - pos),
+                        Start = pos
                     };
 
                     token.Text = token.Text.Trim();
@@ -69,7 +72,7 @@ namespace BotSharp.NLP.Tokenize
                 return matches.Select(x => new Token
                 {
                     Text = x.Value,
-                    Offset = x.Index
+                    Start = x.Index
                 }).ToArray();
             }
         }

BotSharp.NLP/Tokenize/Token.cs

@@ -6,16 +6,53 @@ namespace BotSharp.NLP.Tokenize
 {
     public class Token
     {
+        /// <summary>
+        /// The original word text.
+        /// </summary>
         public string Text { get; set; }
-        public int Offset { get; set; }
+
+        /// <summary>
+        /// The offset of word
+        /// </summary>
+        public int Start { get; set; }
+
+        /// <summary>
+        /// The simple part-of-speech tag.
+        /// Not widely used, Tag is more general.
+        /// </summary>
         public string Pos { get; set; }
+
+        /// <summary>
+        /// The detailed part-of-speech tag.
+        /// https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html
+        /// </summary>
         public string Tag { get; set; }
+
+        /// <summary>
+        /// The base form of the word.
+        /// </summary>
         public string Lemma { get; set; }
+
+        /// <summary>
+        /// The word shape – capitalisation, punctuation, digits.
+        /// </summary>
+        public string Shape { get; set; }
+
+        /// <summary>
+        /// Is the token an alpha character?
+        /// </summary>
+        public bool IsAlpha { get; set; }
+
+        /// <summary>
+        /// Is the token part of a stop list, i.e. the most common words of the language?
+        /// </summary>
+        public bool IsStop { get; set; }
+
         public int End
         {
             get
             {
-                return Offset + Text.Length - 1;
+                return Start + Text.Length - 1;
             }
         }
     }

BotSharp.NLP/Tokenize/TokenizerFactory.cs

@@ -19,9 +19,9 @@ namespace BotSharp.NLP.Tokenize
             _tokenizer = new ITokenize();
         }
 
-        public Token[] Tokenize(string text, TokenizationOptions options)
+        public Token[] Tokenize(string sentence, TokenizationOptions options)
         {
-            return _tokenizer.Tokenize(text, options);
+            return _tokenizer.Tokenize(sentence, options);
         }
     }
 }