Add RegexStemmer and unit test.

BotSharp.NLP.UnitTest/RegexStemmerTest.cs

@@ -0,0 +1,26 @@
+using BotSharp.NLP.Stem;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest
+{
+    [TestClass]
+    public class RegexStemmerTest
+    {
+        [TestMethod]
+        public void StemInDefault()
+        {
+            var stemmer = new StemmerFactory<RegexStemmer>();
+
+            var stem = stemmer.Stem("doing",
+                new StemOptions
+                {
+                    Pattern = RegexStemmer.DEFAULT
+                });
+
+            Assert.IsTrue(stem == "do");
+        }
+    }
+}

BotSharp.NLP.UnitTest/RegexTokenizerTest.cs

@@ -4,7 +4,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting;
 namespace BotSharp.NLP.UnitTest
 {
     [TestClass]
-    public class RegexpTokenizerTest
+    public class RegexTokenizerTest
     {
         [TestMethod]
         public void TokenizeInWhiteSpace()

BotSharp.NLP/Stem/IStemmer.cs

@@ -0,0 +1,28 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Stem
+{
+    /// <summary>
+    /// Stemmer is used to remove morphological affixes from words, leaving only the word stem.
+    /// Stemming algorithms aim to remove those affixes leaving only the stem of the word.
+    /// IStemmer defines a standard interface for stemmers.
+    /// </summary>
+    public interface IStemmer
+    {
+        /// <summary>
+        /// Language
+        /// </summary>
+        SupportedLanguage Lang { get; set; }
+
+        /// <summary>
+        /// Strip affixes from the token and return the stem.
+        /// </summary>
+        /// <param name="word"></param>
+        /// <param name="options"></param>
+        /// <returns></returns>
+        string Stem(string word, StemOptions options);
+    }
+}

BotSharp.NLP/Stem/RegexStemmer.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using BotSharp.NLP.Tokenize;
+
+namespace BotSharp.NLP.Stem
+{
+    /// <summary>
+    /// A stemmer that uses regular expressions to identify morphological affixes.
+    /// Any substrings that match the regular expressions will be removed.
+    /// </summary>
+    public class RegexStemmer : IStemmer
+    {
+        public const string DEFAULT = "ing$|s$|e$|able$";
+
+        public SupportedLanguage Lang { get; set; }
+
+        private Regex _regex;
+
+        public string Stem(string word, StemOptions options)
+        {
+            _regex = new Regex(options.Pattern);
+
+            var match = _regex.Matches(word).Cast<Match>().FirstOrDefault();
+        
+            return match == null ? word : word.Substring(0, match.Index);
+        }
+    }
+}

BotSharp.NLP/Stem/StemOptions.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Stem
+{
+    public class StemOptions
+    {
+        /// <summary>
+        /// Regex pattern
+        /// </summary>
+        public string Pattern { get; set; }
+    }
+}

BotSharp.NLP/Stem/StemmerFactory.cs

@@ -0,0 +1,29 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Stem
+{
+    /// <summary>
+    /// BotSharp Stemmer Factory
+    /// In linguistic morphology and information retrieval, 
+    /// stemming is the process of reducing inflected (or sometimes derived) words to their word stem, 
+    /// base or root form—generally a written word form.
+    /// </summary>
+    /// <typeparam name="IStem"></typeparam>
+    public class StemmerFactory<IStem> where IStem : IStemmer, new()
+    {
+        private IStem _stemmer;
+
+        public StemmerFactory()
+        {
+            _stemmer = new IStem();
+        }
+
+        public string Stem(string word, StemOptions options)
+        {
+            return _stemmer.Stem(word, options);
+        }
+    }
+}

BotSharp.NLP/Tokenize/TokenizationOptions.cs

@@ -16,5 +16,11 @@ namespace BotSharp.NLP.Tokenize
         /// False if this tokenizer's pattern should be used to find the tokens themselves.
         /// </summary>
         public bool IsGap { get; set; }
+
+        /// <summary>
+        /// True if any empty tokens generated by the tokenizer should be discarded.
+        /// Empty tokens can only be generated if `IsGap == True`
+        /// </summary>
+        public bool IgnoreEmpty { get; set; }
     }
 }