Finish RegexTokenizerTest unit test.

BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs

@@ -7,7 +7,7 @@ namespace BotSharp.NLP.UnitTest
     public class RegexpTokenizerTest
     {
         [TestMethod]
-        public void Tokenize()
+        public void TokenizeInWhiteSpace()
         {
             var tokenizer = new TokenizerFactory<RegexTokenizer>();
 
@@ -29,8 +29,72 @@ namespace BotSharp.NLP.UnitTest
             Assert.IsTrue(tokens[3].Offset == 18);
             Assert.IsTrue(tokens[3].Text == "isn't");
 
-            Assert.IsTrue(tokens[3].Offset == 24);
-            Assert.IsTrue(tokens[3].Text == "it?");
+            Assert.IsTrue(tokens[4].Offset == 24);
+            Assert.IsTrue(tokens[4].Text == "it?");
+        }
+
+        [TestMethod]
+        public void TokenizeInWordPunctuation()
+        {
+            var tokenizer = new TokenizerFactory<RegexTokenizer>();
+
+            var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?",
+                new TokenizationOptions
+                {
+                    Pattern = RegexTokenizer.WORD_PUNC
+                });
+
+            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Text == "Chop");
+
+            Assert.IsTrue(tokens[1].Offset == 5);
+            Assert.IsTrue(tokens[1].Text == "into");
+
+            Assert.IsTrue(tokens[2].Offset == 10);
+            Assert.IsTrue(tokens[2].Text == "pieces");
+
+            Assert.IsTrue(tokens[3].Offset == 16);
+            Assert.IsTrue(tokens[3].Text == ",");
+
+            Assert.IsTrue(tokens[4].Offset == 18);
+            Assert.IsTrue(tokens[4].Text == "isn");
+
+            Assert.IsTrue(tokens[5].Offset == 21);
+            Assert.IsTrue(tokens[5].Text == "'");
+
+            Assert.IsTrue(tokens[6].Offset == 22);
+            Assert.IsTrue(tokens[6].Text == "t");
+
+            Assert.IsTrue(tokens[7].Offset == 24);
+            Assert.IsTrue(tokens[7].Text == "it");
+
+            Assert.IsTrue(tokens[8].Offset == 26);
+            Assert.IsTrue(tokens[8].Text == "?");
+        }
+
+        [TestMethod]
+        public void TokenizeInBlankLine()
+        {
+            var tokenizer = new TokenizerFactory<RegexTokenizer>();
+
+            var tokens = tokenizer.Tokenize(@"Chop into pieces, 
+
+isn't
+
+it?",
+                new TokenizationOptions
+                {
+                    Pattern = RegexTokenizer.BLANK_LINE
+                });
+
+            Assert.IsTrue(tokens[0].Offset == 0);
+            Assert.IsTrue(tokens[0].Text == "Chop into pieces,");
+
+            Assert.IsTrue(tokens[1].Offset == 18);
+            Assert.IsTrue(tokens[1].Text == "isn't");
+
+            Assert.IsTrue(tokens[2].Offset == 28);
+            Assert.IsTrue(tokens[2].Text == "it?");
         }
     }
 }

BotSharp.NLP/Tokenize/RegexTokenizer.cs

@@ -42,29 +42,27 @@ namespace BotSharp.NLP.Tokenize
             if (options.IsGap)
             {
                 int pos = 0;
-                int span = 0;
+                var tokens = new Token[matches.Length + 1];
 
-                var tokens = matches.Select(x =>
+                for (int span = 0; span <= matches.Length; span++)
                 {
                     var token = new Token
                     {
-                        Text = (span == matches.Length - 1) ? text.Substring(pos) : text.Substring(pos, x.Index - pos),
+                        Text = (span == matches.Length) ? text.Substring(pos) : text.Substring(pos, matches[span].Index - pos),
                         Offset = pos
                     };
 
-                    pos = x.Index + 1;
+                    token.Text = token.Text.Trim();
 
-                    if (span == matches.Length - 1)
+                    tokens[span] = token;
+
+                    if (span < matches.Length)
                     {
-
+                        pos = matches[span].Index + 1;
                     }
-                    
-                    span++;
+                }
 
-                    return token;
-                }).ToArray();
-
-                return tokens;
+                return tokens.ToArray();
             }
             else
             {
@@ -74,7 +72,6 @@ namespace BotSharp.NLP.Tokenize
                     Offset = x.Index
                 }).ToArray();
             }
-
         }
     }
 }