From 3e55192606e46f6233ff7380e5e3fdedde24b88c Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 5 Sep 2018 13:53:58 -0500 Subject: [PATCH] Added Penn Treebank Tokenizer. --- .../{ => Tokenize}/RegexTokenizerTest.cs | 2 +- .../Tokenize/TreebankTokenizerTest.cs | 52 ++++++ BotSharp.NLP/Tokenize/TokenizationOptions.cs | 5 + BotSharp.NLP/Tokenize/TreebankTokenizer.cs | 160 ++++++++++++++++++ 4 files changed, 218 insertions(+), 1 deletion(-) rename BotSharp.NLP.UnitTest/{ => Tokenize}/RegexTokenizerTest.cs (98%) create mode 100644 BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs create mode 100644 BotSharp.NLP/Tokenize/TreebankTokenizer.cs diff --git a/BotSharp.NLP.UnitTest/RegexTokenizerTest.cs b/BotSharp.NLP.UnitTest/Tokenize/RegexTokenizerTest.cs similarity index 98% rename from BotSharp.NLP.UnitTest/RegexTokenizerTest.cs rename to BotSharp.NLP.UnitTest/Tokenize/RegexTokenizerTest.cs index 795759fc..ce93e189 100644 --- a/BotSharp.NLP.UnitTest/RegexTokenizerTest.cs +++ b/BotSharp.NLP.UnitTest/Tokenize/RegexTokenizerTest.cs @@ -2,7 +2,7 @@ using BotSharp.NLP.Tokenize; using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; -namespace BotSharp.NLP.UnitTest +namespace BotSharp.NLP.UnitTest.Tokenize { [TestClass] public class RegexTokenizerTest diff --git a/BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs b/BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs new file mode 100644 index 00000000..c6e25f70 --- /dev/null +++ b/BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs @@ -0,0 +1,52 @@ +using BotSharp.NLP.Tokenize; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.UnitTest.Tokenize +{ + [TestClass] + public class TreebankTokenizerTest + { + [TestMethod] + public void ReplaceStartingQuoting() + { + var tokenizer = new TokenizerFactory(new TokenizationOptions + { + }, SupportedLanguage.English); + + var tokens = tokenizer.Tokenize("(\"«Hello World."); + } + + [TestMethod] + public void ReplacePunctuation() + { + var tokenizer = new TokenizerFactory(new TokenizationOptions + { + }, SupportedLanguage.English); + + var tokens = tokenizer.Tokenize("Hello World..."); + } + + [TestMethod] + public void ReplaceBrackets() + { + var tokenizer = new TokenizerFactory(new TokenizationOptions + { + }, SupportedLanguage.English); + + var tokens = tokenizer.Tokenize(""); + } + + [TestMethod] + public void ReplaceConventions() + { + var tokenizer = new TokenizerFactory(new TokenizationOptions + { + }, SupportedLanguage.English); + + var tokens = tokenizer.Tokenize("I cannot jump."); + } + } +} diff --git a/BotSharp.NLP/Tokenize/TokenizationOptions.cs b/BotSharp.NLP/Tokenize/TokenizationOptions.cs index 19057802..4501b888 100644 --- a/BotSharp.NLP/Tokenize/TokenizationOptions.cs +++ b/BotSharp.NLP/Tokenize/TokenizationOptions.cs @@ -27,5 +27,10 @@ namespace BotSharp.NLP.Tokenize /// Split "isn't" into "is", "n't" /// public List SpecialWords { get; set; } + + /// + /// Convert bracket-like characters to avoid confusion with parse brackets. + /// + public bool ConvertParentheses { get; set; } } } diff --git a/BotSharp.NLP/Tokenize/TreebankTokenizer.cs b/BotSharp.NLP/Tokenize/TreebankTokenizer.cs new file mode 100644 index 00000000..d4b9c10c --- /dev/null +++ b/BotSharp.NLP/Tokenize/TreebankTokenizer.cs @@ -0,0 +1,160 @@ +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.NLP.Tokenize +{ + /// + /// Penn Treebank Tokenizer + /// The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank. + /// This implementation is a port of the tokenizer sed script written by Robert McIntyre + /// and available at ftp://ftp.cis.upenn.edu/pub/treebank/public_html/tokenizer.sed, + /// or reference ftp://ftp.cis.upenn.edu/pub/treebank/public_html/tokenization.html. + /// + public class TreebankTokenizer : ITokenizer + { + private List> STARTING_QUOTES = new List>(); + private List> PUNCTUATION = new List>(); + private List> PARENS_BRACKETS = new List>(); + private List> CONVERT_PARENTHESES = new List>(); + private List> ENDING_QUOTES = new List>(); + private List> CONVENTIONS = new List>(); + + public TreebankTokenizer() + { + Init(); + } + + public List Tokenize(string sentence, TokenizationOptions options) + { + // starting quoting replace + STARTING_QUOTES.ForEach(x => + { + sentence = Regex.Replace(sentence, x.Item1, x.Item2); + }); + + // replace PUNCTUATION + PUNCTUATION.ForEach(x => + { + sentence = Regex.Replace(sentence, x.Item1, x.Item2); + }); + + // Handles parentheses. + PARENS_BRACKETS.ForEach(x => + { + sentence = Regex.Replace(sentence, x.Item1, x.Item2); + }); + + // convert parentheses + if (options.ConvertParentheses) + { + CONVERT_PARENTHESES.ForEach(x => + { + sentence = Regex.Replace(sentence, x.Item1, x.Item2); + }); + } + + // Handles repeated dash. + sentence = Regex.Replace(sentence, "(-{2,})", " $1 "); + + // replace ending quotes + ENDING_QUOTES.ForEach(x => + { + sentence = Regex.Replace(sentence, x.Item1, x.Item2); + }); + + // replace ending quotes + CONVENTIONS.ForEach(x => + { + sentence = Regex.Replace(sentence, x.Item1, x.Item2); + }); + + // remove duplicated spaces + sentence = Regex.Replace(sentence, "\\s+", " "); + + // split + int pos = 0; + + var results = Regex.Matches(sentence, "\\s") + .Cast() + .Select(x => { + + var token = new Token + { + Start = pos, + Text = sentence.Substring(pos, x.Index - pos) + }; + + pos = x.Index + 1; + + return token; + + }).ToList(); + + return results; + } + + private void Init() + { + STARTING_QUOTES.Add(new Tuple(@"([«“‘„]|[`]+)", " $1 ")); + STARTING_QUOTES.Add(new Tuple("^\"", "``")); + STARTING_QUOTES.Add(new Tuple(@"(``)", " $1 ")); + STARTING_QUOTES.Add(new Tuple("([ ([{<])(\" | '{2})", "$1 `` ")); + + PUNCTUATION.Add(new Tuple(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\'»”’ ]*)\s*$", "$1 $2 $3 ")); + PUNCTUATION.Add(new Tuple(@"([:,])([^\d])", " $1 $2")); + PUNCTUATION.Add(new Tuple(@"([:,])$", " $1 ")); + PUNCTUATION.Add(new Tuple(@"(\.\.\.)", " $1 ")); + PUNCTUATION.Add(new Tuple(@"([;@#$%&])", " $1 ")); + PUNCTUATION.Add(new Tuple(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\']*)\\s*$", "$1 $2 $3 ")); + PUNCTUATION.Add(new Tuple(@"[?!]", " $1 ")); + PUNCTUATION.Add(new Tuple(@"([^'])' ", "$1 ' ")); + + PARENS_BRACKETS.Add(new Tuple(@"([\]\[\(\)\{\}\<\>])", " $1 ")); + + CONVERT_PARENTHESES.Add(new Tuple(@"\(", "-LRB-")); + CONVERT_PARENTHESES.Add(new Tuple(@"\)", "-RRB-")); + CONVERT_PARENTHESES.Add(new Tuple(@"\[", "-LSB-")); + CONVERT_PARENTHESES.Add(new Tuple(@"\]", "-RRB-")); + CONVERT_PARENTHESES.Add(new Tuple(@"\{", "-LCB-")); + CONVERT_PARENTHESES.Add(new Tuple(@"\}", "-RCB-")); + + ENDING_QUOTES.Add(new Tuple(@"([»”’])", " $1 ")); + ENDING_QUOTES.Add(new Tuple("\"", " '' ")); + ENDING_QUOTES.Add(new Tuple(@"(\S)(\'\')", "$1 $2 ")); + ENDING_QUOTES.Add(new Tuple(@"([^' ])('[sS]|'[mM]|'[dD]|') ", "$1 $2 ")); + ENDING_QUOTES.Add(new Tuple(@"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", "$1 $2 ")); + + CONVENTIONS.Add(new Tuple(@"(?i)\b(can)(?#X)(not)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(d)(?#X)('ye)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(gim)(?#X)(me)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(gon)(?#X)(na)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(got)(?#X)(ta)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(lem)(?#X)(me)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(mor)(?#X)('n)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i)\b(wan)(?#X)(na)\s", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i) ('t)(?#X)(is)\b", "$1 $2 ")); + CONVENTIONS.Add(new Tuple(@"(?i) ('t)(?#X)(was)\b", "$1 $2 ")); + } + } +}