From 3f150fbbb749006a7ccaa348bdd6083900b3b203 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 15 Aug 2018 23:17:42 -0500 Subject: [PATCH] Add RegexStemmer and unit test. --- BotSharp.NLP.UnitTest/RegexStemmerTest.cs | 26 ++++++++++++++++ ...TokenizerTest.cs => RegexTokenizerTest.cs} | 2 +- BotSharp.NLP/Stem/IStemmer.cs | 28 +++++++++++++++++ BotSharp.NLP/Stem/RegexStemmer.cs | 31 +++++++++++++++++++ BotSharp.NLP/Stem/StemOptions.cs | 14 +++++++++ BotSharp.NLP/Stem/StemmerFactory.cs | 29 +++++++++++++++++ BotSharp.NLP/Tokenize/TokenizationOptions.cs | 6 ++++ 7 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 BotSharp.NLP.UnitTest/RegexStemmerTest.cs rename BotSharp.NLP.UnitTest/{RegexpTokenizerTest.cs => RegexTokenizerTest.cs} (98%) create mode 100644 BotSharp.NLP/Stem/IStemmer.cs create mode 100644 BotSharp.NLP/Stem/RegexStemmer.cs create mode 100644 BotSharp.NLP/Stem/StemOptions.cs create mode 100644 BotSharp.NLP/Stem/StemmerFactory.cs diff --git a/BotSharp.NLP.UnitTest/RegexStemmerTest.cs b/BotSharp.NLP.UnitTest/RegexStemmerTest.cs new file mode 100644 index 00000000..9cc42c5f --- /dev/null +++ b/BotSharp.NLP.UnitTest/RegexStemmerTest.cs @@ -0,0 +1,26 @@ +using BotSharp.NLP.Stem; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.UnitTest +{ + [TestClass] + public class RegexStemmerTest + { + [TestMethod] + public void StemInDefault() + { + var stemmer = new StemmerFactory(); + + var stem = stemmer.Stem("doing", + new StemOptions + { + Pattern = RegexStemmer.DEFAULT + }); + + Assert.IsTrue(stem == "do"); + } + } +} diff --git a/BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs b/BotSharp.NLP.UnitTest/RegexTokenizerTest.cs similarity index 98% rename from BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs rename to BotSharp.NLP.UnitTest/RegexTokenizerTest.cs index ddf4e937..0a40c37f 100644 --- a/BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs +++ b/BotSharp.NLP.UnitTest/RegexTokenizerTest.cs @@ -4,7 +4,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; namespace BotSharp.NLP.UnitTest { [TestClass] - public class RegexpTokenizerTest + public class RegexTokenizerTest { [TestMethod] public void TokenizeInWhiteSpace() diff --git a/BotSharp.NLP/Stem/IStemmer.cs b/BotSharp.NLP/Stem/IStemmer.cs new file mode 100644 index 00000000..afd5ca55 --- /dev/null +++ b/BotSharp.NLP/Stem/IStemmer.cs @@ -0,0 +1,28 @@ +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Stem +{ + /// + /// Stemmer is used to remove morphological affixes from words, leaving only the word stem. + /// Stemming algorithms aim to remove those affixes leaving only the stem of the word. + /// IStemmer defines a standard interface for stemmers. + /// + public interface IStemmer + { + /// + /// Language + /// + SupportedLanguage Lang { get; set; } + + /// + /// Strip affixes from the token and return the stem. + /// + /// + /// + /// + string Stem(string word, StemOptions options); + } +} diff --git a/BotSharp.NLP/Stem/RegexStemmer.cs b/BotSharp.NLP/Stem/RegexStemmer.cs new file mode 100644 index 00000000..23034418 --- /dev/null +++ b/BotSharp.NLP/Stem/RegexStemmer.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using BotSharp.NLP.Tokenize; + +namespace BotSharp.NLP.Stem +{ + /// + /// A stemmer that uses regular expressions to identify morphological affixes. + /// Any substrings that match the regular expressions will be removed. + /// + public class RegexStemmer : IStemmer + { + public const string DEFAULT = "ing$|s$|e$|able$"; + + public SupportedLanguage Lang { get; set; } + + private Regex _regex; + + public string Stem(string word, StemOptions options) + { + _regex = new Regex(options.Pattern); + + var match = _regex.Matches(word).Cast().FirstOrDefault(); + + return match == null ? word : word.Substring(0, match.Index); + } + } +} diff --git a/BotSharp.NLP/Stem/StemOptions.cs b/BotSharp.NLP/Stem/StemOptions.cs new file mode 100644 index 00000000..6022730c --- /dev/null +++ b/BotSharp.NLP/Stem/StemOptions.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Stem +{ + public class StemOptions + { + /// + /// Regex pattern + /// + public string Pattern { get; set; } + } +} diff --git a/BotSharp.NLP/Stem/StemmerFactory.cs b/BotSharp.NLP/Stem/StemmerFactory.cs new file mode 100644 index 00000000..a94899ed --- /dev/null +++ b/BotSharp.NLP/Stem/StemmerFactory.cs @@ -0,0 +1,29 @@ +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Stem +{ + /// + /// BotSharp Stemmer Factory + /// In linguistic morphology and information retrieval, + /// stemming is the process of reducing inflected (or sometimes derived) words to their word stem, + /// base or root form—generally a written word form. + /// + /// + public class StemmerFactory where IStem : IStemmer, new() + { + private IStem _stemmer; + + public StemmerFactory() + { + _stemmer = new IStem(); + } + + public string Stem(string word, StemOptions options) + { + return _stemmer.Stem(word, options); + } + } +} diff --git a/BotSharp.NLP/Tokenize/TokenizationOptions.cs b/BotSharp.NLP/Tokenize/TokenizationOptions.cs index bb9017d9..b2a5d0cf 100644 --- a/BotSharp.NLP/Tokenize/TokenizationOptions.cs +++ b/BotSharp.NLP/Tokenize/TokenizationOptions.cs @@ -16,5 +16,11 @@ namespace BotSharp.NLP.Tokenize /// False if this tokenizer's pattern should be used to find the tokens themselves. /// public bool IsGap { get; set; } + + /// + /// True if any empty tokens generated by the tokenizer should be discarded. + /// Empty tokens can only be generated if `IsGap == True` + /// + public bool IgnoreEmpty { get; set; } } }