Merge branch 'master' of https://github.com/Oceania2018/BotSharp
This commit is contained in:
Bolo Peng
commit
42e62f0d00
|
|
@ -2,7 +2,7 @@ using BotSharp.NLP.Tokenize;
|
|||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BotSharp.NLP.UnitTest
|
||||
namespace BotSharp.NLP.UnitTest.Tokenize
|
||||
{
|
||||
[TestClass]
|
||||
public class RegexTokenizerTest
|
||||
52
BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs
Normal file
52
BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
using BotSharp.NLP.Tokenize;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace BotSharp.NLP.UnitTest.Tokenize
|
||||
{
|
||||
[TestClass]
|
||||
public class TreebankTokenizerTest
|
||||
{
|
||||
[TestMethod]
|
||||
public void ReplaceStartingQuoting()
|
||||
{
|
||||
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
|
||||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("(\"«Hello World.");
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void ReplacePunctuation()
|
||||
{
|
||||
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
|
||||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("Hello World...");
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void ReplaceBrackets()
|
||||
{
|
||||
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
|
||||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("<Hello World.>");
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void ReplaceConventions()
|
||||
{
|
||||
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
|
||||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("I cannot jump.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -27,5 +27,10 @@ namespace BotSharp.NLP.Tokenize
|
|||
/// Split "isn't" into "is", "n't"
|
||||
/// </summary>
|
||||
public List<string> SpecialWords { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Convert bracket-like characters to avoid confusion with parse brackets.
|
||||
/// </summary>
|
||||
public bool ConvertParentheses { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
160
BotSharp.NLP/Tokenize/TreebankTokenizer.cs
Normal file
160
BotSharp.NLP/Tokenize/TreebankTokenizer.cs
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
/*
|
||||
* BotSharp.NLP Library
|
||||
* Copyright (C) 2018 Haiping Chen
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BotSharp.NLP.Tokenize
|
||||
{
|
||||
/// <summary>
|
||||
/// Penn Treebank Tokenizer
|
||||
/// The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
|
||||
/// This implementation is a port of the tokenizer sed script written by Robert McIntyre
|
||||
/// and available at ftp://ftp.cis.upenn.edu/pub/treebank/public_html/tokenizer.sed,
|
||||
/// or reference ftp://ftp.cis.upenn.edu/pub/treebank/public_html/tokenization.html.
|
||||
/// </summary>
|
||||
public class TreebankTokenizer : ITokenizer
|
||||
{
|
||||
private List<Tuple<String, String>> STARTING_QUOTES = new List<Tuple<string, string>>();
|
||||
private List<Tuple<String, String>> PUNCTUATION = new List<Tuple<string, string>>();
|
||||
private List<Tuple<String, String>> PARENS_BRACKETS = new List<Tuple<string, string>>();
|
||||
private List<Tuple<String, String>> CONVERT_PARENTHESES = new List<Tuple<string, string>>();
|
||||
private List<Tuple<String, String>> ENDING_QUOTES = new List<Tuple<string, string>>();
|
||||
private List<Tuple<String, String>> CONVENTIONS = new List<Tuple<string, string>>();
|
||||
|
||||
public TreebankTokenizer()
|
||||
{
|
||||
Init();
|
||||
}
|
||||
|
||||
public List<Token> Tokenize(string sentence, TokenizationOptions options)
|
||||
{
|
||||
// starting quoting replace
|
||||
STARTING_QUOTES.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// replace PUNCTUATION
|
||||
PUNCTUATION.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// Handles parentheses.
|
||||
PARENS_BRACKETS.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// convert parentheses
|
||||
if (options.ConvertParentheses)
|
||||
{
|
||||
CONVERT_PARENTHESES.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
});
|
||||
}
|
||||
|
||||
// Handles repeated dash.
|
||||
sentence = Regex.Replace(sentence, "(-{2,})", " $1 ");
|
||||
|
||||
// replace ending quotes
|
||||
ENDING_QUOTES.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// replace ending quotes
|
||||
CONVENTIONS.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// remove duplicated spaces
|
||||
sentence = Regex.Replace(sentence, "\\s+", " ");
|
||||
|
||||
// split
|
||||
int pos = 0;
|
||||
|
||||
var results = Regex.Matches(sentence, "\\s")
|
||||
.Cast<Match>()
|
||||
.Select(x => {
|
||||
|
||||
var token = new Token
|
||||
{
|
||||
Start = pos,
|
||||
Text = sentence.Substring(pos, x.Index - pos)
|
||||
};
|
||||
|
||||
pos = x.Index + 1;
|
||||
|
||||
return token;
|
||||
|
||||
}).ToList();
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private void Init()
|
||||
{
|
||||
STARTING_QUOTES.Add(new Tuple<string, string>(@"([«“‘„]|[`]+)", " $1 "));
|
||||
STARTING_QUOTES.Add(new Tuple<string, string>("^\"", "``"));
|
||||
STARTING_QUOTES.Add(new Tuple<string, string>(@"(``)", " $1 "));
|
||||
STARTING_QUOTES.Add(new Tuple<string, string>("([ ([{<])(\" | '{2})", "$1 `` "));
|
||||
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\'»”’ ]*)\s*$", "$1 $2 $3 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])([^\d])", " $1 $2"));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])$", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"(\.\.\.)", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([;@#$%&])", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\']*)\\s*$", "$1 $2 $3 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"[?!]", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^'])' ", "$1 ' "));
|
||||
|
||||
PARENS_BRACKETS.Add(new Tuple<string, string>(@"([\]\[\(\)\{\}\<\>])", " $1 "));
|
||||
|
||||
CONVERT_PARENTHESES.Add(new Tuple<string, string>(@"\(", "-LRB-"));
|
||||
CONVERT_PARENTHESES.Add(new Tuple<string, string>(@"\)", "-RRB-"));
|
||||
CONVERT_PARENTHESES.Add(new Tuple<string, string>(@"\[", "-LSB-"));
|
||||
CONVERT_PARENTHESES.Add(new Tuple<string, string>(@"\]", "-RRB-"));
|
||||
CONVERT_PARENTHESES.Add(new Tuple<string, string>(@"\{", "-LCB-"));
|
||||
CONVERT_PARENTHESES.Add(new Tuple<string, string>(@"\}", "-RCB-"));
|
||||
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([»”’])", " $1 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>("\"", " '' "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"(\S)(\'\')", "$1 $2 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([^' ])('[sS]|'[mM]|'[dD]|') ", "$1 $2 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", "$1 $2 "));
|
||||
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(can)(?#X)(not)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(d)(?#X)('ye)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(gim)(?#X)(me)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(gon)(?#X)(na)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(got)(?#X)(ta)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(lem)(?#X)(me)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(mor)(?#X)('n)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(wan)(?#X)(na)\s", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i) ('t)(?#X)(is)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i) ('t)(?#X)(was)\b", "$1 $2 "));
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue