Add CorrectTokenPosition to modify token start position.
This commit is contained in:
Oceania2018
parent
3e55192606
commit
f61859e5a4
|
|
@ -16,7 +16,35 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
|||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("(\"«Hello World.");
|
||||
var tokens = tokenizer.Tokenize("«Hello!");
|
||||
|
||||
Assert.IsTrue(tokens[0].Text == "«");
|
||||
Assert.IsTrue(tokens[0].Start == 0);
|
||||
|
||||
Assert.IsTrue(tokens[1].Text == "Hello");
|
||||
Assert.IsTrue(tokens[1].Start == 1);
|
||||
|
||||
Assert.IsTrue(tokens[2].Text == "!");
|
||||
Assert.IsTrue(tokens[2].Start == 6);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void ReplaceEndingQuoting()
|
||||
{
|
||||
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
|
||||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("Aren't you");
|
||||
|
||||
Assert.IsTrue(tokens[0].Text == "Are");
|
||||
Assert.IsTrue(tokens[0].Start == 0);
|
||||
|
||||
Assert.IsTrue(tokens[1].Text == "n't");
|
||||
Assert.IsTrue(tokens[1].Start == 3);
|
||||
|
||||
Assert.IsTrue(tokens[2].Text == "you");
|
||||
Assert.IsTrue(tokens[2].Start == 7);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
|
|
@ -27,6 +55,15 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
|||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("Hello World...");
|
||||
|
||||
Assert.IsTrue(tokens[0].Text == "Hello");
|
||||
Assert.IsTrue(tokens[0].Start == 0);
|
||||
|
||||
Assert.IsTrue(tokens[1].Text == "World");
|
||||
Assert.IsTrue(tokens[1].Start == 6);
|
||||
|
||||
Assert.IsTrue(tokens[2].Text == "...");
|
||||
Assert.IsTrue(tokens[2].Start == 11);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
|
|
@ -36,7 +73,19 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
|||
{
|
||||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("<Hello World.>");
|
||||
var tokens = tokenizer.Tokenize("<Hello.>");
|
||||
|
||||
Assert.IsTrue(tokens[0].Text == "<");
|
||||
Assert.IsTrue(tokens[0].Start == 0);
|
||||
|
||||
Assert.IsTrue(tokens[1].Text == "Hello");
|
||||
Assert.IsTrue(tokens[1].Start == 1);
|
||||
|
||||
Assert.IsTrue(tokens[2].Text == ".");
|
||||
Assert.IsTrue(tokens[2].Start == 6);
|
||||
|
||||
Assert.IsTrue(tokens[3].Text == ">");
|
||||
Assert.IsTrue(tokens[3].Start == 7);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
|
|
@ -47,6 +96,21 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
|||
}, SupportedLanguage.English);
|
||||
|
||||
var tokens = tokenizer.Tokenize("I cannot jump.");
|
||||
|
||||
Assert.IsTrue(tokens[0].Text == "I");
|
||||
Assert.IsTrue(tokens[0].Start == 0);
|
||||
|
||||
Assert.IsTrue(tokens[1].Text == "can");
|
||||
Assert.IsTrue(tokens[1].Start == 2);
|
||||
|
||||
Assert.IsTrue(tokens[2].Text == "not");
|
||||
Assert.IsTrue(tokens[2].Start == 5);
|
||||
|
||||
Assert.IsTrue(tokens[3].Text == "jump");
|
||||
Assert.IsTrue(tokens[3].Start == 9);
|
||||
|
||||
Assert.IsTrue(tokens[4].Text == ".");
|
||||
Assert.IsTrue(tokens[4].Start == 13);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,22 +47,24 @@ namespace BotSharp.NLP.Tokenize
|
|||
|
||||
public List<Token> Tokenize(string sentence, TokenizationOptions options)
|
||||
{
|
||||
string text = sentence;
|
||||
|
||||
// starting quoting replace
|
||||
STARTING_QUOTES.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// replace PUNCTUATION
|
||||
PUNCTUATION.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// Handles parentheses.
|
||||
PARENS_BRACKETS.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// convert parentheses
|
||||
|
|
@ -70,39 +72,39 @@ namespace BotSharp.NLP.Tokenize
|
|||
{
|
||||
CONVERT_PARENTHESES.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||
});
|
||||
}
|
||||
|
||||
// Handles repeated dash.
|
||||
sentence = Regex.Replace(sentence, "(-{2,})", " $1 ");
|
||||
text = Regex.Replace(text, "(-{2,})", " $1 ").Trim();
|
||||
|
||||
// replace ending quotes
|
||||
ENDING_QUOTES.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// replace ending quotes
|
||||
CONVENTIONS.ForEach(x =>
|
||||
{
|
||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
||||
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||
});
|
||||
|
||||
// remove duplicated spaces
|
||||
sentence = Regex.Replace(sentence, "\\s+", " ");
|
||||
text = Regex.Replace(text, "\\s+", " ") + " ";
|
||||
|
||||
// split
|
||||
int pos = 0;
|
||||
|
||||
var results = Regex.Matches(sentence, "\\s")
|
||||
var tokens = Regex.Matches(text, "\\s")
|
||||
.Cast<Match>()
|
||||
.Select(x => {
|
||||
|
||||
var token = new Token
|
||||
{
|
||||
Start = pos,
|
||||
Text = sentence.Substring(pos, x.Index - pos)
|
||||
Text = text.Substring(pos, x.Index - pos)
|
||||
};
|
||||
|
||||
pos = x.Index + 1;
|
||||
|
|
@ -111,7 +113,23 @@ namespace BotSharp.NLP.Tokenize
|
|||
|
||||
}).ToList();
|
||||
|
||||
return results;
|
||||
// correct token position
|
||||
CorrectTokenPosition(sentence, tokens);
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
private void CorrectTokenPosition(string sentence, List<Token> tokens)
|
||||
{
|
||||
int startPos = 0;
|
||||
|
||||
for(int i = 0; i < tokens.Count; i++)
|
||||
{
|
||||
var token = tokens[i];
|
||||
token.Start = sentence.IndexOf(token.Text, startPos);
|
||||
|
||||
startPos = token.End + 1;
|
||||
}
|
||||
}
|
||||
|
||||
private void Init()
|
||||
|
|
@ -121,13 +139,13 @@ namespace BotSharp.NLP.Tokenize
|
|||
STARTING_QUOTES.Add(new Tuple<string, string>(@"(``)", " $1 "));
|
||||
STARTING_QUOTES.Add(new Tuple<string, string>("([ ([{<])(\" | '{2})", "$1 `` "));
|
||||
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\'»”’ ]*)\s*$", "$1 $2 $3 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\'»”’ ]*)\s*$", "$1 $2 $3 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])([^\d])", " $1 $2"));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])$", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"(\.\.\.)", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([;@#$%&])", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\']*)\\s*$", "$1 $2 $3 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"[?!]", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"']*)\s*$", "$1 $2 $3 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([?!])", " $1 "));
|
||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^'])' ", "$1 ' "));
|
||||
|
||||
PARENS_BRACKETS.Add(new Tuple<string, string>(@"([\]\[\(\)\{\}\<\>])", " $1 "));
|
||||
|
|
@ -142,8 +160,8 @@ namespace BotSharp.NLP.Tokenize
|
|||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([»”’])", " $1 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>("\"", " '' "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"(\S)(\'\')", "$1 $2 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([^' ])('[sS]|'[mM]|'[dD]|') ", "$1 $2 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", "$1 $2 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"('[sS]|'[mM]|'[dD]|') ", " $1 "));
|
||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", " $1 "));
|
||||
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(can)(?#X)(not)\b", "$1 $2 "));
|
||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(d)(?#X)('ye)\b", "$1 $2 "));
|
||||
|
|
|
|||
Loading…
Reference in a new issue