Add CorrectTokenPosition to modify token start position.
This commit is contained in:
Oceania2018
parent
3e55192606
commit
f61859e5a4
|
|
@ -16,7 +16,35 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
||||||
{
|
{
|
||||||
}, SupportedLanguage.English);
|
}, SupportedLanguage.English);
|
||||||
|
|
||||||
var tokens = tokenizer.Tokenize("(\"«Hello World.");
|
var tokens = tokenizer.Tokenize("«Hello!");
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[0].Text == "«");
|
||||||
|
Assert.IsTrue(tokens[0].Start == 0);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[1].Text == "Hello");
|
||||||
|
Assert.IsTrue(tokens[1].Start == 1);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[2].Text == "!");
|
||||||
|
Assert.IsTrue(tokens[2].Start == 6);
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestMethod]
|
||||||
|
public void ReplaceEndingQuoting()
|
||||||
|
{
|
||||||
|
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
|
||||||
|
{
|
||||||
|
}, SupportedLanguage.English);
|
||||||
|
|
||||||
|
var tokens = tokenizer.Tokenize("Aren't you");
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[0].Text == "Are");
|
||||||
|
Assert.IsTrue(tokens[0].Start == 0);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[1].Text == "n't");
|
||||||
|
Assert.IsTrue(tokens[1].Start == 3);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[2].Text == "you");
|
||||||
|
Assert.IsTrue(tokens[2].Start == 7);
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestMethod]
|
[TestMethod]
|
||||||
|
|
@ -27,6 +55,15 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
||||||
}, SupportedLanguage.English);
|
}, SupportedLanguage.English);
|
||||||
|
|
||||||
var tokens = tokenizer.Tokenize("Hello World...");
|
var tokens = tokenizer.Tokenize("Hello World...");
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[0].Text == "Hello");
|
||||||
|
Assert.IsTrue(tokens[0].Start == 0);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[1].Text == "World");
|
||||||
|
Assert.IsTrue(tokens[1].Start == 6);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[2].Text == "...");
|
||||||
|
Assert.IsTrue(tokens[2].Start == 11);
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestMethod]
|
[TestMethod]
|
||||||
|
|
@ -36,7 +73,19 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
||||||
{
|
{
|
||||||
}, SupportedLanguage.English);
|
}, SupportedLanguage.English);
|
||||||
|
|
||||||
var tokens = tokenizer.Tokenize("<Hello World.>");
|
var tokens = tokenizer.Tokenize("<Hello.>");
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[0].Text == "<");
|
||||||
|
Assert.IsTrue(tokens[0].Start == 0);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[1].Text == "Hello");
|
||||||
|
Assert.IsTrue(tokens[1].Start == 1);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[2].Text == ".");
|
||||||
|
Assert.IsTrue(tokens[2].Start == 6);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[3].Text == ">");
|
||||||
|
Assert.IsTrue(tokens[3].Start == 7);
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestMethod]
|
[TestMethod]
|
||||||
|
|
@ -47,6 +96,21 @@ namespace BotSharp.NLP.UnitTest.Tokenize
|
||||||
}, SupportedLanguage.English);
|
}, SupportedLanguage.English);
|
||||||
|
|
||||||
var tokens = tokenizer.Tokenize("I cannot jump.");
|
var tokens = tokenizer.Tokenize("I cannot jump.");
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[0].Text == "I");
|
||||||
|
Assert.IsTrue(tokens[0].Start == 0);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[1].Text == "can");
|
||||||
|
Assert.IsTrue(tokens[1].Start == 2);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[2].Text == "not");
|
||||||
|
Assert.IsTrue(tokens[2].Start == 5);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[3].Text == "jump");
|
||||||
|
Assert.IsTrue(tokens[3].Start == 9);
|
||||||
|
|
||||||
|
Assert.IsTrue(tokens[4].Text == ".");
|
||||||
|
Assert.IsTrue(tokens[4].Start == 13);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -47,22 +47,24 @@ namespace BotSharp.NLP.Tokenize
|
||||||
|
|
||||||
public List<Token> Tokenize(string sentence, TokenizationOptions options)
|
public List<Token> Tokenize(string sentence, TokenizationOptions options)
|
||||||
{
|
{
|
||||||
|
string text = sentence;
|
||||||
|
|
||||||
// starting quoting replace
|
// starting quoting replace
|
||||||
STARTING_QUOTES.ForEach(x =>
|
STARTING_QUOTES.ForEach(x =>
|
||||||
{
|
{
|
||||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||||
});
|
});
|
||||||
|
|
||||||
// replace PUNCTUATION
|
// replace PUNCTUATION
|
||||||
PUNCTUATION.ForEach(x =>
|
PUNCTUATION.ForEach(x =>
|
||||||
{
|
{
|
||||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Handles parentheses.
|
// Handles parentheses.
|
||||||
PARENS_BRACKETS.ForEach(x =>
|
PARENS_BRACKETS.ForEach(x =>
|
||||||
{
|
{
|
||||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||||
});
|
});
|
||||||
|
|
||||||
// convert parentheses
|
// convert parentheses
|
||||||
|
|
@ -70,39 +72,39 @@ namespace BotSharp.NLP.Tokenize
|
||||||
{
|
{
|
||||||
CONVERT_PARENTHESES.ForEach(x =>
|
CONVERT_PARENTHESES.ForEach(x =>
|
||||||
{
|
{
|
||||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handles repeated dash.
|
// Handles repeated dash.
|
||||||
sentence = Regex.Replace(sentence, "(-{2,})", " $1 ");
|
text = Regex.Replace(text, "(-{2,})", " $1 ").Trim();
|
||||||
|
|
||||||
// replace ending quotes
|
// replace ending quotes
|
||||||
ENDING_QUOTES.ForEach(x =>
|
ENDING_QUOTES.ForEach(x =>
|
||||||
{
|
{
|
||||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||||
});
|
});
|
||||||
|
|
||||||
// replace ending quotes
|
// replace ending quotes
|
||||||
CONVENTIONS.ForEach(x =>
|
CONVENTIONS.ForEach(x =>
|
||||||
{
|
{
|
||||||
sentence = Regex.Replace(sentence, x.Item1, x.Item2);
|
text = Regex.Replace(text, x.Item1, x.Item2);
|
||||||
});
|
});
|
||||||
|
|
||||||
// remove duplicated spaces
|
// remove duplicated spaces
|
||||||
sentence = Regex.Replace(sentence, "\\s+", " ");
|
text = Regex.Replace(text, "\\s+", " ") + " ";
|
||||||
|
|
||||||
// split
|
// split
|
||||||
int pos = 0;
|
int pos = 0;
|
||||||
|
|
||||||
var results = Regex.Matches(sentence, "\\s")
|
var tokens = Regex.Matches(text, "\\s")
|
||||||
.Cast<Match>()
|
.Cast<Match>()
|
||||||
.Select(x => {
|
.Select(x => {
|
||||||
|
|
||||||
var token = new Token
|
var token = new Token
|
||||||
{
|
{
|
||||||
Start = pos,
|
Start = pos,
|
||||||
Text = sentence.Substring(pos, x.Index - pos)
|
Text = text.Substring(pos, x.Index - pos)
|
||||||
};
|
};
|
||||||
|
|
||||||
pos = x.Index + 1;
|
pos = x.Index + 1;
|
||||||
|
|
@ -111,7 +113,23 @@ namespace BotSharp.NLP.Tokenize
|
||||||
|
|
||||||
}).ToList();
|
}).ToList();
|
||||||
|
|
||||||
return results;
|
// correct token position
|
||||||
|
CorrectTokenPosition(sentence, tokens);
|
||||||
|
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CorrectTokenPosition(string sentence, List<Token> tokens)
|
||||||
|
{
|
||||||
|
int startPos = 0;
|
||||||
|
|
||||||
|
for(int i = 0; i < tokens.Count; i++)
|
||||||
|
{
|
||||||
|
var token = tokens[i];
|
||||||
|
token.Start = sentence.IndexOf(token.Text, startPos);
|
||||||
|
|
||||||
|
startPos = token.End + 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void Init()
|
private void Init()
|
||||||
|
|
@ -121,13 +139,13 @@ namespace BotSharp.NLP.Tokenize
|
||||||
STARTING_QUOTES.Add(new Tuple<string, string>(@"(``)", " $1 "));
|
STARTING_QUOTES.Add(new Tuple<string, string>(@"(``)", " $1 "));
|
||||||
STARTING_QUOTES.Add(new Tuple<string, string>("([ ([{<])(\" | '{2})", "$1 `` "));
|
STARTING_QUOTES.Add(new Tuple<string, string>("([ ([{<])(\" | '{2})", "$1 `` "));
|
||||||
|
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\'»”’ ]*)\s*$", "$1 $2 $3 "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\'»”’ ]*)\s*$", "$1 $2 $3 "));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])([^\d])", " $1 $2"));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])([^\d])", " $1 $2"));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])$", " $1 "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([:,])$", " $1 "));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"(\.\.\.)", " $1 "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"(\.\.\.)", " $1 "));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"([;@#$%&])", " $1 "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([;@#$%&])", " $1 "));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"\\\']*)\\s*$", "$1 $2 $3 "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([^\.])(\.)([\]\)}>" + "\"" + @"']*)\s*$", "$1 $2 $3 "));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"[?!]", " $1 "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([?!])", " $1 "));
|
||||||
PUNCTUATION.Add(new Tuple<string, string>(@"([^'])' ", "$1 ' "));
|
PUNCTUATION.Add(new Tuple<string, string>(@"([^'])' ", "$1 ' "));
|
||||||
|
|
||||||
PARENS_BRACKETS.Add(new Tuple<string, string>(@"([\]\[\(\)\{\}\<\>])", " $1 "));
|
PARENS_BRACKETS.Add(new Tuple<string, string>(@"([\]\[\(\)\{\}\<\>])", " $1 "));
|
||||||
|
|
@ -142,8 +160,8 @@ namespace BotSharp.NLP.Tokenize
|
||||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([»”’])", " $1 "));
|
ENDING_QUOTES.Add(new Tuple<string, string>(@"([»”’])", " $1 "));
|
||||||
ENDING_QUOTES.Add(new Tuple<string, string>("\"", " '' "));
|
ENDING_QUOTES.Add(new Tuple<string, string>("\"", " '' "));
|
||||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"(\S)(\'\')", "$1 $2 "));
|
ENDING_QUOTES.Add(new Tuple<string, string>(@"(\S)(\'\')", "$1 $2 "));
|
||||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([^' ])('[sS]|'[mM]|'[dD]|') ", "$1 $2 "));
|
ENDING_QUOTES.Add(new Tuple<string, string>(@"('[sS]|'[mM]|'[dD]|') ", " $1 "));
|
||||||
ENDING_QUOTES.Add(new Tuple<string, string>(@"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", "$1 $2 "));
|
ENDING_QUOTES.Add(new Tuple<string, string>(@"('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", " $1 "));
|
||||||
|
|
||||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(can)(?#X)(not)\b", "$1 $2 "));
|
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(can)(?#X)(not)\b", "$1 $2 "));
|
||||||
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(d)(?#X)('ye)\b", "$1 $2 "));
|
CONVENTIONS.Add(new Tuple<string, string>(@"(?i)\b(d)(?#X)('ye)\b", "$1 $2 "));
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue