SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fix tokenizer compile issue in unit test.

Browse source 
Add Jieba tokenizer for Chinese chabot.
This commit is contained in:
haiping008@gmail.com 2018-09-19 23:13:12 -05:00
parent 67c2cf7e5c
commit 1c22206fe0
9 changed files with 66 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
BotSharp.Core/BotSharp.Core.csproj
View file

@ -78,6 +78,7 @@ If you feel that this project is helpful to you, please Star on the project, we
<PackageReference Include="Colorful.Console" Version="1.2.9" />
<PackageReference Include="DotNetToolkit" Version="1.6.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.9.1" />
<PackageReference Include="JiebaNet.Segmenter" Version="1.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="2.1.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="RestSharp" Version="106.3.1" />

26
BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs Normal file
View file

@ -0,0 +1,26 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.Jieba.NET
{
public class JiebaTokenizer : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
throw new NotImplementedException();
}
public Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
throw new NotImplementedException();
}
}
}

3
BotSharp.Core/Engines/Jieba.NET/README.rst Normal file
View file

@ -0,0 +1,3 @@
BotSharp uses the jieba.NetCore to do tokenization. (https://github.com/1483523635/jieba.NetCore)
Please follow the install instruction (https://github.com/anderscui/jieba.NET/)

4
BotSharp.NLP.UnitTest/DefaultTaggerTest.cs
View file

@ -14,7 +14,9 @@ namespace BotSharp.NLP.UnitTest
[TestMethod]
public void TagInCoNLL2000()
{
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize("How are you doing?");
var tagger = new TaggerFactory<DefaultTagger>(new TagOptions

9
BotSharp.NLP.UnitTest/NGramTaggerTest.cs
View file

@ -18,10 +18,11 @@ namespace BotSharp.NLP.UnitTest
public void UniGramInCoNLL2000()
{
// tokenization
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize("Chancellor of the Exchequer Nigel Lawson's restated commitment");
@ -57,10 +58,11 @@ namespace BotSharp.NLP.UnitTest
public void BiGramInCoNLL2000()
{
// tokenization
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize("Chancellor of the Exchequer Nigel Lawson's restated commitment");
@ -84,10 +86,11 @@ namespace BotSharp.NLP.UnitTest
public void TriGramInCoNLL2000()
{
// tokenization
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize("Chancellor of the Exchequer Nigel Lawson's restated commitment");

11
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -26,7 +26,9 @@ namespace BotSharp.NLP.UnitTest
FileName = "cooking.stackexchange.txt"
});
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
for(int i = 0; i < newSentences.Count; i++)
{
@ -75,10 +77,11 @@ namespace BotSharp.NLP.UnitTest
var corpus = GetLabeledCorpus(options);
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
corpus.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
@ -139,7 +142,9 @@ namespace BotSharp.NLP.UnitTest
FileName = "spotify.txt"
});
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
for (int i = 0; i < newSentences.Count; i++)
{

9
BotSharp.NLP.UnitTest/Tokenize/RegexTokenizerTest.cs
View file

@ -10,10 +10,11 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void TokenizeInWhiteSpace()
{
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.WHITE_SPACE
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?");
@ -36,11 +37,12 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void TokenizeInWordPunctuation()
{
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC,
SpecialWords = new List<string> { "n't" }
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?");
@ -72,10 +74,11 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void TokenizeInBlankLine()
{
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
Pattern = RegexTokenizer.BLANK_LINE
}, SupportedLanguage.English);
tokenizer.GetTokenizer<RegexTokenizer>();
var tokens = tokenizer.Tokenize(@"Chop into pieces,

15
BotSharp.NLP.UnitTest/Tokenize/TreebankTokenizerTest.cs
View file

@ -12,9 +12,10 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void ReplaceStartingQuoting()
{
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
}, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var tokens = tokenizer.Tokenize("«Hello!");
@ -31,9 +32,10 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void ReplaceEndingQuoting()
{
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
}, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var tokens = tokenizer.Tokenize("Aren't you");
@ -50,9 +52,10 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void ReplacePunctuation()
{
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
}, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var tokens = tokenizer.Tokenize("Hello World...");
@ -69,9 +72,10 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void ReplaceBrackets()
{
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
}, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var tokens = tokenizer.Tokenize("<Hello.>");
@ -91,9 +95,10 @@ namespace BotSharp.NLP.UnitTest.Tokenize
[TestMethod]
public void ReplaceConventions()
{
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
var tokenizer = new TokenizerFactory(new TokenizationOptions
{
}, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var tokens = tokenizer.Tokenize("I cannot jump.");

4
BotSharp.NLP.UnitTest/Vector/OneHotEncodingTest.cs
View file

@ -24,7 +24,9 @@ namespace BotSharp.NLP.UnitTest.Vector
FileName = "cooking.stackexchange.txt"
});
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
tokenizer.GetTokenizer<TreebankTokenizer>();
var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
for (int i = 0; i < newSentences.Count; i++)
{