SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Official support Simple Chinese tokenize and POS tagger.

Browse source
This commit is contained in:
haiping008@gmail.com 2018-09-20 11:45:25 -05:00
parent 1c22206fe0
commit f016920462
21 changed files with 559568 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs
View file

@ -3,6 +3,7 @@ using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Corpus;
using BotSharp.NLP.Tag;
using BotSharp.NLP.Tokenize;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
@ -17,37 +18,52 @@ namespace BotSharp.Core.Engines.BotSharp
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
private TaggerFactory<NGramTagger> _tagger;
private TaggerFactory _tagger;
public BotSharpTagger()
{
string dataDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Corpus", "CoNLL");
var data = new CoNLLReader().Read(new ReaderOptions
{
DataDir = dataDir,
FileName = "conll2000_chunking_train.txt"
});
_tagger = new TaggerFactory<NGramTagger>(new TagOptions
{
NGram = 1,
Tag = "NN",
Corpus = data
}, SupportedLanguage.English);
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
doc.Sentences.ForEach(x => _tagger.Tag(new Sentence { Words = x.Tokens }));
Init();
doc.Sentences.ForEach(x => _tagger.Tag(new Sentence
{
Words = x.Tokens,
Text = x.Text
}));
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
doc.Sentences.ForEach(x => _tagger.Tag(new Sentence { Words = x.Tokens }));
Init();
doc.Sentences.ForEach(x => _tagger.Tag(new Sentence
{
Words = x.Tokens,
Text = x.Text
}));
return true;
}
private void Init()
{
if (_tagger == null)
{
_tagger = new TaggerFactory(new TagOptions
{
CorpusDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Corpus")
}, SupportedLanguage.English);
string tokenizerName = Configuration.GetValue<String>($"tagger");
_tagger.GetTagger(tokenizerName);
}
}
}
}

46
BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs Normal file
View file

@ -0,0 +1,46 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Tag;
using JiebaNet.Segmenter.PosSeg;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.Jieba.NET
{
public class JiebaTagger : ITagger
{
private PosSegmenter posSeg;
public void Tag(Sentence sentence, TagOptions options)
{
Init();
var tokens = posSeg.Cut(sentence.Text).ToList();
for(int i = 0; i < sentence.Words.Count; i++)
{
sentence.Words[i].Pos = tokens[i].Flag;
sentence.Words[i].Tag = tokens[i].Flag;
}
}
public void Train(List<Sentence> sentences, TagOptions options)
{
}
private void Init()
{
if (posSeg == null)
{
posSeg = new PosSegmenter();
}
}
}
}

32
BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs
View file

@ -1,26 +1,44 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP.Tokenize;
using JiebaNet.Segmenter;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Token = BotSharp.NLP.Tokenize.Token;
namespace BotSharp.Core.Engines.Jieba.NET
{
public class JiebaTokenizer : INlpTrain, INlpPredict
public class JiebaTokenizer : TokenizerBase, ITokenizer
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
private JiebaSegmenter segmenter;
public Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
public List<Token> Tokenize(string sentence, TokenizationOptions options)
{
throw new NotImplementedException();
Init();
var tokens = segmenter.Cut(sentence)
.Select(x => new Token
{
Text = x
}).ToList();
CorrectTokenPosition(sentence, tokens);
return tokens;
}
public Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
private void Init()
{
throw new NotImplementedException();
if(segmenter == null)
{
segmenter = new JiebaSegmenter();
segmenter.LoadUserDict($"App_Data{Path.DirectorySeparatorChar}userdict.txt");
}
}
}
}

4
BotSharp.NLP.UnitTest/DefaultTaggerTest.cs
View file

@ -19,11 +19,13 @@ namespace BotSharp.NLP.UnitTest
var tokens = tokenizer.Tokenize("How are you doing?");
var tagger = new TaggerFactory<DefaultTagger>(new TagOptions
var tagger = new TaggerFactory(new TagOptions
{
Tag = "NN"
}, SupportedLanguage.English);
tagger.GetTagger<DefaultTagger>();
tagger.Tag(new Sentence { Words = tokens });
}
}

37
BotSharp.NLP.UnitTest/NGramTaggerTest.cs
View file

@ -27,14 +27,15 @@ namespace BotSharp.NLP.UnitTest
var tokens = tokenizer.Tokenize("Chancellor of the Exchequer Nigel Lawson's restated commitment");
// test tag
var data = GetTaggedCorpus();
var tagger = new TaggerFactory<NGramTagger>(new TagOptions
var tagger = new TaggerFactory(new TagOptions
{
CorpusDir = Configuration.GetValue<String>("BotSharp.NLP:dataDir"),
NGram = 1,
Tag = "NN",
Corpus = data
Tag = "NN"
}, SupportedLanguage.English);
tagger.GetTagger<NGramTagger>();
var watch = Stopwatch.StartNew();
tagger.Tag(new Sentence { Words = tokens });
watch.Stop();
@ -67,13 +68,15 @@ namespace BotSharp.NLP.UnitTest
var tokens = tokenizer.Tokenize("Chancellor of the Exchequer Nigel Lawson's restated commitment");
// test tag
var tagger = new TaggerFactory<NGramTagger>(new TagOptions
var tagger = new TaggerFactory(new TagOptions
{
CorpusDir = Configuration.GetValue<String>("BotSharp.NLP:dataDir"),
NGram = 2,
Tag = "NN",
Corpus = GetTaggedCorpus()
Tag = "NN"
}, SupportedLanguage.English);
tagger.GetTagger<NGramTagger>();
tagger.Tag(new Sentence { Words = tokens });
Assert.IsTrue(tokens[0].Pos == "NNP");
@ -95,13 +98,15 @@ namespace BotSharp.NLP.UnitTest
var tokens = tokenizer.Tokenize("Chancellor of the Exchequer Nigel Lawson's restated commitment");
// test tag
var tagger = new TaggerFactory<NGramTagger>(new TagOptions
var tagger = new TaggerFactory(new TagOptions
{
CorpusDir = Configuration.GetValue<String>("BotSharp.NLP:dataDir"),
NGram = 3,
Tag = "NN",
Corpus = GetTaggedCorpus()
Tag = "NN"
}, SupportedLanguage.English);
tagger.GetTagger<NGramTagger>();
tagger.Tag(new Sentence { Words = tokens });
Assert.IsTrue(tokens[0].Pos == "NNP");
@ -109,17 +114,5 @@ namespace BotSharp.NLP.UnitTest
Assert.IsTrue(tokens[2].Pos == "DT");
Assert.IsTrue(tokens[3].Pos == "NNP");
}
private List<Sentence> GetTaggedCorpus()
{
// get training corpus
string dataDir = Path.Combine(Configuration.GetValue<String>("BotSharp.NLP:dataDir"), "CoNLL");
return new CoNLLReader()
.Read(new ReaderOptions
{
DataDir = dataDir,
FileName = "conll2000_chunking_train.txt"
});
}
}
}

14
BotSharp.NLP/Tag/NGramTagger.cs
View file

@ -18,8 +18,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using BotSharp.NLP.Corpus;
using BotSharp.NLP.Tokenize;
namespace BotSharp.NLP.Tag
@ -37,7 +39,13 @@ namespace BotSharp.NLP.Tag
// need training to generate model
if(_contextMapping == null)
{
Train(options.Corpus, options);
var corpus = new CoNLLReader().Read(new ReaderOptions
{
DataDir = Path.Combine(options.CorpusDir, "CoNLL"),
FileName = "conll2000_chunking_train.txt"
});
Train(corpus, options);
}
Fill(sentence, options);
@ -63,9 +71,9 @@ namespace BotSharp.NLP.Tag
{
var cache = new List<NGramFreq>();
for (int idx = 0; idx < options.Corpus.Count; idx++)
for (int idx = 0; idx < sentences.Count; idx++)
{
var sent = options.Corpus[idx];
var sent = sentences[idx];
Fill(sent, options);

11
BotSharp.NLP/Tag/TagOptions.cs
View file

@ -22,9 +22,12 @@ namespace BotSharp.NLP.Tag
/// </summary>
public int NGram { get; set; }
/// <summary>
/// Tagged corpus used for training a model
/// </summary>
public List<Sentence> Corpus { get; set; }
public string CorpusDir { get; set; }
public TagOptions()
{
NGram = 1;
Tag = "NN";
}
}
}

28
BotSharp.NLP/Tag/TaggerFactory.cs
View file

@ -1,14 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
namespace BotSharp.NLP.Tag
{
public class TaggerFactory<ITag> where ITag : ITagger, new()
public class TaggerFactory
{
private SupportedLanguage _lang;
private ITag _tagger;
private ITagger _tagger;
private TagOptions _options;
@ -16,7 +18,27 @@ namespace BotSharp.NLP.Tag
{
_lang = lang;
_options = options;
_tagger = new ITag();
}
public ITagger GetTagger<ITag>() where ITag : ITagger, new()
{
return _tagger = new ITag();
}
public ITagger GetTagger(string name)
{
List<Type> types = new List<Type>();
types.AddRange(Assembly.Load(new AssemblyName("BotSharp.Core"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
types.AddRange(Assembly.Load(new AssemblyName("BotSharp.NLP"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
Type type = types.FirstOrDefault(x => x.Name == name);
var instance = (ITagger)Activator.CreateInstance(type);
return _tagger = instance;
}
public void Tag(Sentence sentence)

22
BotSharp.NLP/Tokenize/TokenizerBase.cs Normal file
View file

@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.NLP.Tokenize
{
public abstract class TokenizerBase
{
protected void CorrectTokenPosition(string sentence, List<Token> tokens)
{
int startPos = 0;
for (int i = 0; i < tokens.Count; i++)
{
var token = tokens[i];
token.Start = sentence.IndexOf(token.Text, startPos);
startPos = token.End;
}
}
}
}

10
BotSharp.NLP/Tokenize/TokenizerFactory.cs
View file

@ -28,8 +28,13 @@ namespace BotSharp.NLP.Tokenize
public ITokenizer GetTokenizer(string name)
{
List<Type> types = Assembly.Load(new AssemblyName("BotSharp.NLP"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList();
List<Type> types = new List<Type>();
types.AddRange(Assembly.Load(new AssemblyName("BotSharp.Core"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
types.AddRange(Assembly.Load(new AssemblyName("BotSharp.NLP"))
.GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList());
Type type = types.FirstOrDefault(x => x.Name == name);
var instance = (ITokenizer)Activator.CreateInstance(type);
@ -41,7 +46,6 @@ namespace BotSharp.NLP.Tokenize
{
_lang = lang;
_options = options;
}
public List<Token> Tokenize(string sentence)

15
BotSharp.NLP/Tokenize/TreebankTokenizer.cs
View file

@ -31,7 +31,7 @@ namespace BotSharp.NLP.Tokenize
/// and available at ftp://ftp.cis.upenn.edu/pub/treebank/public_html/tokenizer.sed,
/// or reference ftp://ftp.cis.upenn.edu/pub/treebank/public_html/tokenization.html.
/// </summary>
public class TreebankTokenizer : ITokenizer
public class TreebankTokenizer : TokenizerBase, ITokenizer
{
private List<Tuple<String, String>> STARTING_QUOTES = new List<Tuple<string, string>>();
private List<Tuple<String, String>> PUNCTUATION = new List<Tuple<string, string>>();
@ -119,19 +119,6 @@ namespace BotSharp.NLP.Tokenize
return tokens;
}
private void CorrectTokenPosition(string sentence, List<Token> tokens)
{
int startPos = 0;
for(int i = 0; i < tokens.Count; i++)
{
var token = tokens[i];
token.Start = sentence.IndexOf(token.Text, startPos);
startPos = token.End;
}
}
private void Init()
{
STARTING_QUOTES.Add(new Tuple<string, string>(@"([«“‘„]|[`]+)", " $1 "));

79460
BotSharp.WebHost/App_Data/Resources/char_state_tab.json Normal file
View file

File diff suppressed because it is too large Load diff

349046
BotSharp.WebHost/App_Data/Resources/dict.txt Normal file
View file

File diff suppressed because it is too large Load diff

89711
BotSharp.WebHost/App_Data/Resources/pos_prob_emit.json Normal file
View file

File diff suppressed because it is too large Load diff

258
BotSharp.WebHost/App_Data/Resources/pos_prob_start.json Normal file
View file

@ -0,0 +1,258 @@
{
"E-e": -3.14e+100,
"E-d": -3.14e+100,
"E-g": -3.14e+100,
"E-f": -3.14e+100,
"E-a": -3.14e+100,
"E-c": -3.14e+100,
"E-b": -3.14e+100,
"E-m": -3.14e+100,
"S-rg": -10.275268591948773,
"E-o": -3.14e+100,
"E-n": -3.14e+100,
"E-i": -3.14e+100,
"E-h": -3.14e+100,
"E-k": -3.14e+100,
"E-j": -3.14e+100,
"E-u": -3.14e+100,
"E-t": -3.14e+100,
"E-w": -3.14e+100,
"E-v": -3.14e+100,
"E-q": -3.14e+100,
"E-p": -3.14e+100,
"E-s": -3.14e+100,
"M-bg": -3.14e+100,
"M-uj": -3.14e+100,
"E-y": -3.14e+100,
"E-x": -3.14e+100,
"E-z": -3.14e+100,
"B-uz": -3.14e+100,
"S-d": -3.903919764181873,
"M-rg": -3.14e+100,
"E-nt": -3.14e+100,
"B-d": -3.9750475297585357,
"B-uv": -3.14e+100,
"E-vi": -3.14e+100,
"B-mq": -6.78695300139688,
"M-rr": -3.14e+100,
"S-ag": -6.954113917960154,
"M-jn": -3.14e+100,
"E-l": -3.14e+100,
"M-rz": -3.14e+100,
"B-ud": -3.14e+100,
"S-an": -12.84021794941031,
"B-qg": -3.14e+100,
"B-ug": -3.14e+100,
"M-y": -3.14e+100,
"S-qg": -3.14e+100,
"S-z": -3.14e+100,
"S-y": -6.1970794699489575,
"S-x": -8.427419656069674,
"S-w": -3.14e+100,
"S-v": -3.053292303412302,
"S-u": -6.940320595827818,
"S-t": -3.14e+100,
"B-nrt": -4.985642733519195,
"S-r": -2.7635336784127853,
"S-q": -4.888658618255058,
"M-zg": -3.14e+100,
"S-o": -8.464460927750023,
"S-n": -3.8551483897645107,
"B-zg": -3.14e+100,
"S-l": -3.14e+100,
"S-k": -6.940320595827818,
"S-in": -3.14e+100,
"S-i": -3.14e+100,
"S-h": -8.650563207383884,
"S-g": -6.507826815331734,
"B-f": -5.491630418482717,
"S-e": -5.942513006281674,
"M-en": -3.14e+100,
"S-c": -4.786966795861212,
"S-b": -6.472888763970454,
"S-a": -3.9025396831295227,
"B-g": -3.14e+100,
"B-b": -5.018374362109218,
"B-c": -3.423880184954888,
"M-ug": -3.14e+100,
"B-a": -4.762305214596967,
"E-qe": -3.14e+100,
"M-x": -3.14e+100,
"E-nz": -3.14e+100,
"M-z": -3.14e+100,
"M-u": -3.14e+100,
"B-k": -3.14e+100,
"M-w": -3.14e+100,
"B-jn": -3.14e+100,
"S-yg": -13.533365129970255,
"B-o": -8.433498702146057,
"B-l": -4.905883584659895,
"B-m": -3.6524299819046386,
"M-m": -3.14e+100,
"M-l": -3.14e+100,
"M-o": -3.14e+100,
"M-n": -3.14e+100,
"M-i": -3.14e+100,
"M-h": -3.14e+100,
"B-t": -3.3647479094528574,
"M-ul": -3.14e+100,
"B-z": -7.045681111485645,
"M-d": -3.14e+100,
"M-mg": -3.14e+100,
"B-y": -9.844485675856319,
"M-a": -3.14e+100,
"S-nrt": -3.14e+100,
"M-c": -3.14e+100,
"M-uz": -3.14e+100,
"E-mg": -3.14e+100,
"B-i": -6.1157847275557105,
"M-b": -3.14e+100,
"E-uz": -3.14e+100,
"B-n": -1.6966257797548328,
"E-uv": -3.14e+100,
"M-ud": -3.14e+100,
"M-p": -3.14e+100,
"E-ul": -3.14e+100,
"E-mq": -3.14e+100,
"M-s": -3.14e+100,
"M-yg": -3.14e+100,
"E-uj": -3.14e+100,
"E-ud": -3.14e+100,
"S-ln": -3.14e+100,
"M-r": -3.14e+100,
"E-ng": -3.14e+100,
"B-r": -3.4098187790818413,
"E-en": -3.14e+100,
"M-qg": -3.14e+100,
"B-s": -5.522673590839954,
"S-rr": -3.14e+100,
"B-p": -4.200984132085048,
"B-dg": -3.14e+100,
"M-uv": -3.14e+100,
"S-zg": -3.14e+100,
"B-v": -2.6740584874265685,
"S-tg": -6.272842531880403,
"B-w": -3.14e+100,
"B-e": -8.563551830394255,
"M-k": -3.14e+100,
"M-j": -3.14e+100,
"B-df": -8.888974230828882,
"M-e": -3.14e+100,
"E-tg": -3.14e+100,
"M-t": -3.14e+100,
"E-nr": -3.14e+100,
"M-nrfg": -3.14e+100,
"B-nr": -2.2310495913769506,
"E-df": -3.14e+100,
"E-dg": -3.14e+100,
"S-jn": -3.14e+100,
"M-q": -3.14e+100,
"B-mg": -3.14e+100,
"B-ln": -3.14e+100,
"M-f": -3.14e+100,
"E-ln": -3.14e+100,
"E-yg": -3.14e+100,
"S-bg": -3.14e+100,
"E-ns": -3.14e+100,
"B-tg": -3.14e+100,
"E-qg": -3.14e+100,
"S-nr": -4.483663103956885,
"S-ns": -3.14e+100,
"M-vn": -3.14e+100,
"S-nt": -12.147070768850364,
"S-nz": -3.14e+100,
"S-ad": -11.048458480182255,
"B-yg": -3.14e+100,
"M-v": -3.14e+100,
"E-vn": -3.14e+100,
"S-ng": -4.913434861102905,
"M-g": -3.14e+100,
"M-nt": -3.14e+100,
"S-en": -3.14e+100,
"M-nr": -3.14e+100,
"M-ns": -3.14e+100,
"S-vq": -3.14e+100,
"B-uj": -3.14e+100,
"M-nz": -3.14e+100,
"B-qe": -3.14e+100,
"M-in": -3.14e+100,
"M-ng": -3.14e+100,
"S-vn": -11.453923588290419,
"E-zg": -3.14e+100,
"S-vi": -3.14e+100,
"S-vg": -5.9430181843676895,
"S-vd": -3.14e+100,
"B-ad": -6.680066036784177,
"E-rz": -3.14e+100,
"B-ag": -3.14e+100,
"B-vd": -9.044728760238115,
"S-mq": -3.14e+100,
"B-vi": -12.434752841302146,
"E-rr": -3.14e+100,
"B-rr": -12.434752841302146,
"M-vq": -3.14e+100,
"E-jn": -3.14e+100,
"B-vn": -4.3315610890163585,
"S-mg": -10.825314928868044,
"B-in": -3.14e+100,
"M-vi": -3.14e+100,
"M-an": -3.14e+100,
"M-vd": -3.14e+100,
"B-rg": -3.14e+100,
"M-vg": -3.14e+100,
"M-ad": -3.14e+100,
"M-ag": -3.14e+100,
"E-rg": -3.14e+100,
"S-uz": -9.299258625372996,
"B-en": -3.14e+100,
"S-uv": -8.15808672228609,
"S-df": -3.14e+100,
"S-dg": -8.948397651299683,
"M-qe": -3.14e+100,
"B-ng": -3.14e+100,
"E-bg": -3.14e+100,
"S-ul": -8.4153713175535,
"S-uj": -6.85251045118004,
"S-ug": -7.5394037026636855,
"B-ns": -2.8228438314969213,
"S-ud": -7.728230161053767,
"B-nt": -4.846091668182416,
"B-ul": -3.14e+100,
"E-in": -3.14e+100,
"B-bg": -3.14e+100,
"M-df": -3.14e+100,
"M-dg": -3.14e+100,
"M-nrt": -3.14e+100,
"B-j": -5.0576191284681915,
"E-ug": -3.14e+100,
"E-vq": -3.14e+100,
"B-vg": -3.14e+100,
"B-nz": -3.94698846057672,
"S-qe": -3.14e+100,
"B-rz": -7.946116471570005,
"B-nrfg": -5.873722175405573,
"E-ad": -3.14e+100,
"E-ag": -3.14e+100,
"B-u": -9.163917277503234,
"M-ln": -3.14e+100,
"B-an": -8.697083223018778,
"M-mq": -3.14e+100,
"E-an": -3.14e+100,
"S-s": -3.14e+100,
"B-q": -6.998123858956596,
"E-nrt": -3.14e+100,
"B-h": -13.533365129970255,
"E-r": -3.14e+100,
"S-p": -2.9868401813596317,
"M-tg": -3.14e+100,
"S-rz": -3.14e+100,
"S-nrfg": -3.14e+100,
"B-vq": -12.147070768850364,
"B-x": -3.14e+100,
"E-vd": -3.14e+100,
"E-nrfg": -3.14e+100,
"S-m": -3.269200652116097,
"E-vg": -3.14e+100,
"S-f": -5.194820249981676,
"S-j": -4.911992119644354
}

5639
BotSharp.WebHost/App_Data/Resources/pos_prob_trans.json Normal file
View file

File diff suppressed because it is too large Load diff

35234
BotSharp.WebHost/App_Data/Resources/prob_emit.json Normal file
View file

File diff suppressed because it is too large Load diff

18
BotSharp.WebHost/App_Data/Resources/prob_trans.json Normal file
View file

@ -0,0 +1,18 @@
{
"M": {
"M": -1.2603623820268226,
"E": -0.33344856811948514
},
"S": {
"S": -0.6658631448798212,
"B": -0.7211965654669841
},
"B": {
"M": -0.916290731874155,
"E": -0.51082562376599
},
"E": {
"S": -0.8085250474669937,
"B": -0.5897149736854513
}
}

5
BotSharp.WebHost/App_Data/userdict.txt Normal file
View file

@ -0,0 +1,5 @@
创新办 3 i
云计算 5
凱特琳 nz
机器学习 3
深度学习 8

4
BotSharp.sln
View file

@ -19,9 +19,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NL
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{4AAA7A40-0389-41AC-B55E-CFAF7C8600B1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Voice", "BotSharp.Voice\BotSharp.Voice.csproj", "{184F8B93-68C2-4849-9625-4023C4360990}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Voice", "BotSharp.Voice\BotSharp.Voice.csproj", "{184F8B93-68C2-4849-9625-4023C4360990}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Vision", "BotSharp.Vision\BotSharp.Vision.csproj", "{75B02A7C-EDB3-4082-9C1F-471773E760C3}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Vision", "BotSharp.Vision\BotSharp.Vision.csproj", "{75B02A7C-EDB3-4082-9C1F-471773E760C3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

4
Settings/bot.json
View file

@ -12,7 +12,7 @@
},
"BotSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
"tokenizer": "JiebaTokenizer"
},
"BotSharpNBayesClassifier": {
@ -23,7 +23,7 @@
},
"BotSharpTagger": {
"tagger": "JiebaTagger"
},
"BotSharpCRFNer": {