From 67c2cf7e5cfe189acde757a08e58618b61750fd5 Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Wed, 19 Sep 2018 19:19:04 -0500 Subject: [PATCH] Make pipline part options more configurable. --- BotSharp.Core/Engines/BotPredictor.cs | 5 ++-- .../Engines/BotSharp/BotSharpCRFNer.cs | 2 +- .../Engines/BotSharp/BotSharpTokenizer.cs | 25 ++++++++++++++++--- BotSharp.Core/Engines/BotTrainer.cs | 3 ++- BotSharp.NLP/Tokenize/TokenizerFactory.cs | 23 ++++++++++++++--- Settings/bot.json | 5 ++++ 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/BotSharp.Core/Engines/BotPredictor.cs b/BotSharp.Core/Engines/BotPredictor.cs index 38484dcb..3edbf28a 100644 --- a/BotSharp.Core/Engines/BotPredictor.cs +++ b/BotSharp.Core/Engines/BotPredictor.cs @@ -52,8 +52,7 @@ namespace BotSharp.Core.Engines var settings = new PipeSettings { ModelDir = dir, - ProjectDir = request.AgentDir, - AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") + ProjectDir = request.AgentDir }; @@ -66,7 +65,7 @@ namespace BotSharp.Core.Engines for(int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++) { var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpPredict; - pipe.Configuration = provider.Configuration; + pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]); pipe.Settings = settings; await pipe.Predict(agent, data, meta.Pipeline.FirstOrDefault(x => x.Name == pipelines[pipeIdx])); } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs index 42b6160b..e997c81c 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs @@ -54,7 +54,7 @@ namespace BotSharp.Core.Engines.BotSharp } string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); - string template = Configuration.GetValue($"BotSharpCRFNer:template"); + string template = Configuration.GetValue($"template"); template = template.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar); var encoder = new CRFEncoder(); diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs index 40d2476a..62a32e71 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs @@ -15,17 +15,17 @@ namespace BotSharp.Core.Engines.BotSharp { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } - private TokenizerFactory _tokenizer; + private TokenizerFactory _tokenizer; public BotSharpTokenizer() { - _tokenizer = new TokenizerFactory(new TokenizationOptions - { - }, SupportedLanguage.English); + } public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { + Init(); + doc.Tokenizer = this; // same as train @@ -39,6 +39,8 @@ namespace BotSharp.Core.Engines.BotSharp public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { + Init(); + doc.Tokenizer = this; doc.Sentences = new List(); @@ -54,5 +56,20 @@ namespace BotSharp.Core.Engines.BotSharp return true; } + + private void Init() + { + if(_tokenizer == null) + { + _tokenizer = new TokenizerFactory(new TokenizationOptions + { + Pattern = Configuration.GetValue("options:pattern") + }, SupportedLanguage.English); + + string tokenizerName = Configuration.GetValue($"tokenizer"); + + _tokenizer.GetTokenizer(tokenizerName); + } + } } } diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index d5b35e5d..cb6f3f7b 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -97,7 +97,8 @@ namespace BotSharp.Core.Engines for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++) { var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain; - pipe.Configuration = provider.Configuration; + // set configuration to current section + pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]); pipe.Settings = settings; pipeModel = new PipeModel { diff --git a/BotSharp.NLP/Tokenize/TokenizerFactory.cs b/BotSharp.NLP/Tokenize/TokenizerFactory.cs index fd8b22ec..35923d08 100644 --- a/BotSharp.NLP/Tokenize/TokenizerFactory.cs +++ b/BotSharp.NLP/Tokenize/TokenizerFactory.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Text; using System.Threading.Tasks; @@ -12,19 +13,35 @@ namespace BotSharp.NLP.Tokenize /// The particular tokenizer requires implement interface /// models to be installed.BotSharp.NLP also provides a simpler, regular-expression based tokenizer, which splits text on whitespace and punctuation. /// - public class TokenizerFactory where ITokenize : ITokenizer, new() + public class TokenizerFactory { private SupportedLanguage _lang; - private ITokenize _tokenizer; + private ITokenizer _tokenizer; private TokenizationOptions _options; + public ITokenizer GetTokenizer() where ITokenize : ITokenizer, new() + { + return _tokenizer = new ITokenize(); + } + + public ITokenizer GetTokenizer(string name) + { + List types = Assembly.Load(new AssemblyName("BotSharp.NLP")) + .GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList(); + + Type type = types.FirstOrDefault(x => x.Name == name); + var instance = (ITokenizer)Activator.CreateInstance(type); + + return _tokenizer = instance; + } + public TokenizerFactory(TokenizationOptions options, SupportedLanguage lang) { _lang = lang; _options = options; - _tokenizer = new ITokenize(); + } public List Tokenize(string sentence) diff --git a/Settings/bot.json b/Settings/bot.json index b41b7b92..5e921de7 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -11,6 +11,10 @@ "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier" }, + "BotSharpTokenizer": { + "tokenizer": "TreebankTokenizer" + }, + "BotSharpNBayesClassifier": { }, @@ -19,6 +23,7 @@ }, "BotSharpTagger": { + }, "BotSharpCRFNer": {