From a68e920ebfb6f3c53b87a1547ba37e4db3d8c7c9 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Sun, 12 Aug 2018 23:14:02 -0500 Subject: [PATCH] Separate training and predictive processing pipelines so that new processes can be added at any time during prediction without retraining the model. --- BotSharp.Core/Abstractions/INlpNer.cs | 2 +- BotSharp.Core/Abstractions/INlpPipeline.cs | 10 --------- BotSharp.Core/Abstractions/INlpPredict.cs | 14 +++++++++++++ BotSharp.Core/Abstractions/INlpProvider.cs | 14 +++++++++++++ BotSharp.Core/Abstractions/INlpTrain.cs | 21 +++++++++++++++++++ BotSharp.Core/Engines/BotEngineBase.cs | 2 +- .../{BotPreditor.cs => BotPredictor.cs} | 19 ++++++++++------- BotSharp.Core/Engines/BotTrainer.cs | 6 +++--- .../Engines/Classifiers/FasttextClassifier.cs | 2 +- .../Engines/NERs/CRFsuiteEntityRecognizer.cs | 2 +- .../Engines/NERs/DucklingEntityRecognizer.cs | 4 ++++ .../Engines/NERs/WitAiEntityRecognizer.cs | 2 +- BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs | 9 ++------ BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs | 2 +- BotSharp.WebHost/Settings/bot.json | 7 +++++-- 15 files changed, 81 insertions(+), 35 deletions(-) create mode 100644 BotSharp.Core/Abstractions/INlpPredict.cs create mode 100644 BotSharp.Core/Abstractions/INlpProvider.cs create mode 100644 BotSharp.Core/Abstractions/INlpTrain.cs rename BotSharp.Core/Engines/{BotPreditor.cs => BotPredictor.cs} (83%) diff --git a/BotSharp.Core/Abstractions/INlpNer.cs b/BotSharp.Core/Abstractions/INlpNer.cs index 19c911c7..ee020c99 100644 --- a/BotSharp.Core/Abstractions/INlpNer.cs +++ b/BotSharp.Core/Abstractions/INlpNer.cs @@ -5,7 +5,7 @@ using System.Text; namespace BotSharp.Core.Abstractions { - public interface INlpNer + public interface INlpNer : INlpPipeline { List Ontologies { get; } } diff --git a/BotSharp.Core/Abstractions/INlpPipeline.cs b/BotSharp.Core/Abstractions/INlpPipeline.cs index 68112a6d..3b556c3a 100644 --- a/BotSharp.Core/Abstractions/INlpPipeline.cs +++ b/BotSharp.Core/Abstractions/INlpPipeline.cs @@ -20,15 +20,5 @@ namespace BotSharp.Core.Abstractions /// Common settings for Pipeline /// PipeSettings Settings { get; set; } - - /// - /// Process - /// - /// - /// Intermediate result - /// Meta data which is packed to model - /// - Task Train(Agent agent, NlpDoc doc, PipeModel meta); - Task Predict(Agent agent, NlpDoc doc, PipeModel meta); } } diff --git a/BotSharp.Core/Abstractions/INlpPredict.cs b/BotSharp.Core/Abstractions/INlpPredict.cs new file mode 100644 index 00000000..dbac47b8 --- /dev/null +++ b/BotSharp.Core/Abstractions/INlpPredict.cs @@ -0,0 +1,14 @@ +using BotSharp.Core.Agents; +using BotSharp.Core.Engines; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Abstractions +{ + public interface INlpPredict : INlpPipeline + { + Task Predict(Agent agent, NlpDoc doc, PipeModel meta); + } +} diff --git a/BotSharp.Core/Abstractions/INlpProvider.cs b/BotSharp.Core/Abstractions/INlpProvider.cs new file mode 100644 index 00000000..5a7975cd --- /dev/null +++ b/BotSharp.Core/Abstractions/INlpProvider.cs @@ -0,0 +1,14 @@ +using BotSharp.Core.Agents; +using BotSharp.Core.Engines; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Abstractions +{ + public interface INlpProvider : INlpPipeline + { + Task Load(Agent agent, PipeModel meta); + } +} diff --git a/BotSharp.Core/Abstractions/INlpTrain.cs b/BotSharp.Core/Abstractions/INlpTrain.cs new file mode 100644 index 00000000..7a8d7209 --- /dev/null +++ b/BotSharp.Core/Abstractions/INlpTrain.cs @@ -0,0 +1,21 @@ +using BotSharp.Core.Agents; +using BotSharp.Core.Engines; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Abstractions +{ + public interface INlpTrain : INlpPipeline + { + /// + /// Process + /// + /// + /// Intermediate result + /// Meta data which is packed to model + /// + Task Train(Agent agent, NlpDoc doc, PipeModel meta); + } +} diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index c80a36ea..50bd2a40 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -37,7 +37,7 @@ namespace BotSharp.Core.Engines public AIResponse TextRequest(AIRequest request) { - var preditor = new BotPreditor(); + var preditor = new BotPredictor(); var doc = preditor.Predict(agent, request).Result; var parameters = new Dictionary(); doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value)); diff --git a/BotSharp.Core/Engines/BotPreditor.cs b/BotSharp.Core/Engines/BotPredictor.cs similarity index 83% rename from BotSharp.Core/Engines/BotPreditor.cs rename to BotSharp.Core/Engines/BotPredictor.cs index d8345d61..0d0754c3 100644 --- a/BotSharp.Core/Engines/BotPreditor.cs +++ b/BotSharp.Core/Engines/BotPredictor.cs @@ -15,7 +15,7 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines { - public class BotPreditor + public class BotPredictor { public async Task Predict(Agent agent, AIRequest request) { @@ -30,7 +30,7 @@ namespace BotSharp.Core.Engines var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies"); var providerPipe = meta.Pipeline.First(); - var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline; + var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpProvider; provider.Configuration = config.GetSection(meta.Platform); var data = new NlpDoc @@ -44,7 +44,7 @@ namespace BotSharp.Core.Engines } }; - await provider.Train(agent, data, providerPipe); + await provider.Load(agent, providerPipe); meta.Pipeline.RemoveAt(0); var settings = new PipeSettings @@ -58,14 +58,19 @@ namespace BotSharp.Core.Engines { Directory.CreateDirectory(settings.PredictDir); } - + // pipe process - meta.Pipeline.ForEach(async pipeMeta => + var pipelines = provider.Configuration.GetValue($"Pipe:predict") + .Split(',') + .Select(x => x.Trim()) + .ToList(); + + pipelines.ForEach(async pipeName => { - var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline; + var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPredict; pipe.Configuration = provider.Configuration; pipe.Settings = settings; - await pipe.Predict(agent, data, pipeMeta); + await pipe.Predict(agent, data, meta.Pipeline.FirstOrDefault(x => x.Name == pipeName)); }); Console.WriteLine(JsonConvert.SerializeObject(data, new JsonSerializerSettings diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 7f2b1364..9c6cadb4 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -47,7 +47,7 @@ namespace BotSharp.Core.Engines var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies"); var platform = config.GetSection($"BotPlatform").Value; string providerName = config.GetSection($"{platform}:Provider").Value; - var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline; + var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpTrain; provider.Configuration = config.GetSection(platform); var pipeModel = new PipeModel @@ -87,14 +87,14 @@ namespace BotSharp.Core.Engines } // pipe process - var pipelines = provider.Configuration.GetSection($"Pipe").Value + var pipelines = provider.Configuration.GetValue($"Pipe:train") .Split(',') .Select(x => x.Trim()) .ToList(); pipelines.ForEach(async pipeName => { - var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline; + var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpTrain; pipe.Configuration = provider.Configuration; pipe.Settings = settings; pipeModel = new PipeModel diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs index f91fd12b..357ef5c6 100644 --- a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs +++ b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs @@ -13,7 +13,7 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines.Classifiers { - public class FasttextClassifier : INlpPipeline + public class FasttextClassifier : INlpTrain, INlpPredict { public IConfiguration Configuration { get; set; } diff --git a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs index b9e4ca59..e8d5de8f 100644 --- a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs @@ -18,7 +18,7 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines.NERs { - public class CRFsuiteEntityRecognizer : INlpPipeline, INlpNer + public class CRFsuiteEntityRecognizer : INlpTrain, INlpPredict, INlpNer { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } diff --git a/BotSharp.Core/Engines/NERs/DucklingEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/DucklingEntityRecognizer.cs index b55591ba..2405aa20 100644 --- a/BotSharp.Core/Engines/NERs/DucklingEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/DucklingEntityRecognizer.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Abstractions; +using Microsoft.Extensions.Configuration; using System; using System.Collections.Generic; using System.Text; @@ -8,5 +9,8 @@ namespace BotSharp.Core.Engines.NERs public class DucklingEntityRecognizer : INlpNer { public List Ontologies => throw new NotImplementedException(); + + public IConfiguration Configuration { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public PipeSettings Settings { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } } } diff --git a/BotSharp.Core/Engines/NERs/WitAiEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/WitAiEntityRecognizer.cs index 8be3369a..9f85cfc5 100644 --- a/BotSharp.Core/Engines/NERs/WitAiEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/WitAiEntityRecognizer.cs @@ -15,7 +15,7 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines.NERs { - public class WitAiEntityRecognizer : INlpPipeline, INlpNer + public class WitAiEntityRecognizer : INlpPredict, INlpNer { public List Ontologies { diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index 39a0a878..55a9f978 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -12,12 +12,12 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { - public class SpaCyProvider : INlpPipeline + public class SpaCyProvider : INlpProvider { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } - public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + public async Task Load(Agent agent, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("load", Method.GET); @@ -30,11 +30,6 @@ namespace BotSharp.Core.Engines.SpaCy return response.IsSuccessful; } - public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) - { - return true; - } - private class Result { [JsonProperty("spaCy ver")] diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index 25672ffd..8a834d1c 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -14,7 +14,7 @@ using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { - public class SpaCyTokenizer : INlpPipeline + public class SpaCyTokenizer : INlpTrain, INlpPredict { public IConfiguration Configuration { get; set; } public PipeSettings Settings { get; set; } diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index b498eadb..7765b13d 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -10,9 +10,12 @@ "url": "http://localhost:5005" }, - "Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier", + "Pipe": { + "train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier", + "predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer" + }, + "SpaCyTokenizer": { - "url": "http://localhost:5005" }, "CRFsuiteEntityRecognizer": { "fields": "y w pos chk",