diff --git a/BotSharp.Core/Abstractions/INlpPipeline.cs b/BotSharp.Core/Abstractions/INlpPipeline.cs index eb16b353..09e98807 100644 --- a/BotSharp.Core/Abstractions/INlpPipeline.cs +++ b/BotSharp.Core/Abstractions/INlpPipeline.cs @@ -16,6 +16,11 @@ namespace BotSharp.Core.Abstractions { IConfiguration Configuration { get; set; } + /// + /// Common settings for Pipeline + /// + PipeSettings Settings { get; set; } + /// /// Process /// diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index fec84743..fca3948d 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -35,7 +35,7 @@ - + diff --git a/BotSharp.Core/Engines/BotPreditor.cs b/BotSharp.Core/Engines/BotPreditor.cs index e6522a58..260a36f2 100644 --- a/BotSharp.Core/Engines/BotPreditor.cs +++ b/BotSharp.Core/Engines/BotPreditor.cs @@ -34,16 +34,24 @@ namespace BotSharp.Core.Engines var data = JObject.FromObject(new { + Text = request.Query.FirstOrDefault() }); await provider.Train(agent, data, providerPipe); meta.Pipeline.RemoveAt(0); + var settings = new PipeSettings + { + ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id), + AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") + }; + // pipe process meta.Pipeline.ForEach(async pipeMeta => { var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline; pipe.Configuration = provider.Configuration; + pipe.Settings = settings; await pipe.Predict(agent, data, pipeMeta); }); diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index ed56e360..1dd1986c 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -71,16 +71,21 @@ namespace BotSharp.Core.Engines Pipeline = new List() { pipeModel } }; - var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); - if (!Directory.Exists(dirTrain)) + var settings = new PipeSettings { - Directory.CreateDirectory(dirTrain); + TrainDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id), + ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id), + AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") + }; + + if (!Directory.Exists(settings.TrainDir)) + { + Directory.CreateDirectory(settings.TrainDir); } - var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); - if (!Directory.Exists(dirModel)) + if (!Directory.Exists(settings.ModelDir)) { - Directory.CreateDirectory(dirModel); + Directory.CreateDirectory(settings.ModelDir); } // pipe process @@ -93,6 +98,7 @@ namespace BotSharp.Core.Engines { var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline; pipe.Configuration = provider.Configuration; + pipe.Settings = settings; pipeModel = new PipeModel { Name = pipeName, diff --git a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs index 673d27da..8614ed7f 100644 --- a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs @@ -20,6 +20,7 @@ namespace BotSharp.Core.Engines.CRFsuite public class CRFsuiteEntityRecognizer : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Train(Agent agent, JObject data, PipeModel meta) { @@ -32,11 +33,9 @@ namespace BotSharp.Core.Engines.CRFsuite List> userSays = corpus.UserSays; List> list = new List>(); - var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); - var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); - string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt"); - string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt"); - string modelFileName = Path.Join(dirModel, meta.Model); + string rawTrainingDataFileName = Path.Join(Settings.TrainDir, "ner-crf.corpus.txt"); + string parsedTrainingDataFileName = Path.Join(Settings.TrainDir, "ner-crf.parsed.txt"); + string modelFileName = Path.Join(Settings.ModelDir, meta.Model); using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) { @@ -65,9 +64,7 @@ namespace BotSharp.Core.Engines.CRFsuite new MachineLearning.CRFsuite.Ner() .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" ")); - var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); - - CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x + CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs index 3d442da1..388d0b9b 100644 --- a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs +++ b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs @@ -14,21 +14,22 @@ namespace BotSharp.Core.Engines.Classifiers public class FasttextClassifier : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } - public Task Predict(Agent agent, JObject data, PipeModel meta) + public async Task Predict(Agent agent, JObject data, PipeModel meta) { - throw new NotImplementedException(); + string modelFileName = Path.Join(Settings.ModelDir, meta.Model); + var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), "predict {modelFileName}.bin test.txt"); + + return true; } public async Task Train(Agent agent, JObject data, PipeModel meta) { meta.Model = "classification-fasttext.model"; - var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); - var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); - var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); - string parsedTrainingDataFileName = Path.Join(dirTrain, $"classification-fasttext.parsed.txt"); - string modelFileName = Path.Join(dirModel, meta.Model); + string parsedTrainingDataFileName = Path.Join(Settings.TrainDir, $"classification-fasttext.parsed.txt"); + string modelFileName = Path.Join(Settings.ModelDir, meta.Model); // assemble corpus StringBuilder corpus = new StringBuilder(); @@ -36,7 +37,7 @@ namespace BotSharp.Core.Engines.Classifiers File.WriteAllText(parsedTrainingDataFileName, corpus.ToString()); - var output = CmdHelper.Run(Path.Join(algorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}"); + var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}"); Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); diff --git a/BotSharp.Core/Engines/PipeSettings.cs b/BotSharp.Core/Engines/PipeSettings.cs new file mode 100644 index 00000000..1c699a6d --- /dev/null +++ b/BotSharp.Core/Engines/PipeSettings.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public class PipeSettings + { + public string TrainDir { get; set; } + public string ModelDir { get; set; } + public string AlgorithmDir { get; set; } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs index 8d32ed33..dba99988 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs @@ -15,6 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy public class SpaCyEntitizer : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Predict(Agent agent, JObject data, PipeModel meta) { diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs index acfb1f59..328583d9 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs @@ -17,6 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy { List entitiesInTrainingSet = new List(); public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Train(Agent agent, JObject data, PipeModel meta) { diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index f61d9810..38b5dd27 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -15,6 +15,8 @@ namespace BotSharp.Core.Engines.SpaCy public class SpaCyProvider : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + public async Task Train(Agent agent, JObject data, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs index 15878f76..bd372e71 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs @@ -15,6 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy public class SpaCyTagger : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Train(Agent agent, JObject data, PipeModel meta) { diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs index 6ae93765..48bbf845 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs @@ -16,6 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy public class SpaCyTextCategorizer : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Train(Agent agent, JObject data, PipeModel meta) { diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index f68d5417..7e4d28ef 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -17,6 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy public class SpaCyTokenizer : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Train(Agent agent, JObject data, PipeModel meta) { diff --git a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs index 2293a2fb..6aaa30d1 100644 --- a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs @@ -14,6 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy public class SpacyFeaturizer : INlpPipeline { public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } public async Task Train(Agent agent, JObject data, PipeModel meta) {