From ec67e768a0ecc8cba8d42c1deb25aee758fa7f26 Mon Sep 17 00:00:00 2001 From: "haiping008@gmail.com" Date: Wed, 8 Aug 2018 16:39:00 -0500 Subject: [PATCH] Add persist and load model function. --- .gitignore | 4 + BotSharp.Core/Abstractions/IBotPlatform.cs | 3 +- BotSharp.Core/Abstractions/INlpPipeline.cs | 12 ++- BotSharp.Core/Engines/BotEngineBase.cs | 13 +++- BotSharp.Core/Engines/BotPreditor.cs | 53 +++++++++++++ BotSharp.Core/Engines/BotSharp/BotSharpAi.cs | 9 +-- BotSharp.Core/Engines/BotTrainer.cs | 51 +++++++++++-- .../CRFsuite/CRFsuiteEntityRecognizer.cs | 43 ++++++++--- BotSharp.Core/Engines/ModelMetaData.cs | 17 +++++ BotSharp.Core/Engines/PipeModel.cs | 27 +++++++ BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs | 8 +- .../Engines/SpaCy/SpaCyEntityRecognizer.cs | 8 +- BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs | 24 +++++- BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs | 11 ++- .../Engines/SpaCy/SpaCyTextCategorizer.cs | 8 +- BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs | 11 ++- .../Engines/SpaCy/SpacyFeaturizer.cs | 8 +- BotSharp.MachineLearning/CRFsuite/Ner.cs | 5 +- BotSharp.RestApi/Dialogs/DialogController.cs | 9 +-- .../DbInitializer/Accounts/users.json | 1 + BotSharp.WebHost/BotSharp.WebHost.csproj | 8 ++ BotSharp.WebHost/Settings/app.json | 3 +- BotSharp.WebHost/Settings/bot.json | 1 + BotSharp.WebHost/Startup.cs | 74 ++++++------------- 24 files changed, 310 insertions(+), 101 deletions(-) create mode 100644 BotSharp.Core/Engines/BotPreditor.cs create mode 100644 BotSharp.Core/Engines/ModelMetaData.cs create mode 100644 BotSharp.Core/Engines/PipeModel.cs diff --git a/.gitignore b/.gitignore index 6c9e8cd6..52ba8fc0 100644 --- a/.gitignore +++ b/.gitignore @@ -297,3 +297,7 @@ __pycache__/ /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.parsed.txt /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.model /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.corpus.txt +/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.model +/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/metadata.json +/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.parsed.txt +/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.corpus.txt diff --git a/BotSharp.Core/Abstractions/IBotPlatform.cs b/BotSharp.Core/Abstractions/IBotPlatform.cs index a07c4198..546c6b11 100644 --- a/BotSharp.Core/Abstractions/IBotPlatform.cs +++ b/BotSharp.Core/Abstractions/IBotPlatform.cs @@ -4,6 +4,7 @@ using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Engines { @@ -18,6 +19,6 @@ namespace BotSharp.Core.Engines AIResponse TextRequest(AIRequest request); - void Train(); + Task Train(); } } diff --git a/BotSharp.Core/Abstractions/INlpPipeline.cs b/BotSharp.Core/Abstractions/INlpPipeline.cs index f1e7a2e4..eb16b353 100644 --- a/BotSharp.Core/Abstractions/INlpPipeline.cs +++ b/BotSharp.Core/Abstractions/INlpPipeline.cs @@ -1,9 +1,11 @@ using BotSharp.Core.Agents; +using BotSharp.Core.Engines; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Abstractions { @@ -14,6 +16,14 @@ namespace BotSharp.Core.Abstractions { IConfiguration Configuration { get; set; } - bool ProcessAsync(Agent agent, JObject data); + /// + /// Process + /// + /// + /// Intermediate result + /// Meta data which is packed to model + /// + Task Train(Agent agent, JObject data, PipeModel meta); + Task Predict(Agent agent, JObject data, PipeModel meta); } } diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index 6445072b..c5854450 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -4,11 +4,13 @@ using BotSharp.Core.Intents; using BotSharp.Core.Models; using EntityFrameworkCore.BootKit; using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Engines { @@ -32,6 +34,13 @@ namespace BotSharp.Core.Engines DbInitializerPath = Path.Join(dataPath, $"DbInitializer"); } + public AIResponse TextRequest(AIRequest request) + { + var preditor = new BotPreditor(); + var text = preditor.Predict(agent, request); + return null; + } + public Agent LoadAgent(string id) { if (agent == null) @@ -177,9 +186,9 @@ namespace BotSharp.Core.Engines return corpus; } - public virtual void Train() + public virtual Task Train() { - + return Task.CompletedTask; } } diff --git a/BotSharp.Core/Engines/BotPreditor.cs b/BotSharp.Core/Engines/BotPreditor.cs new file mode 100644 index 00000000..e6522a58 --- /dev/null +++ b/BotSharp.Core/Engines/BotPreditor.cs @@ -0,0 +1,53 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.Core.Models; +using DotNetToolkit; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines +{ + public class BotPreditor + { + public async Task Predict(Agent agent, AIRequest request) + { + // load model + var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); + Console.WriteLine($"Load model from {dir}"); + var metaJson = File.ReadAllText(Path.Join(dir, "metadata.json")); + var meta = JsonConvert.DeserializeObject(metaJson); + + // Get NLP Provider + var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration"); + var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies"); + + var providerPipe = meta.Pipeline.First(); + var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline; + provider.Configuration = config.GetSection(meta.Platform); + + var data = JObject.FromObject(new + { + }); + + await provider.Train(agent, data, providerPipe); + meta.Pipeline.RemoveAt(0); + + // pipe process + meta.Pipeline.ForEach(async pipeMeta => + { + var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline; + pipe.Configuration = provider.Configuration; + await pipe.Predict(agent, data, pipeMeta); + }); + + return ""; + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs index bec9333e..4ea57d6f 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs @@ -1,21 +1,18 @@ using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; using BotSharp.Core.Models; namespace BotSharp.Core.Engines.BotSharp { public class BotSharpAi : BotEngineBase, IBotPlatform { - public AIResponse TextRequest(AIRequest request) - { - throw new NotImplementedException(); - } - public override void Train() + public override async Task Train() { agent.Corpus = GetIntentExpressions(); var trainer = new BotTrainer(agent.Id, dc); - trainer.Train(agent); + await trainer.Train(agent); } } } diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 3461c9fb..b102c18f 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.Core.Intents; @@ -9,7 +11,9 @@ using DotNetToolkit; using EntityFrameworkCore.BootKit; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; namespace BotSharp.Core.Engines { @@ -25,7 +29,7 @@ namespace BotSharp.Core.Engines this.agentId = agentId; } - public string Train(Agent agent) + public async Task Train(Agent agent) { agent.Intents = dc.Table() .Include(x => x.Contexts) @@ -47,9 +51,25 @@ namespace BotSharp.Core.Engines string providerName = config.GetSection($"{platform}:Provider").Value; var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline; provider.Configuration = config.GetSection(platform); - provider.ProcessAsync(agent, data); - //var corpus = agent.GrabCorpus(dc); + var pipeModel = new PipeModel + { + Name = providerName, + Class = provider.ToString(), + Meta = new JObject(), + Time = DateTime.UtcNow + }; + + await provider.Train(agent, data, pipeModel); + + var meta = new ModelMetaData + { + Platform = platform, + Language = agent.Language, + TrainingDate = DateTime.UtcNow, + Version = config.GetValue($"Version"), + Pipeline = new List() { pipeModel } + }; // pipe process var pipelines = provider.Configuration.GetSection($"Pipe").Value @@ -57,15 +77,34 @@ namespace BotSharp.Core.Engines .Select(x => x.Trim()) .ToList(); - pipelines.ForEach(pipeName => + pipelines.ForEach(async pipeName => { var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline; pipe.Configuration = provider.Configuration; - pipe.ProcessAsync(agent, data); + pipeModel = new PipeModel + { + Name = pipeName, + Class = pipe.ToString(), + Time = DateTime.UtcNow + }; + meta.Pipeline.Add(pipeModel); + + await pipe.Train(agent, data, pipeModel); }); + // save model meta data + var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); + var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings + { + Formatting = Formatting.Indented, + NullValueHandling = NullValueHandling.Ignore, + ContractResolver = new CamelCasePropertyNamesContractResolver() + }); + File.WriteAllText(Path.Join(dir, "metadata.json"), metaJson); - return ""; + Console.WriteLine(metaJson); + + return metaJson; } } } diff --git a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs index 9f85f453..8d082818 100644 --- a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs @@ -21,7 +21,7 @@ namespace BotSharp.Core.Engines.CRFsuite { public IConfiguration Configuration { get; set; } - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { var dc = new DefaultDataContextLoader().GetDefaultDc(); var corpus = agent.Corpus; @@ -30,11 +30,22 @@ namespace BotSharp.Core.Engines.CRFsuite List> userSays = corpus.UserSays; List> list = new List>(); - var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles"); - string rawTrainingDataFileName = Path.Join(dir, $"{agent.Id}.corpus.txt"); - string parsedTrainingDataFileName = Path.Join(dir, $"{agent.Id}.parsed.txt"); - string modelFileName = Path.Join(dir, $"{agent.Id}.model"); - string logFileName = Path.Join(dir, $"{agent.Id}.log.txt"); + var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id); + if (!Directory.Exists(dirTrain)) + { + Directory.CreateDirectory(dirTrain); + } + + var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); + if (!Directory.Exists(dirModel)) + { + Directory.CreateDirectory(dirModel); + } + + string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt"); + string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt"); + string modelFileName = Path.Join(dirModel, $"ner-crf.model"); + string logFileName = Path.Join(dirTrain, $"ner-crf.log.txt"); using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) { @@ -56,18 +67,23 @@ namespace BotSharp.Core.Engines.CRFsuite } } - - var uniFeatures = Configuration.GetValue($"CRFsuiteEntityRecognizer:uniFeatures").Split(" "); - var biFeatures = Configuration.GetValue($"CRFsuiteEntityRecognizer:biFeatures").Split(" "); + var fields = Configuration.GetValue($"CRFsuiteEntityRecognizer:fields"); + var uniFeatures = Configuration.GetValue($"CRFsuiteEntityRecognizer:uniFeatures"); + var biFeatures = Configuration.GetValue($"CRFsuiteEntityRecognizer:biFeatures"); new MachineLearning.CRFsuite.Ner() - .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, uniFeatures, biFeatures); + .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" ")); var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x Console.WriteLine($"Saved model to {modelFileName}"); + meta.Meta = new JObject(); + meta.Meta["model"] = $"ner-crf.model"; + meta.Meta["fields"] = fields; + meta.Meta["uniFeatures"] = uniFeatures; + meta.Meta["biFeatures"] = biFeatures; return true; } @@ -125,8 +141,13 @@ namespace BotSharp.Core.Engines.CRFsuite i = i + wordCandidateCount - 1; } } - return trainingTuple; + return trainingTuple; + } + + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; } } diff --git a/BotSharp.Core/Engines/ModelMetaData.cs b/BotSharp.Core/Engines/ModelMetaData.cs new file mode 100644 index 00000000..6ea1ea64 --- /dev/null +++ b/BotSharp.Core/Engines/ModelMetaData.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public class ModelMetaData + { + public string Platform { get; set; } + public string Language { get; set; } + + public string Version { get; set; } + public DateTime TrainingDate { get; set; } + + public List Pipeline { get; set; } + } +} diff --git a/BotSharp.Core/Engines/PipeModel.cs b/BotSharp.Core/Engines/PipeModel.cs new file mode 100644 index 00000000..5791b7ac --- /dev/null +++ b/BotSharp.Core/Engines/PipeModel.cs @@ -0,0 +1,27 @@ +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public class PipeModel + { + /// + /// Pipe name + /// + public string Name { get; set; } + + /// + /// Pipe type name + /// + public string Class { get; set; } + + public DateTime Time { get; set; } + + /// + /// Extra meta data according to pipe + /// + public JObject Meta { get; set; } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs index f2dc390c..8d32ed33 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.Core.Models; @@ -15,7 +16,12 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool ProcessAsync(Agent agent, JObject data) + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } + + public async Task Train(Agent agent, JObject data, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("entitize", Method.GET); diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs index 4aa38442..acfb1f59 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { @@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy List entitiesInTrainingSet = new List(); public IConfiguration Configuration { get; set; } - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { String modelPath = "./entity_rec_output"; String newModelName = "test"; @@ -54,6 +55,11 @@ namespace BotSharp.Core.Engines.SpaCy return true; } + + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } } public class Result diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index 18ede571..e5991559 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -1,26 +1,44 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using RestSharp; using System; using System.Collections.Generic; using System.Net.Http; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { public class SpaCyProvider : INlpPipeline { public IConfiguration Configuration { get; set; } - - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("load", Method.GET); - var response = client.Execute(request); + var response = client.Execute(request); + + meta.Meta = JObject.FromObject(response.Data); return response.IsSuccessful; } + + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } + + private class Result + { + [JsonProperty("spaCy ver")] + public string Version { get; set; } + [JsonProperty("models")] + public string Models { get; set; } + [JsonProperty("python ver")] + public string Python { get; set; } + } } } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs index 307802fb..15878f76 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Text; using BotSharp.MachineLearning.NLP; +using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { @@ -15,8 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tagger", Method.GET); @@ -36,7 +36,12 @@ namespace BotSharp.Core.Engines.SpaCy return res; } - public class Result + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } + + private class Result { public List Tags { get; set; } } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs index e2e13830..6ae93765 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { @@ -16,7 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { //var input = new List>(); @@ -62,6 +63,11 @@ namespace BotSharp.Core.Engines.SpaCy return true; } + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } + public class Result { public String ModelName { get; set; } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index 17e2b672..f68d5417 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -10,6 +10,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace BotSharp.Core.Engines.SpaCy { @@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tokenizer", Method.GET); @@ -41,9 +42,13 @@ namespace BotSharp.Core.Engines.SpaCy return res; } - - public class Result + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } + + private class Result { public List Tokens { get; set; } } diff --git a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs index cd44837d..2293a2fb 100644 --- a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using EntityFrameworkCore.BootKit; @@ -14,7 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool ProcessAsync(Agent agent, JObject data) + public async Task Train(Agent agent, JObject data, PipeModel meta) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("featurize", Method.GET); @@ -35,6 +36,11 @@ namespace BotSharp.Core.Engines.SpaCy return res; } + public async Task Predict(Agent agent, JObject data, PipeModel meta) + { + return true; + } + public class Result { public List Vectors { get; set; } diff --git a/BotSharp.MachineLearning/CRFsuite/Ner.cs b/BotSharp.MachineLearning/CRFsuite/Ner.cs index ec4b9222..c4b63a90 100644 --- a/BotSharp.MachineLearning/CRFsuite/Ner.cs +++ b/BotSharp.MachineLearning/CRFsuite/Ner.cs @@ -11,8 +11,7 @@ namespace BotSharp.MachineLearning.CRFsuite { // Separator of field values. string separator = " "; - // Field names of the input data. - string fields = "y w pos chk"; + Template templates = new Template(); public string GetShape (string token) @@ -515,7 +514,7 @@ namespace BotSharp.MachineLearning.CRFsuite } } - public void NerStart (string rawFile, string parsedName, string[] uniFeatures, string[] biFeatures) + public void NerStart(string rawFile, string parsedName, string fields, string[] uniFeatures, string[] biFeatures) { InitialTemplate(uniFeatures, biFeatures); new Crfutils().CRFFileGenerator(FeatureExtractor, fields, rawFile, parsedName, separator); diff --git a/BotSharp.RestApi/Dialogs/DialogController.cs b/BotSharp.RestApi/Dialogs/DialogController.cs index 85487baf..701c541d 100644 --- a/BotSharp.RestApi/Dialogs/DialogController.cs +++ b/BotSharp.RestApi/Dialogs/DialogController.cs @@ -13,6 +13,7 @@ namespace BotSharp.RestApi.Dialogs /// /// Conversation controller /// + [Authorize] [Route("v1/[controller]")] public class DialogController : ControllerBase { @@ -33,16 +34,10 @@ namespace BotSharp.RestApi.Dialogs /// /// /// - [AllowAnonymous] [HttpPost("/v1/query")] public ActionResult Query([FromBody] QueryModel request) { - String clientAccessToken = Request.Headers["Authorization"]; - if (String.IsNullOrEmpty(clientAccessToken)) - { - return Unauthorized(); - } - + String clientAccessToken = Request.Headers["ClientAccessToken"]; var config = new AIConfiguration(clientAccessToken, SupportedLanguage.English); config.SessionId = request.SessionId; diff --git a/BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json b/BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json index 9b7fabcb..16d0b63b 100644 --- a/BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json +++ b/BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json @@ -1,5 +1,6 @@ [ { + "id": "54cc19ee-e3d5-4d59-a011-fa0121450e36", "userName": "botsharp", "email": "support@botsharp.io", "firstName": "Support", diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index c467e00c..ed0fcc38 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -6,10 +6,18 @@ + + + + + + + + diff --git a/BotSharp.WebHost/Settings/app.json b/BotSharp.WebHost/Settings/app.json index e60729fd..9ea54790 100644 --- a/BotSharp.WebHost/Settings/app.json +++ b/BotSharp.WebHost/Settings/app.json @@ -1,4 +1,5 @@ { "Assemblies": "BotSharp.Core", - "BotPlatform": "BotSharpAi" + "BotPlatform": "BotSharpAi", + "Version": "0.1.0" } diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json index c6c3d52f..959f4f61 100644 --- a/BotSharp.WebHost/Settings/bot.json +++ b/BotSharp.WebHost/Settings/bot.json @@ -11,6 +11,7 @@ }, "Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer", "CRFsuiteEntityRecognizer": { + "fields": "y w pos chk", "uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs", "biFeatures": "w pos chk shaped type" } diff --git a/BotSharp.WebHost/Startup.cs b/BotSharp.WebHost/Startup.cs index 140496da..0518ebe5 100644 --- a/BotSharp.WebHost/Startup.cs +++ b/BotSharp.WebHost/Startup.cs @@ -16,6 +16,8 @@ using Swashbuckle.AspNetCore.Swagger; using BotSharp.Core.Engines.BotSharp; using System.Collections.Generic; using Newtonsoft.Json; +using DotNetToolkit.JwtHelper; +using BotSharp.Core.Agents; namespace BotSharp.WebHost { @@ -31,6 +33,7 @@ namespace BotSharp.WebHost public void ConfigureServices(IServiceCollection services) { services.AddCors(); + services.AddJwtAuth(Configuration); services.AddMvc(options => { @@ -43,6 +46,18 @@ namespace BotSharp.WebHost services.AddSwaggerGen(c => { + c.AddSecurityDefinition("Bearer", new ApiKeyScheme() + { + In = "header", + Description = "Please insert JWT with Bearer schema. Example: \"Authorization: Bearer {token}\"", + Name = "Authorization", + Type = "apiKey" + }); + + c.AddSecurityRequirement(new Dictionary> { + { "Bearer", Enumerable.Empty() }, + }); + var info = Configuration.GetSection("Swagger").Get(); c.SwaggerDoc(info.Version, info); @@ -74,10 +89,7 @@ namespace BotSharp.WebHost app.UseDefaultFiles(); app.UseStaticFiles(); - app.UseSwagger(c => - { - - }); + app.UseSwagger(); app.UseSwaggerUI(c => { var info = Configuration.GetSection("Swagger").Get(); @@ -95,8 +107,15 @@ namespace BotSharp.WebHost app.Use(async (context, next) => { string token = context.Request.Headers["Authorization"]; - if (string.IsNullOrWhiteSpace(token)) + if (!string.IsNullOrWhiteSpace(token) && (token = token.Split(' ').Last()).Length == 32) { + var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration"); + context.Request.Headers["ClientAccessToken"] = token; + + var dc = new DefaultDataContextLoader().GetDefaultDc(); + var userId = dc.Table().FirstOrDefault(x => x.ClientAccessToken == token)?.UserId; + + context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId); } await next.Invoke(); @@ -114,51 +133,6 @@ namespace BotSharp.WebHost loader.Env = env; loader.Config = Configuration; loader.Load(); - - /*Runcmd(); - var ai = new BotSharpAi(); - ai.LoadAgent("6a9fd374-c43d-447a-97f2-f37540d0c725"); - ai.Train();*/ - } - - public void Runcmd () - { - string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/1.txt"; - System.Diagnostics.Process p = new System.Diagnostics.Process(); - p.StartInfo.FileName = "sh"; - p.StartInfo.UseShellExecute = false; //是否使用操作系统shell启动 - p.StartInfo.RedirectStandardInput = true;//接受来自调用程序的输入信息 - p.StartInfo.RedirectStandardOutput = true;//由调用程序获取输出信息 - p.StartInfo.RedirectStandardError = true;//重定向标准错误输出 - p.StartInfo.CreateNoWindow = false;//不显示程序窗口 - p.Start();//启动程序 - - //向cmd窗口发送输入信息 - p.StandardInput.WriteLine(cmd + "&exit"); - - p.StandardInput.AutoFlush = false; - //p.StandardInput.WriteLine("exit"); - //向标准输入写入要执行的命令。这里使用&是批处理命令的符号，表示前面一个命令不管是否执行成功都执行后面(exit)命令，如果不执行exit命令，后面调用ReadToEnd()方法会假死 - //同类的符号还有&&和||前者表示必须前一个命令执行成功才会执行后面的命令，后者表示必须前一个命令执行失败才会执行后面的命令 - - - - //获取cmd窗口的输出信息 - string output = p.StandardOutput.ReadToEnd(); - - //StreamReader reader = p.StandardOutput; - //string line=reader.ReadLine(); - //while (!reader.EndOfStream) - //{ - // str += line + " "; - // line = reader.ReadLine(); - //} - - p.WaitForExit();//等待程序执行完退出进程 - p.Close(); - - - Console.WriteLine(output); } } } \ No newline at end of file