Add persist and load model function.

.gitignore

@@ -297,3 +297,7 @@ __pycache__/
 /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.parsed.txt
 /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.model
 /BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.corpus.txt
+/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.model
+/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/metadata.json
+/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.parsed.txt
+/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.corpus.txt

BotSharp.Core/Abstractions/IBotPlatform.cs

@@ -4,6 +4,7 @@ using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines
 {
@@ -18,6 +19,6 @@ namespace BotSharp.Core.Engines
 
         AIResponse TextRequest(AIRequest request);
 
-        void Train();
+        Task Train();
     }
 }

BotSharp.Core/Abstractions/INlpPipeline.cs

@@ -1,9 +1,11 @@
 using BotSharp.Core.Agents;
+using BotSharp.Core.Engines;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Abstractions
 {
@@ -14,6 +16,14 @@ namespace BotSharp.Core.Abstractions
     {
         IConfiguration Configuration { get; set; }
 
-        bool ProcessAsync(Agent agent, JObject data);
+        /// <summary>
+        /// Process
+        /// </summary>
+        /// <param name="agent"></param>
+        /// <param name="data">Intermediate result</param>
+        /// <param name="meta">Meta data which is packed to model</param>
+        /// <returns></returns>
+        Task<bool> Train(Agent agent, JObject data, PipeModel meta);
+        Task<bool> Predict(Agent agent, JObject data, PipeModel meta);
     }
 }

BotSharp.Core/Engines/BotEngineBase.cs

@@ -4,11 +4,13 @@ using BotSharp.Core.Intents;
 using BotSharp.Core.Models;
 using EntityFrameworkCore.BootKit;
 using Microsoft.EntityFrameworkCore;
+using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines
 {
@@ -32,6 +34,13 @@ namespace BotSharp.Core.Engines
             DbInitializerPath = Path.Join(dataPath, $"DbInitializer");
         }
 
+        public AIResponse TextRequest(AIRequest request)
+        {
+            var preditor = new BotPreditor();
+            var text = preditor.Predict(agent, request);
+            return null;
+        }
+
         public Agent LoadAgent(string id)
         {
             if (agent == null)
@@ -177,9 +186,9 @@ namespace BotSharp.Core.Engines
             return corpus;
         }
 
-        public virtual void Train()
+        public virtual Task Train()
         {
-
+            return Task.CompletedTask;
         }
 
     }

BotSharp.Core/Engines/BotPreditor.cs

@@ -0,0 +1,53 @@
+using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using BotSharp.Core.Models;
+using DotNetToolkit;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Core.Engines
+{
+    public class BotPreditor
+    {
+        public async Task<string> Predict(Agent agent, AIRequest request)
+        {
+            // load model
+            var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
+            Console.WriteLine($"Load model from {dir}");
+            var metaJson = File.ReadAllText(Path.Join(dir, "metadata.json"));
+            var meta = JsonConvert.DeserializeObject<ModelMetaData>(metaJson);
+
+            // Get NLP Provider
+            var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
+            var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
+
+            var providerPipe = meta.Pipeline.First();
+            var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline;
+            provider.Configuration = config.GetSection(meta.Platform);
+
+            var data = JObject.FromObject(new
+            {
+            });
+
+            await provider.Train(agent, data, providerPipe);
+            meta.Pipeline.RemoveAt(0);
+
+            // pipe process
+            meta.Pipeline.ForEach(async pipeMeta =>
+            {
+                var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
+                pipe.Configuration = provider.Configuration;
+                await pipe.Predict(agent, data, pipeMeta);
+            });
+
+            return "";
+        }
+    }
+}

BotSharp.Core/Engines/BotSharp/BotSharpAi.cs

@@ -1,21 +1,18 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Threading.Tasks;
 using BotSharp.Core.Models;
 
 namespace BotSharp.Core.Engines.BotSharp
 {
     public class BotSharpAi : BotEngineBase, IBotPlatform
     {
-        public AIResponse TextRequest(AIRequest request)
+        public override async Task Train()
-        {
-            throw new NotImplementedException();
-        }
-        public override void Train()
         {
             agent.Corpus = GetIntentExpressions();
             var trainer = new BotTrainer(agent.Id, dc);
-            trainer.Train(agent);
+            await trainer.Train(agent);
         }
     }
 }

BotSharp.Core/Engines/BotTrainer.cs

@@ -1,7 +1,9 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 using BotSharp.Core.Abstractions;
 using BotSharp.Core.Agents;
 using BotSharp.Core.Intents;
@@ -9,7 +11,9 @@ using DotNetToolkit;
 using EntityFrameworkCore.BootKit;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
 
 namespace BotSharp.Core.Engines
 {
@@ -25,7 +29,7 @@ namespace BotSharp.Core.Engines
             this.agentId = agentId;
         }
 
-        public string Train(Agent agent)
+        public async Task<string> Train(Agent agent)
         {
             agent.Intents = dc.Table<Intent>()
                 .Include(x => x.Contexts)
@@ -47,9 +51,25 @@ namespace BotSharp.Core.Engines
             string providerName = config.GetSection($"{platform}:Provider").Value;
             var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline;
             provider.Configuration = config.GetSection(platform);
-            provider.ProcessAsync(agent, data);
 
-            //var corpus = agent.GrabCorpus(dc);
+            var pipeModel = new PipeModel
+            {
+                Name = providerName,
+                Class = provider.ToString(),
+                Meta = new JObject(),
+                Time = DateTime.UtcNow
+            };
+
+            await provider.Train(agent, data, pipeModel);
+
+            var meta = new ModelMetaData
+            {
+                Platform = platform,
+                Language = agent.Language,
+                TrainingDate = DateTime.UtcNow,
+                Version = config.GetValue<String>($"Version"),
+                Pipeline = new List<PipeModel>() { pipeModel }
+            };
 
             // pipe process
             var pipelines = provider.Configuration.GetSection($"Pipe").Value
@@ -57,15 +77,34 @@ namespace BotSharp.Core.Engines
                 .Select(x => x.Trim())
                 .ToList();
 
-            pipelines.ForEach(pipeName =>
+            pipelines.ForEach(async pipeName =>
             {
                 var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
                 pipe.Configuration = provider.Configuration;
-                pipe.ProcessAsync(agent, data);
+                pipeModel = new PipeModel
+                {
+                    Name = pipeName,
+                    Class = pipe.ToString(),
+                    Time = DateTime.UtcNow
+                };
+                meta.Pipeline.Add(pipeModel);
+
+                await pipe.Train(agent, data, pipeModel);
             });
 
+            // save model meta data
+            var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
+            var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
+            {
+                Formatting = Formatting.Indented,
+                NullValueHandling = NullValueHandling.Ignore,
+                ContractResolver = new CamelCasePropertyNamesContractResolver()
+            });
+            File.WriteAllText(Path.Join(dir, "metadata.json"), metaJson);
 
-            return "";
+            Console.WriteLine(metaJson);
+
+            return metaJson;
         }
     }
 }

BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs

@@ -21,7 +21,7 @@ namespace BotSharp.Core.Engines.CRFsuite
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool ProcessAsync(Agent agent, JObject data)
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             var dc = new DefaultDataContextLoader().GetDefaultDc();
             var corpus = agent.Corpus;
@@ -30,11 +30,22 @@ namespace BotSharp.Core.Engines.CRFsuite
             List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
             List<List<TrainingData>> list = new List<List<TrainingData>>();
 
-            var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles");
+            var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
-            string rawTrainingDataFileName = Path.Join(dir, $"{agent.Id}.corpus.txt");
+            if (!Directory.Exists(dirTrain))
-            string parsedTrainingDataFileName = Path.Join(dir, $"{agent.Id}.parsed.txt");
+            {
-            string modelFileName = Path.Join(dir, $"{agent.Id}.model");
+                Directory.CreateDirectory(dirTrain);
-            string logFileName = Path.Join(dir, $"{agent.Id}.log.txt");
+            }
+
+            var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
+            if (!Directory.Exists(dirModel))
+            {
+                Directory.CreateDirectory(dirModel);
+            }
+
+            string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt");
+            string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt");
+            string modelFileName = Path.Join(dirModel, $"ner-crf.model");
+            string logFileName = Path.Join(dirTrain, $"ner-crf.log.txt");
 
             using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
             {
@@ -56,18 +67,23 @@ namespace BotSharp.Core.Engines.CRFsuite
                 }
             }
 
-
+            var fields = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:fields");
-            var uniFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:uniFeatures").Split(" ");
+            var uniFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:uniFeatures");
-            var biFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:biFeatures").Split(" ");
+            var biFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:biFeatures");
 
             new MachineLearning.CRFsuite.Ner()
-                .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, uniFeatures, biFeatures);
+                .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" "));
 
             var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
 
             CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
 
             Console.WriteLine($"Saved model to {modelFileName}");
+            meta.Meta = new JObject();
+            meta.Meta["model"] = $"ner-crf.model";
+            meta.Meta["fields"] = fields;
+            meta.Meta["uniFeatures"] = uniFeatures;
+            meta.Meta["biFeatures"] = biFeatures;
 
             return true;
         }
@@ -125,8 +141,13 @@ namespace BotSharp.Core.Engines.CRFsuite
                     i = i + wordCandidateCount - 1;
                 }
             }
-            return trainingTuple;
 
+            return trainingTuple;
+        }
+
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
         }
     }
 

BotSharp.Core/Engines/ModelMetaData.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines
+{
+    public class ModelMetaData
+    {
+        public string Platform { get; set; }
+        public string Language { get; set; }
+
+        public string Version { get; set; }
+        public DateTime TrainingDate { get; set; }
+
+        public List<PipeModel> Pipeline { get; set; }
+    }
+}

BotSharp.Core/Engines/PipeModel.cs

@@ -0,0 +1,27 @@
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines
+{
+    public class PipeModel
+    {
+        /// <summary>
+        /// Pipe name
+        /// </summary>
+        public string Name { get; set; }
+
+        /// <summary>
+        /// Pipe type name
+        /// </summary>
+        public string Class { get; set; }
+
+        public DateTime Time { get; set; }
+
+        /// <summary>
+        /// Extra meta data according to pipe
+        /// </summary>
+        public JObject Meta { get; set; }
+    }
+}

BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Threading.Tasks;
 using BotSharp.Core.Abstractions;
 using BotSharp.Core.Agents;
 using BotSharp.Core.Models;
@@ -15,7 +16,12 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool ProcessAsync(Agent agent, JObject data)
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
+
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("entitize", Method.GET);

BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs

@@ -9,6 +9,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.SpaCy
 {
@@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy
         List<String> entitiesInTrainingSet = new List<string>();
         public IConfiguration Configuration { get; set; }
 
-        public bool ProcessAsync(Agent agent, JObject data)
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             String modelPath = "./entity_rec_output";
             String newModelName = "test";
@@ -54,6 +55,11 @@ namespace BotSharp.Core.Engines.SpaCy
 
             return true;
         }
+
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
     }
 
     public class Result

BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs

@@ -1,26 +1,44 @@
 using BotSharp.Core.Abstractions;
 using BotSharp.Core.Agents;
 using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using RestSharp;
 using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.SpaCy
 {
     public class SpaCyProvider : INlpPipeline
     {
         public IConfiguration Configuration { get; set; }
-
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
-        public bool ProcessAsync(Agent agent, JObject data)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("load", Method.GET);
-            var response = client.Execute(request);
+            var response = client.Execute<Result>(request);
+
+            meta.Meta = JObject.FromObject(response.Data);
 
             return response.IsSuccessful;
         }
+
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
+
+        private class Result
+        {
+            [JsonProperty("spaCy ver")]
+            public string Version { get; set; }
+            [JsonProperty("models")]
+            public string Models { get; set; }
+            [JsonProperty("python ver")]
+            public string Python { get; set; }
+        }
     }
 }

BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs

@@ -8,6 +8,7 @@ using System;
 using System.Collections.Generic;
 using System.Text;
 using BotSharp.MachineLearning.NLP;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.SpaCy
 {
@@ -15,8 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
-        public bool ProcessAsync(Agent agent, JObject data)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("tagger", Method.GET);
@@ -36,7 +36,12 @@ namespace BotSharp.Core.Engines.SpaCy
             return res;
         }
 
-        public class Result
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
+
+        private class Result
         {
             public List<String> Tags { get; set; }
         }

BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs

@@ -9,6 +9,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.SpaCy
 {
@@ -16,7 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool ProcessAsync(Agent agent, JObject data)
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             //var input = new List<Tuple<String, JObject>>();
 
@@ -62,6 +63,11 @@ namespace BotSharp.Core.Engines.SpaCy
             return true;
         }
 
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
+
         public class Result
         {
             public String ModelName { get; set; }

BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs

@@ -10,6 +10,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.SpaCy
 {
@@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool ProcessAsync(Agent agent, JObject data)
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("tokenizer", Method.GET);
@@ -41,9 +42,13 @@ namespace BotSharp.Core.Engines.SpaCy
 
             return res;
         }
-        
 
-        public class Result
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
+
+        private class Result
         {
             public List<NlpToken> Tokens { get; set; }
         }

BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Threading.Tasks;
 using BotSharp.Core.Abstractions;
 using BotSharp.Core.Agents;
 using EntityFrameworkCore.BootKit;
@@ -14,7 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool ProcessAsync(Agent agent, JObject data)
+        public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("featurize", Method.GET);
@@ -35,6 +36,11 @@ namespace BotSharp.Core.Engines.SpaCy
             return res;
         }
 
+        public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
+        {
+            return true;
+        }
+
         public class Result
         {
             public List<decimal> Vectors { get; set; }

BotSharp.MachineLearning/CRFsuite/Ner.cs

@@ -11,8 +11,7 @@ namespace BotSharp.MachineLearning.CRFsuite
     {
         // Separator of field values.
         string separator = " ";
-        // Field names of the input data.
+
-        string fields = "y w pos chk";
         Template templates = new Template();
         
         public string GetShape (string token) 
@@ -515,7 +514,7 @@ namespace BotSharp.MachineLearning.CRFsuite
             }
         }
 
-        public void NerStart (string rawFile, string parsedName, string[] uniFeatures, string[] biFeatures) 
+        public void NerStart(string rawFile, string parsedName, string fields, string[] uniFeatures, string[] biFeatures)
         {
             InitialTemplate(uniFeatures, biFeatures);
             new Crfutils().CRFFileGenerator(FeatureExtractor, fields, rawFile, parsedName, separator);

BotSharp.RestApi/Dialogs/DialogController.cs

@@ -13,6 +13,7 @@ namespace BotSharp.RestApi.Dialogs
     /// <summary>
     /// Conversation controller
     /// </summary>
+    [Authorize]
     [Route("v1/[controller]")]
     public class DialogController : ControllerBase
     {
@@ -33,16 +34,10 @@ namespace BotSharp.RestApi.Dialogs
         /// </summary>
         /// <param name="request"></param>
         /// <returns></returns>
-        [AllowAnonymous]
         [HttpPost("/v1/query")]
         public ActionResult<AIResponse> Query([FromBody] QueryModel request)
         {
-            String clientAccessToken = Request.Headers["Authorization"];
+            String clientAccessToken = Request.Headers["ClientAccessToken"];
-            if (String.IsNullOrEmpty(clientAccessToken))
-            {
-                return Unauthorized();
-            }
-
             var config = new AIConfiguration(clientAccessToken, SupportedLanguage.English);
             config.SessionId = request.SessionId;
 

BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json

@@ -1,5 +1,6 @@
 [
   {
+    "id": "54cc19ee-e3d5-4d59-a011-fa0121450e36",
     "userName": "botsharp",
     "email": "support@botsharp.io",
     "firstName": "Support",

BotSharp.WebHost/BotSharp.WebHost.csproj

@@ -6,10 +6,18 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Remove="App_Data\ModelFiles\**" />
     <Compile Remove="App_Data\NewFolder\**" />
+    <Compile Remove="App_Data\TrainingFiles\**" />
+    <Content Remove="App_Data\ModelFiles\**" />
     <Content Remove="App_Data\NewFolder\**" />
+    <Content Remove="App_Data\TrainingFiles\**" />
+    <EmbeddedResource Remove="App_Data\ModelFiles\**" />
     <EmbeddedResource Remove="App_Data\NewFolder\**" />
+    <EmbeddedResource Remove="App_Data\TrainingFiles\**" />
+    <None Remove="App_Data\ModelFiles\**" />
     <None Remove="App_Data\NewFolder\**" />
+    <None Remove="App_Data\TrainingFiles\**" />
   </ItemGroup>
 
   <ItemGroup>

BotSharp.WebHost/Settings/app.json

@@ -1,4 +1,5 @@
 {
   "Assemblies": "BotSharp.Core",
-  "BotPlatform": "BotSharpAi"
+  "BotPlatform": "BotSharpAi",
+  "Version": "0.1.0"
 }

BotSharp.WebHost/Settings/bot.json

@@ -11,6 +11,7 @@
     },
     "Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer",
     "CRFsuiteEntityRecognizer": {
+      "fields": "y w pos chk",
       "uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs",
       "biFeatures": "w pos chk shaped type"
     }

BotSharp.WebHost/Startup.cs

@@ -16,6 +16,8 @@ using Swashbuckle.AspNetCore.Swagger;
 using BotSharp.Core.Engines.BotSharp;
 using System.Collections.Generic;
 using Newtonsoft.Json;
+using DotNetToolkit.JwtHelper;
+using BotSharp.Core.Agents;
 
 namespace BotSharp.WebHost
 {
@@ -31,6 +33,7 @@ namespace BotSharp.WebHost
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddCors();
+            services.AddJwtAuth(Configuration);
 
             services.AddMvc(options =>
             {
@@ -43,6 +46,18 @@ namespace BotSharp.WebHost
 
             services.AddSwaggerGen(c =>
             {
+                c.AddSecurityDefinition("Bearer", new ApiKeyScheme()
+                {
+                    In = "header",
+                    Description = "Please insert JWT with Bearer schema. Example: \"Authorization: Bearer {token}\"",
+                    Name = "Authorization",
+                    Type = "apiKey"
+                });
+
+                c.AddSecurityRequirement(new Dictionary<string, IEnumerable<string>> {
+                  { "Bearer", Enumerable.Empty<string>() },
+                });
+
                 var info = Configuration.GetSection("Swagger").Get<Info>();
                 c.SwaggerDoc(info.Version, info);
 
@@ -74,10 +89,7 @@ namespace BotSharp.WebHost
             app.UseDefaultFiles();
             app.UseStaticFiles();
 
-            app.UseSwagger(c =>
+            app.UseSwagger();
-            {
-
-            });
             app.UseSwaggerUI(c =>
             {
                 var info = Configuration.GetSection("Swagger").Get<Info>();
@@ -95,8 +107,15 @@ namespace BotSharp.WebHost
             app.Use(async (context, next) =>
             {
                 string token = context.Request.Headers["Authorization"];
-                if (string.IsNullOrWhiteSpace(token))
+                if (!string.IsNullOrWhiteSpace(token) && (token = token.Split(' ').Last()).Length == 32)
                 {
+                    var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
+                    context.Request.Headers["ClientAccessToken"] = token;
+
+                    var dc = new DefaultDataContextLoader().GetDefaultDc();
+                    var userId = dc.Table<Agent>().FirstOrDefault(x => x.ClientAccessToken == token)?.UserId;
+
+                    context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId);
                 }
 
                 await next.Invoke();
@@ -114,51 +133,6 @@ namespace BotSharp.WebHost
             loader.Env = env;
             loader.Config = Configuration;
             loader.Load();
-
-            /*Runcmd();
-            var ai = new BotSharpAi();
-            ai.LoadAgent("6a9fd374-c43d-447a-97f2-f37540d0c725");
-            ai.Train();*/
-        }
-
-        public void Runcmd () 
-        {
-            string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/1.txt";
-            System.Diagnostics.Process p = new System.Diagnostics.Process();
-            p.StartInfo.FileName = "sh";
-            p.StartInfo.UseShellExecute = false;    //是否使用操作系统shell启动
-            p.StartInfo.RedirectStandardInput = true;//接受来自调用程序的输入信息
-            p.StartInfo.RedirectStandardOutput = true;//由调用程序获取输出信息
-            p.StartInfo.RedirectStandardError = true;//重定向标准错误输出
-            p.StartInfo.CreateNoWindow = false;//不显示程序窗口
-            p.Start();//启动程序
-
-            //向cmd窗口发送输入信息
-            p.StandardInput.WriteLine(cmd + "&exit");
-
-            p.StandardInput.AutoFlush = false;
-            //p.StandardInput.WriteLine("exit");
-            //向标准输入写入要执行的命令。这里使用&是批处理命令的符号，表示前面一个命令不管是否执行成功都执行后面(exit)命令，如果不执行exit命令，后面调用ReadToEnd()方法会假死
-            //同类的符号还有&&和||前者表示必须前一个命令执行成功才会执行后面的命令，后者表示必须前一个命令执行失败才会执行后面的命令
-
-
-
-            //获取cmd窗口的输出信息
-            string output = p.StandardOutput.ReadToEnd();
-
-            //StreamReader reader = p.StandardOutput;
-            //string line=reader.ReadLine();
-            //while (!reader.EndOfStream)
-            //{
-            //    str += line + "  ";
-            //    line = reader.ReadLine();
-            //}
-
-            p.WaitForExit();//等待程序执行完退出进程
-            p.Close();
-
-
-            Console.WriteLine(output);
         }
     }
 }