From f933f8d183bd1c17b42472ab25d88b5fce2fc7a7 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 13 Jun 2018 07:23:03 -0500 Subject: [PATCH] components --- BotSharp.Core/Agents/AgentDriver.cs | 6 +-- BotSharp.Core/Engines/BotTrainer.cs | 40 ++++++++++++++ .../{IBotEngine.cs => IBotPlatform.cs} | 2 +- BotSharp.Core/Engines/INlpProvider.cs | 12 +++++ BotSharp.Core/Engines/ITokenizer.cs | 13 +++++ BotSharp.Core/Engines/RasaAi.cs | 54 +++++++++++++++++-- BotSharp.Core/Engines/RequestExtension.cs | 47 ---------------- BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs | 24 +++++++++ BotSharp.Core/Intents/IntentDriver.cs | 2 +- BotSharp.Core/Utilities/JsonContent.cs | 16 ++++++ BotSharp.UnitTest/AgentTest.cs | 2 +- BotSharp.UnitTest/BotTrainerTest.cs | 19 +++++++ BotSharp.UnitTest/Settings/settings.bot.json | 11 ++++ 13 files changed, 192 insertions(+), 56 deletions(-) create mode 100644 BotSharp.Core/Engines/BotTrainer.cs rename BotSharp.Core/Engines/{IBotEngine.cs => IBotPlatform.cs} (77%) create mode 100644 BotSharp.Core/Engines/INlpProvider.cs create mode 100644 BotSharp.Core/Engines/ITokenizer.cs create mode 100644 BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs create mode 100644 BotSharp.Core/Utilities/JsonContent.cs create mode 100644 BotSharp.UnitTest/BotTrainerTest.cs diff --git a/BotSharp.Core/Agents/AgentDriver.cs b/BotSharp.Core/Agents/AgentDriver.cs index 0c72ca82..fd1faefd 100644 --- a/BotSharp.Core/Agents/AgentDriver.cs +++ b/BotSharp.Core/Agents/AgentDriver.cs @@ -14,7 +14,7 @@ namespace BotSharp.Core.Agents { public static class AgentDriver { - public static Agent LoadAgentById(this IBotEngine engine, Database dc, string agentId) + public static Agent LoadAgentById(this IBotPlatform engine, Database dc, string agentId) { var clientAccessToken = dc.Table().Find(agentId).ClientAccessToken; @@ -26,7 +26,7 @@ namespace BotSharp.Core.Agents return rasa.agent; } - public static Agent LoadAgent(this IBotEngine engine, Database dc, AIConfiguration aiConfig) + public static Agent LoadAgent(this IBotPlatform engine, Database dc, AIConfiguration aiConfig) { return dc.Table() .Include(x => x.Intents).ThenInclude(x => x.Contexts) @@ -40,7 +40,7 @@ namespace BotSharp.Core.Agents /// /// /// - public static Agent RestoreAgent(this IBotEngine engine, IAgentImporter importer, String agentId, string dataDir) + public static Agent RestoreAgent(this IBotPlatform engine, IAgentImporter importer, String agentId, string dataDir) { // Load agent summary var agent = importer.LoadAgent(agentId, dataDir); diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs new file mode 100644 index 00000000..cdee7c8e --- /dev/null +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Text; +using DotNetToolkit; +using EntityFrameworkCore.BootKit; + +namespace BotSharp.Core.Engines +{ + public class BotTrainer + { + private Database dc; + + private string agentId; + + private string config; + + public BotTrainer(string agentId, Database dc, string config = "BotSharpAi") + { + this.dc = dc; + this.agentId = agentId; + this.config = config; + } + + public string Train() + { + // Get NLP Provider + string providerName = Database.Configuration.GetSection($"{config}:Provider").Value; + var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpProvider; + + + + // tokenize + ITokenizer tokenizer; + + INlpProvider nlpProvider; + + return ""; + } + } +} diff --git a/BotSharp.Core/Engines/IBotEngine.cs b/BotSharp.Core/Engines/IBotPlatform.cs similarity index 77% rename from BotSharp.Core/Engines/IBotEngine.cs rename to BotSharp.Core/Engines/IBotPlatform.cs index 77bec918..bba58ca4 100644 --- a/BotSharp.Core/Engines/IBotEngine.cs +++ b/BotSharp.Core/Engines/IBotPlatform.cs @@ -4,7 +4,7 @@ using System.Text; namespace BotSharp.Core.Engines { - public interface IBotEngine + public interface IBotPlatform { } } diff --git a/BotSharp.Core/Engines/INlpProvider.cs b/BotSharp.Core/Engines/INlpProvider.cs new file mode 100644 index 00000000..14cd9008 --- /dev/null +++ b/BotSharp.Core/Engines/INlpProvider.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public interface INlpProvider + { + void LoadModel(); + Object GetDoc(); + } +} diff --git a/BotSharp.Core/Engines/ITokenizer.cs b/BotSharp.Core/Engines/ITokenizer.cs new file mode 100644 index 00000000..82fbcafe --- /dev/null +++ b/BotSharp.Core/Engines/ITokenizer.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + /// + /// + /// + public interface ITokenizer + { + } +} diff --git a/BotSharp.Core/Engines/RasaAi.cs b/BotSharp.Core/Engines/RasaAi.cs index 55f4edeb..06c74f87 100644 --- a/BotSharp.Core/Engines/RasaAi.cs +++ b/BotSharp.Core/Engines/RasaAi.cs @@ -6,6 +6,8 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; +using RestSharp; using System; using System.Collections.Generic; using System.IO; @@ -15,13 +17,12 @@ using System.Text; namespace BotSharp.Core.Engines { /// - /// Rasa nlu 0.11.x + /// Rasa nlu 0.12.x /// - public class RasaAi : IBotEngine + public class RasaAi : IBotPlatform { public Database dc { get; set; } public AIConfiguration AiConfig { get; set; } - public static IConfiguration Configuration { get; set; } public Agent agent { get; set; } @@ -38,5 +39,52 @@ namespace BotSharp.Core.Engines agent = this.LoadAgent(dc, aiConfig); aiConfig.DevMode = agent.DeveloperAccessToken == aiConfig.ClientAccessToken; } + + public string Train() + { + var client = new RestClient($"{Database.Configuration.GetSection("Rasa:Nlu").Value}"); + var rest = new RestRequest("train", Method.POST); + rest.AddQueryParameter("project", agent.Id); + + var corpus = agent.GrabCorpus(dc); + + // remove Default Fallback Intent + corpus.UserSays = corpus.UserSays.Where(x => x.Intent != "Default Fallback Intent").ToList(); + + string json = JsonConvert.SerializeObject(new { rasa_nlu_data = corpus }, + new JsonSerializerSettings + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + NullValueHandling = NullValueHandling.Ignore + }); + +#if RASA_NLU_0_11 + rest.AddParameter("application/json", json, ParameterType.RequestBody); +#else + string trainingConfig = agent.Language == "zh" ? "config_jieba_mitie_sklearn.yml" : "config_mitie_sklearn.yml"; + string body = File.ReadAllText($"{Database.ContentRootPath}{Path.DirectorySeparatorChar}Settings{Path.DirectorySeparatorChar}{trainingConfig}"); + body = $"{body}\r\ndata: {json}"; + rest.AddParameter("application/x-yml", body, ParameterType.RequestBody); +#endif + + var response = client.Execute(rest); + + if (response.IsSuccessful) + { + var result = JObject.Parse(response.Content); + + string modelName = result["info"].Value().Split(": ")[1]; + + return modelName; + } + else + { + var result = JObject.Parse(response.Content); + + Console.WriteLine(result["error"]); + + return String.Empty; + } + } } } diff --git a/BotSharp.Core/Engines/RequestExtension.cs b/BotSharp.Core/Engines/RequestExtension.cs index 7c11981d..eefc64ad 100644 --- a/BotSharp.Core/Engines/RequestExtension.cs +++ b/BotSharp.Core/Engines/RequestExtension.cs @@ -471,53 +471,6 @@ namespace BotSharp.Core.Engines return aiResponse; } - public static string Train(this RasaAi console, Database dc) - { - var client = new RestClient($"{Database.Configuration.GetSection("Rasa:Nlu").Value}"); - var rest = new RestRequest("train", Method.POST); - rest.AddQueryParameter("project", console.agent.Id); - - var corpus = console.agent.GrabCorpus(dc); - - // remove Default Fallback Intent - corpus.UserSays = corpus.UserSays.Where(x => x.Intent != "Default Fallback Intent").ToList(); - - string json = JsonConvert.SerializeObject(new { rasa_nlu_data = corpus }, - new JsonSerializerSettings - { - ContractResolver = new CamelCasePropertyNamesContractResolver(), - NullValueHandling = NullValueHandling.Ignore - }); - -#if RASA_NLU_0_11 - rest.AddParameter("application/json", json, ParameterType.RequestBody); -#else - string trainingConfig = console.agent.Language == "zh" ? "config_jieba_mitie_sklearn.yml" : "config_mitie_sklearn.yml"; - string body = File.ReadAllText($"{Database.ContentRootPath}{Path.DirectorySeparatorChar}Settings{Path.DirectorySeparatorChar}{trainingConfig}"); - body = $"{body}\r\ndata: {json}"; - rest.AddParameter("application/x-yml", body, ParameterType.RequestBody); -#endif - - var response = client.Execute(rest); - - if (response.IsSuccessful) - { - var result = JObject.Parse(response.Content); - - string modelName = result["info"].Value().Split(": ")[1]; - - return modelName; - } - else - { - var result = JObject.Parse(response.Content); - - Console.WriteLine(result["error"]); - - return String.Empty; - } - } - /// /// Need two categories at least /// diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs new file mode 100644 index 00000000..a6e27937 --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -0,0 +1,24 @@ +using BotSharp.Core.Utilities; +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; + +namespace BotSharp.Core.Engines.SpaCy +{ + public class SpaCyProvider : INlpProvider + { + public async void LoadModel() + { + using (var client = new HttpClient()) + { + var response = await client.PostAsync("", new JsonContent(new { })); + } + } + + public object GetDoc() + { + throw new NotImplementedException(); + } + } +} diff --git a/BotSharp.Core/Intents/IntentDriver.cs b/BotSharp.Core/Intents/IntentDriver.cs index 48e9b780..7e9700cb 100644 --- a/BotSharp.Core/Intents/IntentDriver.cs +++ b/BotSharp.Core/Intents/IntentDriver.cs @@ -11,7 +11,7 @@ namespace BotSharp.Core.Intents { public static class IntentDriver { - public static Intent GetIntent(this IBotEngine bot, Database dc, string intentId) + public static Intent GetIntent(this IBotPlatform bot, Database dc, string intentId) { var intent = dc.Table() .Include(x => x.Contexts) diff --git a/BotSharp.Core/Utilities/JsonContent.cs b/BotSharp.Core/Utilities/JsonContent.cs new file mode 100644 index 00000000..c1983806 --- /dev/null +++ b/BotSharp.Core/Utilities/JsonContent.cs @@ -0,0 +1,16 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; + +namespace BotSharp.Core.Utilities +{ + public class JsonContent : StringContent + { + public JsonContent(object obj) : + base(JsonConvert.SerializeObject(obj), Encoding.UTF8, "application/json") + { + } + } +} diff --git a/BotSharp.UnitTest/AgentTest.cs b/BotSharp.UnitTest/AgentTest.cs index 21b25ff0..ad36ab05 100644 --- a/BotSharp.UnitTest/AgentTest.cs +++ b/BotSharp.UnitTest/AgentTest.cs @@ -66,7 +66,7 @@ namespace BotSharp.UnitTest config.SessionId = Guid.NewGuid().ToString(); var rasa = new RasaAi(dc, config); - string msg = rasa.Train(dc); + string msg = rasa.Train(); Assert.IsTrue(!String.IsNullOrEmpty(msg)); } diff --git a/BotSharp.UnitTest/BotTrainerTest.cs b/BotSharp.UnitTest/BotTrainerTest.cs new file mode 100644 index 00000000..098cb3a3 --- /dev/null +++ b/BotSharp.UnitTest/BotTrainerTest.cs @@ -0,0 +1,19 @@ +using BotSharp.Core.Engines; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.UnitTest +{ + [TestClass] + public class BotTrainerTest : TestEssential + { + [TestMethod] + public void TrainingTest() + { + var trainer = new BotTrainer(BOT_ID, dc); + trainer.Train(); + } + } +} diff --git a/BotSharp.UnitTest/Settings/settings.bot.json b/BotSharp.UnitTest/Settings/settings.bot.json index 6f5ed12c..a78d1f72 100644 --- a/BotSharp.UnitTest/Settings/settings.bot.json +++ b/BotSharp.UnitTest/Settings/settings.bot.json @@ -1,5 +1,16 @@ { "Rasa": { "Nlu": "http://localhost:5000" + }, + + "BotSharpAi": { + "Lang": "en", + "Provider": "SpaCyProvider", + "SpaCyProvider": { + "Url": "http://localhost:5005" + }, + "Pipe": [ + + ] } }