From 867ebc71f95ab5c7914b2086235a35b6d6451f42 Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Tue, 28 Aug 2018 09:58:35 -0500 Subject: [PATCH] refactor training. --- BotSharp.Core.UnitTest/BotTrainerTest.cs | 4 +- BotSharp.Core/Abstractions/IAgentImporter.cs | 2 +- BotSharp.Core/Abstractions/IBotPlatform.cs | 10 +- BotSharp.Core/Engines/BotEngineBase.cs | 106 +++++++++++++++++- BotSharp.Core/Engines/BotSharp/BotSharpAi.cs | 6 +- BotSharp.Core/Engines/BotTrainOptions.cs | 5 + BotSharp.Core/Engines/BotTrainer.cs | 9 -- .../Dialogflow/AgentImporterInDialogflow.cs | 7 +- .../Engines/Rasa/AgentImporterInRasa.cs | 4 +- .../Engines/Sebis/AgentImporterInSebis.cs | 4 +- BotSharp.RestApi/AgentController.cs | 8 +- BotSharp.RestApi/Rasa/ParseController.cs | 2 +- BotSharp.RestApi/Rasa/TrainController.cs | 2 +- 13 files changed, 130 insertions(+), 39 deletions(-) diff --git a/BotSharp.Core.UnitTest/BotTrainerTest.cs b/BotSharp.Core.UnitTest/BotTrainerTest.cs index f5954101..2f232448 100644 --- a/BotSharp.Core.UnitTest/BotTrainerTest.cs +++ b/BotSharp.Core.UnitTest/BotTrainerTest.cs @@ -12,11 +12,11 @@ namespace BotSharp.Core.UnitTest public class BotTrainerTest : TestEssential { [TestMethod] - public void TrainingTest() + public async void TrainingTest() { var ai = new BotSharpAi(); ai.LoadAgent(BOT_ID); - ai.Train(); + await ai.Train(new BotTrainOptions { }); } } } diff --git a/BotSharp.Core/Abstractions/IAgentImporter.cs b/BotSharp.Core/Abstractions/IAgentImporter.cs index 42540d54..5fecc47e 100644 --- a/BotSharp.Core/Abstractions/IAgentImporter.cs +++ b/BotSharp.Core/Abstractions/IAgentImporter.cs @@ -16,7 +16,7 @@ namespace BotSharp.Core.Engines /// Load agent summary /// /// - Agent LoadAgent(); + Agent LoadAgent(AgentImportHeader agentHeader); /// /// Load user customized entity type which defined in dictionary diff --git a/BotSharp.Core/Abstractions/IBotPlatform.cs b/BotSharp.Core/Abstractions/IBotPlatform.cs index cc2a4b45..ddbd4ec4 100644 --- a/BotSharp.Core/Abstractions/IBotPlatform.cs +++ b/BotSharp.Core/Abstractions/IBotPlatform.cs @@ -17,10 +17,16 @@ namespace BotSharp.Core.Engines /// Agent LoadAgent(string id); - Agent LoadAgentFromFile(string dataDir) where TAgentImporter : IAgentImporter, new(); + /// + /// Load agent from files. + /// There must contain a meta.json + /// + /// + /// + Agent LoadAgentFromFile(string dataDir); AIResponse TextRequest(AIRequest request); - Task Train(); + Task Train(BotTrainOptions options); } } diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index bd0f387c..27315c48 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Agents; +using BotSharp.Core.Engines.Rasa; using BotSharp.Core.Entities; using BotSharp.Core.Intents; using BotSharp.Core.Models; @@ -93,20 +94,37 @@ namespace BotSharp.Core.Engines string dataDir = Path.Combine(DbInitializerPath, "Agents"); int row = dc.DbTran(() => { - LoadAgentFromFile(dataDir); + LoadAgentFromFile(dataDir); SaveAgent(); }); return row > 0; } - public Agent LoadAgentFromFile(string dataDir) where TAgentImporter : IAgentImporter, new() + public Agent LoadAgentFromFile(string dataDir) { - var importer = new TAgentImporter(); + var meta = LoadMeta(dataDir); + IAgentImporter importer = null; + + switch (meta.Platform) + { + case "Dialogflow": + importer = new AgentImporterInDialogflow(); + break; + case "Rasa": + importer = new AgentImporterInRasa(); + break; + case "Sebis": + importer = new AgentImporterInSebis(); + break; + default: + break; + } + importer.AgentDir = dataDir; // Load agent summary - agent = importer.LoadAgent(); + agent = importer.LoadAgent(meta); // Load user custom entities importer.LoadCustomEntities(agent); @@ -123,6 +141,14 @@ namespace BotSharp.Core.Engines return agent; } + private AgentImportHeader LoadMeta(string dataDir) + { + // load meta + string metaJson = File.ReadAllText(Path.Combine(dataDir, "meta.json")); + + return JsonConvert.DeserializeObject(metaJson); + } + public String SaveAgent() { var existedAgent = dc.Table().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name); @@ -138,6 +164,76 @@ namespace BotSharp.Core.Engines } } + public TrainingCorpus GetIntentExpressions(Agent agent) + { + TrainingCorpus corpus = new TrainingCorpus() + { + UserSays = new List>(), + Entities = new List() + }; + + var expressParts = new List(); + + var intents = agent.Intents; + + intents.ForEach(intent => + { + intent.UserSays.ForEach(exp => + { + exp.Data = exp.Data.OrderBy(x => x.UpdatedTime).ToList(); + + var say = new TrainingIntentExpression + { + Intent = intent.Name, + Text = String.Join("", exp.Data.Select(x => x.Text)), + ContextHash = intent.ContextHash + }; + + // convert entity format + exp.Data.Where(x => !String.IsNullOrEmpty(x.Meta)) + .ToList() + .ForEach(x => + { + var part = new TrainingIntentExpressionPart + { + Value = x.Text, + Entity = $"{x.Meta}:{x.Alias}", + Start = x.Start + }; + + if (say.Entities == null) say.Entities = new List(); + say.Entities.Add(part); + + // assemble entity synonmus + /*if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text)) + { + var allSynonyms = (from e in dc.Table() + join ee in dc.Table() on e.Id equals ee.EntityId + join ees in dc.Table() on ee.Id equals ees.EntityEntryId + where e.Name == x.Alias && ee.Value == x.Text & ees.Synonym != x.Text + select ees.Synonym).ToList(); + + var te = new TrainingEntity + { + EntityType = $"{x.Meta}:{x.Alias}", + EntityValue = x.Text, + Synonyms = allSynonyms + }; + + trainingData.Entities.Add(te); + }*/ + }); + + corpus.UserSays.Add(say); + }); + }); + + // remove Default Fallback Intent + corpus.UserSays = corpus.UserSays.Where(x => x.Intent != "Default Fallback Intent").ToList(); + + return corpus; + } + public TrainingCorpus GetIntentExpressions() { TrainingCorpus corpus = new TrainingCorpus() @@ -212,7 +308,7 @@ namespace BotSharp.Core.Engines return corpus; } - public virtual Task Train() + public virtual Task Train(BotTrainOptions options) { return Task.CompletedTask; } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs index 71821fc0..6c8bfbbc 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs @@ -8,11 +8,11 @@ namespace BotSharp.Core.Engines.BotSharp { public class BotSharpAi : BotEngineBase, IBotPlatform { - public override async Task Train() + public override async Task Train(BotTrainOptions options) { - agent.Corpus = GetIntentExpressions(); + agent.Corpus = GetIntentExpressions(agent); var trainer = new BotTrainer(agent.Id, dc); - await trainer.Train(agent, new BotTrainOptions { }); + await trainer.Train(agent, options); } } } diff --git a/BotSharp.Core/Engines/BotTrainOptions.cs b/BotSharp.Core/Engines/BotTrainOptions.cs index 75ad6a48..981bdb3f 100644 --- a/BotSharp.Core/Engines/BotTrainOptions.cs +++ b/BotSharp.Core/Engines/BotTrainOptions.cs @@ -6,6 +6,11 @@ namespace BotSharp.Core.Engines { public class BotTrainOptions { + /// + /// Agent data direcotry + /// + public string AgentDir { get; set; } + /// /// Model Name /// diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 02089825..cdf5d954 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -35,15 +35,6 @@ namespace BotSharp.Core.Engines public async Task Train(Agent agent, BotTrainOptions options) { - /*agent.Intents = dc.Table() - .Include(x => x.Contexts) - .Include(x => x.Responses).ThenInclude(x => x.Contexts) - .Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts) - .Include(x => x.Responses).ThenInclude(x => x.Messages) - .Include(x => x.UserSays).ThenInclude(x => x.Data) - .Where(x => x.AgentId == agentId) - .ToList();*/ - var data = new NlpDoc(); // Get NLP Provider diff --git a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs index d70eca76..3470b533 100644 --- a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs +++ b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs @@ -27,13 +27,8 @@ namespace BotSharp.Core.Engines /// /// /// - public Agent LoadAgent() + public Agent LoadAgent(AgentImportHeader agentHeader) { - // load meta - string metaJson = File.ReadAllText(Path.Combine(AgentDir, "meta.json")); - - AgentImportHeader agentHeader = JsonConvert.DeserializeObject(metaJson); - // load agent profile string data = File.ReadAllText(Path.Combine(AgentDir, "agent.json")); var agent = JsonConvert.DeserializeObject(data); diff --git a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs index 6ee9d4c8..f3811c40 100644 --- a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs +++ b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs @@ -16,10 +16,8 @@ namespace BotSharp.Core.Engines.Rasa { public string AgentDir { get; set; } - public Agent LoadAgent() + public Agent LoadAgent(AgentImportHeader agentHeader) { - AgentImportHeader agentHeader = null; - var agent = new Agent(); agent.ClientAccessToken = Guid.NewGuid().ToString("N"); agent.DeveloperAccessToken = Guid.NewGuid().ToString("N"); diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index 1338361c..4ee9b6d6 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -27,10 +27,8 @@ namespace BotSharp.Core.Engines /// /// /// - public Agent LoadAgent() + public Agent LoadAgent(AgentImportHeader agentHeader) { - AgentImportHeader agentHeader = null; - // load agent profile string data = File.ReadAllText(Path.Combine(AgentDir, "Sebis", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json")); var agent = JsonConvert.DeserializeObject(data); diff --git a/BotSharp.RestApi/AgentController.cs b/BotSharp.RestApi/AgentController.cs index 41f0e6b5..573d552c 100644 --- a/BotSharp.RestApi/AgentController.cs +++ b/BotSharp.RestApi/AgentController.cs @@ -68,7 +68,7 @@ namespace BotSharp.RestApi System.IO.File.Delete(filePath); - var agent = _platform.LoadAgentFromFile(dest); + var agent = _platform.LoadAgentFromFile(dest); return Ok(agent.Id); } @@ -81,8 +81,10 @@ namespace BotSharp.RestApi [HttpGet("{agentId}")] public string Train([FromRoute] String agentId) { - _platform.LoadAgent(agentId); - _platform.Train(); + string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agentId); + string dest = Directory.GetDirectories(agentDir).Last(); + var agent = _platform.LoadAgentFromFile(dest); + _platform.Train(new BotTrainOptions { AgentDir = agentDir, Model = dest.Split(Path.DirectorySeparatorChar).Last() }); return ""; } diff --git a/BotSharp.RestApi/Rasa/ParseController.cs b/BotSharp.RestApi/Rasa/ParseController.cs index e073a1e6..b5900968 100644 --- a/BotSharp.RestApi/Rasa/ParseController.cs +++ b/BotSharp.RestApi/Rasa/ParseController.cs @@ -52,7 +52,7 @@ namespace BotSharp.RestApi.Rasa var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project); var modelPath = Path.Combine(projectPath, request.Model); - _platform.LoadAgentFromFile(modelPath); + _platform.LoadAgentFromFile(modelPath); var aIResponse = _platform.TextRequest(new AIRequest { diff --git a/BotSharp.RestApi/Rasa/TrainController.cs b/BotSharp.RestApi/Rasa/TrainController.cs index 68dbb44d..e7054375 100644 --- a/BotSharp.RestApi/Rasa/TrainController.cs +++ b/BotSharp.RestApi/Rasa/TrainController.cs @@ -83,7 +83,7 @@ namespace BotSharp.RestApi.Rasa ContractResolver = new CamelCasePropertyNamesContractResolver() })); - var agent = _platform.LoadAgentFromFile(modelPath); + var agent = _platform.LoadAgentFromFile(modelPath); var info = await trainer.Train(agent, new BotTrainOptions { Model = request.Model });