From f796df748e5dbe754e474a6ed1861f49a2ae3e69 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 1 Oct 2018 12:15:17 -0500 Subject: [PATCH] training api. --- BotSharp.Core/Abstractions/IBotPlatform.cs | 1 + BotSharp.Core/AgentStorageInRedis.cs | 4 +- BotSharp.Core/Engines/BotSharp/BotSharpAi.cs | 1 + BotSharp.Core/Engines/BotTrainer.cs | 1 + BotSharp.Core/Engines/Rasa/RasaAi.cs | 6 +- BotSharp.Core/PlatformBuilderBase.cs | 49 +++++++++--- .../IPlatformBuilder.cs | 10 ++- .../BotTrainOptions.cs | 2 +- BotSharp.Platform.Models/TrainingEntity.cs | 2 +- .../TrainingEntitySynonym.cs | 13 +++ BotSharp.WebHost/Settings/ArticulateAi.json | 2 + Platform.Articulate/ArticulateAi.cs | 79 +++++++++++++++---- .../Controllers/AgentController.cs | 10 ++- .../Controllers/DomainController.cs | 9 ++- .../Controllers/EntityController.cs | 9 ++- .../Controllers/IntentController.cs | 34 +++++++- .../Controllers/ParseControllercs.cs | 3 + .../Controllers/ScenarioController.cs | 9 ++- .../Controllers/SettingsController.cs | 3 + .../Controllers/TrainController.cs | 17 ++-- 20 files changed, 205 insertions(+), 59 deletions(-) rename {BotSharp.Core/Engines => BotSharp.Platform.Models}/BotTrainOptions.cs (91%) create mode 100644 BotSharp.Platform.Models/TrainingEntitySynonym.cs diff --git a/BotSharp.Core/Abstractions/IBotPlatform.cs b/BotSharp.Core/Abstractions/IBotPlatform.cs index aee495ba..9c4123cf 100644 --- a/BotSharp.Core/Abstractions/IBotPlatform.cs +++ b/BotSharp.Core/Abstractions/IBotPlatform.cs @@ -1,5 +1,6 @@ using BotSharp.Core.Agents; using BotSharp.Core.Models; +using BotSharp.Platform.Models; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; diff --git a/BotSharp.Core/AgentStorageInRedis.cs b/BotSharp.Core/AgentStorageInRedis.cs index 4d00bb7d..f02accd1 100644 --- a/BotSharp.Core/AgentStorageInRedis.cs +++ b/BotSharp.Core/AgentStorageInRedis.cs @@ -14,7 +14,7 @@ namespace BotSharp.Core where TAgent : AgentBase { private static CSRedisClient csredis; - private string prefix = String.Empty; + private static string prefix = String.Empty; public AgentStorageInRedis() { @@ -47,7 +47,7 @@ namespace BotSharp.Core { var agents = new List(); - var keys = csredis.Keys($"{prefix }*"); + var keys = csredis.Keys($"{prefix}*"); foreach (string key in keys) { var data = csredis.Get(key.Substring(prefix.Length)); diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs index 6c8bfbbc..3d6e2341 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Text; using System.Threading.Tasks; using BotSharp.Core.Models; +using BotSharp.Platform.Models; namespace BotSharp.Core.Engines.BotSharp { diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 8b9ada22..650092c1 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.Core.Intents; +using BotSharp.Platform.Models; using DotNetToolkit; using EntityFrameworkCore.BootKit; using Microsoft.EntityFrameworkCore; diff --git a/BotSharp.Core/Engines/Rasa/RasaAi.cs b/BotSharp.Core/Engines/Rasa/RasaAi.cs index 2480252a..350b6a5f 100644 --- a/BotSharp.Core/Engines/Rasa/RasaAi.cs +++ b/BotSharp.Core/Engines/Rasa/RasaAi.cs @@ -175,8 +175,8 @@ namespace BotSharp.Core.Engines }); // set empty synonym to null - data.Entities - .Where(x => x.Synonyms != null) + /*data.Entities + .Where(x => x.Entity != null) .ToList() .ForEach(entity => { @@ -184,7 +184,7 @@ namespace BotSharp.Core.Engines { entity.Synonyms = null; } - }); + });*/ string json = JsonConvert.SerializeObject(new { rasa_nlu_data = data }, new JsonSerializerSettings diff --git a/BotSharp.Core/PlatformBuilderBase.cs b/BotSharp.Core/PlatformBuilderBase.cs index db2e842e..235dc22f 100644 --- a/BotSharp.Core/PlatformBuilderBase.cs +++ b/BotSharp.Core/PlatformBuilderBase.cs @@ -1,43 +1,70 @@ using BotSharp.Platform.Abstraction; using BotSharp.Platform.Models; +using DotNetToolkit; +using Microsoft.Extensions.Configuration; using System; using System.Collections.Generic; +using System.Linq; using System.Text; namespace BotSharp.Core { - public abstract class PlatformBuilderBase - where TStorage : IAgentStorage, new () + public abstract class PlatformBuilderBase where TAgent : AgentBase { - protected static TStorage storage; + public IAgentStorage Storage { get; set; } + + public IConfiguration PlatformConfig { get; set; } - public PlatformBuilderBase() - { - if (storage == null) storage = new TStorage(); - } public List GetAllAgents() { - return storage.Query(); + GetStorage(); + + return Storage.Query(); } public TAgent GetAgentById(string agentId) { - return storage.FetchById(agentId); + GetStorage(); + + return Storage.FetchById(agentId); } public TAgent GetAgentByName(string agentName) { - return storage.FetchByName(agentName); + GetStorage(); + + return Storage.FetchByName(agentName); } public virtual bool SaveAgent(TAgent agent) { + GetStorage(); + // default save agent in FileStorage - storage.Persist(agent); + Storage.Persist(agent); return true; } + + private IAgentStorage GetStorage() + { + if (Storage == null) + { + string storageName = PlatformConfig.GetValue("AgentStorage"); + switch (storageName) + { + case "AgentStorageInRedis": + Storage = Activator.CreateInstance>(); + break; + case "AgentStorageInMemory": + Storage = Activator.CreateInstance>(); + break; + } + } + + return Storage; + } } } diff --git a/BotSharp.Platform.Abstraction/IPlatformBuilder.cs b/BotSharp.Platform.Abstraction/IPlatformBuilder.cs index 216f28c5..3fd8f1b0 100644 --- a/BotSharp.Platform.Abstraction/IPlatformBuilder.cs +++ b/BotSharp.Platform.Abstraction/IPlatformBuilder.cs @@ -10,9 +10,13 @@ namespace BotSharp.Platform.Abstraction /// Platform abstraction /// Implement this interface to build a Chatbot platform /// - public interface IPlatformBuilder - where TStorage : IAgentStorage, new() + public interface IPlatformBuilder { + /// + /// Agent storage + /// + IAgentStorage Storage { get; set; } + /// /// Parse options for the incoming text or voice request from the sender. /// @@ -33,6 +37,6 @@ namespace BotSharp.Platform.Abstraction /// bool SaveAgent(TAgent agent); - bool Train(TrainingCorpus corpus); + Task Train(TAgent agent, TrainingCorpus corpus); } } diff --git a/BotSharp.Core/Engines/BotTrainOptions.cs b/BotSharp.Platform.Models/BotTrainOptions.cs similarity index 91% rename from BotSharp.Core/Engines/BotTrainOptions.cs rename to BotSharp.Platform.Models/BotTrainOptions.cs index 981bdb3f..cd7fce13 100644 --- a/BotSharp.Core/Engines/BotTrainOptions.cs +++ b/BotSharp.Platform.Models/BotTrainOptions.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace BotSharp.Core.Engines +namespace BotSharp.Platform.Models { public class BotTrainOptions { diff --git a/BotSharp.Platform.Models/TrainingEntity.cs b/BotSharp.Platform.Models/TrainingEntity.cs index 259255d2..81d480cd 100644 --- a/BotSharp.Platform.Models/TrainingEntity.cs +++ b/BotSharp.Platform.Models/TrainingEntity.cs @@ -8,6 +8,6 @@ namespace BotSharp.Platform.Models { public virtual String Entity { get; set; } - public List Synonyms { get; set; } + public List Values { get; set; } } } diff --git a/BotSharp.Platform.Models/TrainingEntitySynonym.cs b/BotSharp.Platform.Models/TrainingEntitySynonym.cs new file mode 100644 index 00000000..fb80a09b --- /dev/null +++ b/BotSharp.Platform.Models/TrainingEntitySynonym.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Platform.Models +{ + public class TrainingEntitySynonym + { + public String Value { get; set; } + + public List Synonyms { get; set; } + } +} diff --git a/BotSharp.WebHost/Settings/ArticulateAi.json b/BotSharp.WebHost/Settings/ArticulateAi.json index 4bd4aa87..3380ac68 100644 --- a/BotSharp.WebHost/Settings/ArticulateAi.json +++ b/BotSharp.WebHost/Settings/ArticulateAi.json @@ -2,6 +2,8 @@ "ArticulateAi": { "Lang": "en", + "AgentStorage": "AgentStorageInRedis", + "Provider": "BotSharpProvider", "BotSharpProvider": { }, diff --git a/Platform.Articulate/ArticulateAi.cs b/Platform.Articulate/ArticulateAi.cs index d13d55ec..62057895 100644 --- a/Platform.Articulate/ArticulateAi.cs +++ b/Platform.Articulate/ArticulateAi.cs @@ -1,11 +1,16 @@ using BotSharp.Core; +using BotSharp.Core.Agents; +using BotSharp.Core.Engines; using BotSharp.Platform.Abstraction; using BotSharp.Platform.Models; +using DotNetToolkit; using Platform.Articulate.Models; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; namespace Platform.Articulate { @@ -14,10 +19,10 @@ namespace Platform.Articulate /// http://spg.ai/projects/articulate /// This implementation takes over APIs of Articulate's 7500 port. /// - public class ArticulateAi : - PlatformBuilderBase, - IPlatformBuilder - where TStorage : IAgentStorage, new() + public class ArticulateAi : + PlatformBuilderBase, + IPlatformBuilder + where TAgent : AgentBase { public DialogRequestOptions RequestOptions { get; set; } @@ -80,23 +85,67 @@ namespace Platform.Articulate return intents; } - public TrainingCorpus ExtractorCorpus(TAgent specificAgent) + public TrainingCorpus ExtractorCorpus(TAgent agent) { - var agent1 = specificAgent as AgentModel; - - var standardAgent = new StandardAgent + var corpus = new TrainingCorpus(); + var agt = agent as AgentModel; + corpus.Entities = agt.Entities.Select(x => new TrainingEntity { - Name = agent1.Name, - Language = agent1.Language, - Description = agent1.Description - }; + Entity = x.EntityName, + Values = x.Examples.Select(y => new TrainingEntitySynonym + { + Value = y.Value, + Synonyms = y.Synonyms + }).ToList() + }).ToList(); - return new TrainingCorpus(); + corpus.UserSays = new List>(); + + foreach(DomainModel domain in agt.Domains) + { + foreach(IntentModel intent in domain.Intents) + { + foreach(IntentExampleModel example in intent.Examples) + { + var say = new TrainingIntentExpression() + { + Intent = intent.IntentName, + Text = example.UserSays, + Entities = example.Entities.Select(x => new TrainingIntentExpressionPart + { + Entity = x.Entity, + Start = x.Start, + Value = x.Value + }).ToList() + }; + + corpus.UserSays.Add(say); + } + } + } + + return corpus; } - public bool Train(TrainingCorpus corpus) + public async Task Train(TAgent agent, TrainingCorpus corpus) { - throw new NotImplementedException(); + string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id); + + // save corpus to agent dir + var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id); + var model = "model_" + DateTime.UtcNow.ToString("yyyyMMdd"); + var modelPath = Path.Combine(projectPath, model); + + var trainer = new BotTrainer(); + var parsedAgent = agent.ToObject(); + + var info = await trainer.Train(parsedAgent, new BotTrainOptions + { + AgentDir = projectPath, + Model = model + }); + + return true; } } } diff --git a/Platform.Articulate/Controllers/AgentController.cs b/Platform.Articulate/Controllers/AgentController.cs index 75310101..03a93af0 100644 --- a/Platform.Articulate/Controllers/AgentController.cs +++ b/Platform.Articulate/Controllers/AgentController.cs @@ -4,6 +4,7 @@ using BotSharp.Core.Engines; using BotSharp.Platform.Abstraction; using BotSharp.Platform.Models; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Platform.Articulate; using Platform.Articulate.Models; @@ -20,17 +21,19 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class AgentController : ControllerBase { + private readonly IConfiguration configuration; private readonly IBotPlatform _platform; - private ArticulateAi, AgentModel> builder; + private ArticulateAi builder; /// /// Initialize agent controller and get a platform instance /// /// - public AgentController(IBotPlatform platform) + public AgentController(IBotPlatform platform, IConfiguration configuration) { _platform = platform; - builder = new ArticulateAi, AgentModel>(); + builder = new ArticulateAi(); + builder.PlatformConfig = configuration.GetSection("ArticulateAi"); } [HttpPost] @@ -45,7 +48,6 @@ namespace Platform.Articulate.Controllers } // convert to standard Agent structure - var builder = new ArticulateAi, AgentModel>(); agent.Id = Guid.NewGuid().ToString(); agent.Name = agent.AgentName; builder.SaveAgent(agent); diff --git a/Platform.Articulate/Controllers/DomainController.cs b/Platform.Articulate/Controllers/DomainController.cs index 4115d66a..83df0437 100644 --- a/Platform.Articulate/Controllers/DomainController.cs +++ b/Platform.Articulate/Controllers/DomainController.cs @@ -1,6 +1,7 @@ using BotSharp.Core; using BotSharp.Platform.Models; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Platform.Articulate.Models; @@ -18,11 +19,13 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class DomainController : ControllerBase { - private ArticulateAi, AgentModel> builder; + private readonly IConfiguration configuration; + private ArticulateAi builder; - public DomainController() + public DomainController(IConfiguration configuration) { - builder = new ArticulateAi, AgentModel>(); + builder = new ArticulateAi(); + builder.PlatformConfig = configuration.GetSection("ArticulateAi"); } [HttpGet("{domainId}")] diff --git a/Platform.Articulate/Controllers/EntityController.cs b/Platform.Articulate/Controllers/EntityController.cs index c9bcb08c..d31e1137 100644 --- a/Platform.Articulate/Controllers/EntityController.cs +++ b/Platform.Articulate/Controllers/EntityController.cs @@ -1,6 +1,7 @@ using BotSharp.Core; using BotSharp.Platform.Models; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Platform.Articulate.Models; using Platform.Articulate.ViewModels; @@ -16,11 +17,13 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class EntityController : ControllerBase { - private ArticulateAi, AgentModel> builder; + private readonly IConfiguration configuration; + private ArticulateAi builder; - public EntityController() + public EntityController(IConfiguration configuration) { - builder = new ArticulateAi, AgentModel>(); + builder = new ArticulateAi(); + builder.PlatformConfig = configuration.GetSection("ArticulateAi"); } [HttpGet("{entityId}")] diff --git a/Platform.Articulate/Controllers/IntentController.cs b/Platform.Articulate/Controllers/IntentController.cs index 305c2d00..2e12db68 100644 --- a/Platform.Articulate/Controllers/IntentController.cs +++ b/Platform.Articulate/Controllers/IntentController.cs @@ -2,6 +2,7 @@ using BotSharp.Platform.Models; using DotNetToolkit; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Platform.Articulate.Models; using Platform.Articulate.ViewModels; @@ -17,11 +18,13 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class IntentController : ControllerBase { - private ArticulateAi, AgentModel> builder; + private readonly IConfiguration configuration; + private ArticulateAi builder; - public IntentController() + public IntentController(IConfiguration configuration) { - builder = new ArticulateAi, AgentModel>(); + builder = new ArticulateAi(); + builder.PlatformConfig = configuration.GetSection("ArticulateAi"); } [HttpGet("{intentId}")] @@ -52,6 +55,31 @@ namespace Platform.Articulate.Controllers return intent; } + [HttpPut("{intentId}")] + public IntentViewModel PutIntent([FromRoute] string intentId) + { + IntentViewModel intent = null; + + using (var reader = new StreamReader(Request.Body)) + { + string body = reader.ReadToEnd(); + intent = JsonConvert.DeserializeObject(body); + } + + var agent = builder.GetAgentByIntentId(intentId); + + var updateAgent = agent.Item1; + var updateIntents = updateAgent.Domains.First(x => x.Id == agent.Item2.Id).Intents; + var updateIntent = updateIntents.First(x => x.Id == agent.Item3.Id); + + updateIntent.IntentName = intent.IntentName; + updateIntent.Examples = intent.Examples; + + builder.SaveAgent(updateAgent); + + return intent; + } + [HttpGet("{intentId}/webhook")] public void GetIntentWebhook([FromRoute] string intentId) { diff --git a/Platform.Articulate/Controllers/ParseControllercs.cs b/Platform.Articulate/Controllers/ParseControllercs.cs index b30880ca..1e93215d 100644 --- a/Platform.Articulate/Controllers/ParseControllercs.cs +++ b/Platform.Articulate/Controllers/ParseControllercs.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using System; using System.Collections.Generic; using System.Text; @@ -9,6 +10,8 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class ParseControllercs : ControllerBase { + private readonly IConfiguration configuration; + [HttpGet("/agent/{agentId}/converse")] public void ParseText([FromRoute] string agentId, [FromQuery] string text, [FromQuery] string sessionId) { diff --git a/Platform.Articulate/Controllers/ScenarioController.cs b/Platform.Articulate/Controllers/ScenarioController.cs index 3b959063..4b3ad548 100644 --- a/Platform.Articulate/Controllers/ScenarioController.cs +++ b/Platform.Articulate/Controllers/ScenarioController.cs @@ -1,6 +1,7 @@ using BotSharp.Core; using DotNetToolkit; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Platform.Articulate.Models; using System; @@ -15,11 +16,13 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class ScenarioController : ControllerBase { - private ArticulateAi, AgentModel> builder; + private readonly IConfiguration configuration; + private ArticulateAi builder; - public ScenarioController() + public ScenarioController(IConfiguration configuration) { - builder = new ArticulateAi, AgentModel>(); + builder = new ArticulateAi(); + builder.PlatformConfig = configuration.GetSection("ArticulateAi"); } [HttpGet("/intent/{intentId}/scenario")] diff --git a/Platform.Articulate/Controllers/SettingsController.cs b/Platform.Articulate/Controllers/SettingsController.cs index 6608ffe0..7c3ec6de 100644 --- a/Platform.Articulate/Controllers/SettingsController.cs +++ b/Platform.Articulate/Controllers/SettingsController.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Platform.Articulate.Models; using System; @@ -12,6 +13,8 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class SettingsController : ControllerBase { + private readonly IConfiguration configuration; + [HttpGet] public SettingsModel GetSettings() { diff --git a/Platform.Articulate/Controllers/TrainController.cs b/Platform.Articulate/Controllers/TrainController.cs index 13ade32d..9db75630 100644 --- a/Platform.Articulate/Controllers/TrainController.cs +++ b/Platform.Articulate/Controllers/TrainController.cs @@ -1,9 +1,11 @@ using BotSharp.Core; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Platform.Articulate.Models; using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; namespace Platform.Articulate.Controllers { @@ -11,20 +13,21 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class TrainController : ControllerBase { - private ArticulateAi, AgentModel> builder; + private readonly IConfiguration configuration; + private ArticulateAi builder; - public TrainController() + public TrainController(IConfiguration configuration) { - builder = new ArticulateAi, AgentModel>(); + builder = new ArticulateAi(); + builder.PlatformConfig = configuration.GetSection("ArticulateAi"); } [HttpGet("/agent/{agentId}/train")] - public AgentModel TrainAgent([FromRoute] string agentId) + public async Task TrainAgent([FromRoute] string agentId) { var agent = builder.GetAgentById(agentId); - - - + var corpus = builder.ExtractorCorpus(agent); + await builder.Train(agent, corpus); agent.Status = "Ready"; return agent;