From 16ab106ce72e45c98864904e7ca8b336771adba9 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 22 Aug 2018 23:24:13 -0500 Subject: [PATCH] Fix traing issue using Rasa UI. --- BotSharp.Core/Engines/BotSharp/BotSharpAi.cs | 2 +- BotSharp.Core/Engines/BotTrainOptions.cs | 14 +++++ BotSharp.Core/Engines/BotTrainer.cs | 4 +- BotSharp.RestApi/Rasa/ParseController.cs | 27 ++++++++-- .../Rasa/RasaTrainRequestModel.cs | 2 + .../Rasa/RasaUiMiddlewareRequestDataModel.cs | 13 ----- .../Rasa/RasaUiMiddlewareRequestModel.cs | 21 -------- BotSharp.RestApi/Rasa/StatusController.cs | 53 +++++++++++++++++-- BotSharp.RestApi/Rasa/TrainController.cs | 33 ++++++++++-- 9 files changed, 117 insertions(+), 52 deletions(-) create mode 100644 BotSharp.Core/Engines/BotTrainOptions.cs delete mode 100644 BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestDataModel.cs delete mode 100644 BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestModel.cs diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs index 4ea57d6f..71821fc0 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpAi.cs @@ -12,7 +12,7 @@ namespace BotSharp.Core.Engines.BotSharp { agent.Corpus = GetIntentExpressions(); var trainer = new BotTrainer(agent.Id, dc); - await trainer.Train(agent); + await trainer.Train(agent, new BotTrainOptions { }); } } } diff --git a/BotSharp.Core/Engines/BotTrainOptions.cs b/BotSharp.Core/Engines/BotTrainOptions.cs new file mode 100644 index 00000000..75ad6a48 --- /dev/null +++ b/BotSharp.Core/Engines/BotTrainOptions.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public class BotTrainOptions + { + /// + /// Model Name + /// + public string Model { get; set; } + } +} diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index b092ed41..02089825 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -33,7 +33,7 @@ namespace BotSharp.Core.Engines this.agentId = agentId; } - public async Task Train(Agent agent) + public async Task Train(Agent agent, BotTrainOptions options) { /*agent.Intents = dc.Table() .Include(x => x.Contexts) @@ -70,7 +70,7 @@ namespace BotSharp.Core.Engines AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") }; - settings.ModelDir = Path.Combine(settings.ProjectDir, "model" + DateTime.UtcNow.ToString("MMddyyyyHHmm")); + settings.ModelDir = Path.Combine(settings.ProjectDir, String.IsNullOrEmpty(options.Model) ? "model" + DateTime.UtcNow.ToString("MMddyyyyHHmm") : options.Model); if (!Directory.Exists(settings.ProjectDir)) { diff --git a/BotSharp.RestApi/Rasa/ParseController.cs b/BotSharp.RestApi/Rasa/ParseController.cs index 56ab6d1f..81e6a9d5 100644 --- a/BotSharp.RestApi/Rasa/ParseController.cs +++ b/BotSharp.RestApi/Rasa/ParseController.cs @@ -2,13 +2,18 @@ using BotSharp.Core.Models; using BotSharp.NLP; using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json; using System; using System.Collections.Generic; +using System.IO; using System.Text; namespace BotSharp.RestApi.Rasa { #if RASA_UI + /// + /// send a text request + /// [Route("[controller]")] public class ParseController : ControllerBase { @@ -23,14 +28,26 @@ namespace BotSharp.RestApi.Rasa _platform = platform; } - [HttpPost] - public ActionResult Parse(RasaRequestModel request) + /// + /// parse request + /// + /// + /// + [HttpPost, HttpGet] + public ActionResult Parse() { String clientAccessToken = Request.Headers["ClientAccessToken"]; - var config = new AIConfiguration(clientAccessToken, SupportedLanguage.English); + var config = new AIConfiguration("", SupportedLanguage.English); config.SessionId = "rasa nlu"; - _platform.LoadAgent(clientAccessToken); + string body = ""; + using (var reader = new StreamReader(Request.Body)) + { + body = reader.ReadToEnd(); + } + var request = JsonConvert.DeserializeObject(body); + + //_platform.LoadAgent(clientAccessToken); var aIResponse = _platform.TextRequest(new AIRequest { @@ -48,7 +65,7 @@ namespace BotSharp.RestApi.Rasa { }, - Text = request.Text + Text = "" }; } } diff --git a/BotSharp.RestApi/Rasa/RasaTrainRequestModel.cs b/BotSharp.RestApi/Rasa/RasaTrainRequestModel.cs index e2213eb9..5fd8b59e 100644 --- a/BotSharp.RestApi/Rasa/RasaTrainRequestModel.cs +++ b/BotSharp.RestApi/Rasa/RasaTrainRequestModel.cs @@ -10,6 +10,8 @@ namespace BotSharp.RestApi.Rasa { public string Project { get; set; } + public string Model { get; set; } + [JsonProperty("rasa_nlu_data")] public RasaTrainingData Corpus { get; set; } } diff --git a/BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestDataModel.cs b/BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestDataModel.cs deleted file mode 100644 index e0d5cb3e..00000000 --- a/BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestDataModel.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.RestApi.Rasa -{ - public class RasaUiMiddlewareRequestDataModel - { - public string Project { get; set; } - public string Agent { get; set; } - public RasaTrainRequestModel Data { get; set; } - } -} diff --git a/BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestModel.cs b/BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestModel.cs deleted file mode 100644 index e061e6d5..00000000 --- a/BotSharp.RestApi/Rasa/RasaUiMiddlewareRequestModel.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.RestApi.Rasa -{ - public class RasaUiMiddlewareRequestModel - { - [JsonProperty("ip_address")] - public string IP { get; set; } - - public string Query { get; set; } - - [JsonProperty("event_type")] - public string EventType { get; set; } - - /*[JsonProperty("event_data")] - public T EventData { get; set; }*/ - } -} diff --git a/BotSharp.RestApi/Rasa/StatusController.cs b/BotSharp.RestApi/Rasa/StatusController.cs index 0c2d657c..67e5b034 100644 --- a/BotSharp.RestApi/Rasa/StatusController.cs +++ b/BotSharp.RestApi/Rasa/StatusController.cs @@ -1,24 +1,67 @@ -using Microsoft.AspNetCore.Mvc; +using BotSharp.Core.Engines; +using Microsoft.AspNetCore.Mvc; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Text; namespace BotSharp.RestApi.Rasa { #if RASA_UI + /// + /// This returns all the currently available projects. + /// [Route("[controller]")] public class StatusController : ControllerBase { + private readonly IBotPlatform _platform; + + /// + /// Initialize status controller and get a platform instance + /// + /// + public StatusController(IBotPlatform platform) + { + _platform = platform; + } + + /// + /// Returns a list of available projects the server can use to fulfill /parse requests. + /// + /// [HttpGet] public ActionResult Get() { var status = new RasaStatusModel(); - status.AvailableProjects = JObject.FromObject(new RasaProjectModel + status.AvailableProjects = JObject.FromObject(new { }); + + // scan dir, get all models + var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects"); + var projectDirs = Directory.GetDirectories(projectPath); + for(int idx = 0; idx < projectDirs.Length; idx++) { - Status = "ready", - AvailableModels = new List { "" } - }); + string project = projectDirs[idx].Split('\\').Last(); + var modelDirs = Directory.GetDirectories(projectDirs[idx]); + + List availableModels = new List(); + + for (int mIdx = 0; mIdx < modelDirs.Length; mIdx++) + { + string model = modelDirs[mIdx].Split('\\').Last(); + if (model.StartsWith(project + "_")) + { + availableModels.Add(model); + } + } + + status.AvailableProjects.Add(project, JObject.FromObject(new RasaProjectModel + { + Status = "ready", + AvailableModels = availableModels + })); + } return Ok(status); } diff --git a/BotSharp.RestApi/Rasa/TrainController.cs b/BotSharp.RestApi/Rasa/TrainController.cs index ba936aea..244b9a5e 100644 --- a/BotSharp.RestApi/Rasa/TrainController.cs +++ b/BotSharp.RestApi/Rasa/TrainController.cs @@ -15,6 +15,10 @@ using System.Threading.Tasks; namespace BotSharp.RestApi.Rasa { #if RASA_UI + /// + /// You can post your training data to this endpoint to train a new model for a project. + /// This request will wait for the server answer: either the model was trained successfully or the training exited with an error. + /// [Route("[controller]")] public class TrainController : ControllerBase { @@ -29,12 +33,29 @@ namespace BotSharp.RestApi.Rasa _platform = platform; } + /// + /// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. + /// + /// Model name + /// + /// [HttpPost] - public async Task> Train(RasaUiMiddlewareRequestModel request) + public async Task> Train([FromQuery] string agent, [FromQuery] string project) { - return Ok(); + string body = ""; + using (var reader = new StreamReader(Request.Body)) + { + body = reader.ReadToEnd(); + } + + var rasa_nlu_data = JsonConvert.DeserializeObject(body); + rasa_nlu_data.Model = agent; + var trainResult = await Train(rasa_nlu_data, project); + + return trainResult; } - /*public async Task> Train([FromBody] RasaTrainRequestModel request, [FromQuery] string project) + + private async Task> Train([FromBody] RasaTrainRequestModel request, [FromQuery] string project) { var trainer = new BotTrainer(); if (String.IsNullOrEmpty(request.Project)) @@ -52,6 +73,8 @@ namespace BotSharp.RestApi.Rasa Directory.CreateDirectory(agentPath); } + // Save raw data to file, then parse it to Agent instance. + // in order to unify the process. var fileName = Path.Combine(agentPath, "corpus.json"); System.IO.File.WriteAllText(fileName, JsonConvert.SerializeObject(request.Corpus, new JsonSerializerSettings @@ -69,10 +92,10 @@ namespace BotSharp.RestApi.Rasa Name = project }); - var info = await trainer.Train(agent); + var info = await trainer.Train(agent, new BotTrainOptions { Model = request.Model }); return Ok(new { info = info.Model }); - }*/ + } } #endif }