using BotSharp.Core.Engines; using BotSharp.Platform.Abstractions; using BotSharp.Platform.Models; using BotSharp.Platform.Models.MachineLearning; using BotSharp.Platform.Rasa.Models; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace BotSharp.Platform.Rasa.Controllers { /// /// You can post your training data to this endpoint to train a new model for a project. /// This request will wait for the server answer: either the model was trained successfully or the training exited with an error. /// [Route("[controller]")] public class TrainController : ControllerBase { private RasaAi builder; private readonly IPlatformSettings settings; public TrainController(RasaAi configuration, IPlatformSettings settings) { builder = configuration; this.settings = settings; } /// /// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. /// /// Model name /// Agent name or agent id /// [HttpPost] public async Task> Train([FromQuery] string project, [FromQuery] string model) { string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project); if (!Directory.Exists(agentDir)) { Directory.CreateDirectory(agentDir); } string body = ""; using (var reader = new StreamReader(Request.Body)) { body = reader.ReadToEnd(); } var agent = await ImportAgent(project, body); var corpus = await builder.ExtractorCorpus(agent); var meta = await builder.Train(agent, corpus, new BotTrainOptions { Model = model }); return meta; } private async Task ImportAgent(string project, string body) { Console.WriteLine($"Update agent from http post, data length: {body.Length}"); // save to file // save corpus to agent dir var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project); var rawPath = Path.Combine(projectPath, "tmp"); // clear tmp dir if (Directory.Exists(rawPath)) { Directory.Delete(rawPath, true); } Directory.CreateDirectory(rawPath); // Save raw data to file, then parse it to Agent instance. var metaFileName = Path.Combine(rawPath, "meta.json"); System.IO.File.WriteAllText(metaFileName, JsonConvert.SerializeObject(new AgentImportHeader { Name = project, Platform = PlatformType.Rasa, Id = Guid.NewGuid().ToString() }, new JsonSerializerSettings { Formatting = Formatting.Indented, NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver() })); // in order to unify the process. var fileName = Path.Combine(rawPath, "corpus.json"); System.IO.File.WriteAllText(fileName, body); /*string lang = Regex.Match(body, @"language:.+")?.Value; if (!String.IsNullOrEmpty(lang)) { lang = lang.Substring(11, 2); } string data = Regex.Match(body, @"data:([\s\S]*)")?.Value; if (String.IsNullOrEmpty(data)) { data = body; } else { data = data.Substring(6); }*/ /*var agent = builder.GetAgentById(project); if (agent == null) { agent = builder.GetAgentByName(project); } var corpus = builder.ExtractorCorpus(agent); var meta = await builder.Train(agent, corpus);*/ // var rasa_nlu_data = JsonConvert.DeserializeObject(data); //rasa_nlu_data.Model = model; //rasa_nlu_data.Project = project; var agent = await builder.LoadAgentFromFile>(rawPath); await builder.SaveAgent(agent); return agent; } private async Task> Train([FromBody] RasaTrainRequestViewModel request, [FromQuery] string project) { var trainer = new BotTrainer(settings); if (String.IsNullOrEmpty(request.Project)) { request.Project = project; } // save corpus to agent dir var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project); var modelPath = Path.Combine(projectPath, request.Model); if (!Directory.Exists(modelPath)) { Directory.CreateDirectory(modelPath); } // Save raw data to file, then parse it to Agent instance. var metaFileName = Path.Combine(modelPath, "meta.json"); System.IO.File.WriteAllText(metaFileName, JsonConvert.SerializeObject(new AgentImportHeader { Name = project })); // in order to unify the process. var fileName = Path.Combine(modelPath, "corpus.json"); System.IO.File.WriteAllText(fileName, JsonConvert.SerializeObject(request, new JsonSerializerSettings { Formatting = Formatting.Indented, NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver() })); var agent = await builder.GetAgentByName(project); var info = await trainer.Train(agent, new BotTrainOptions { AgentDir = projectPath, Model = request.Model }); return Ok(new { info = info.Model }); } } }