using BotSharp.Core.Engines;
using BotSharp.Platform.Abstractions;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using BotSharp.Platform.Rasa.Models;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace BotSharp.Platform.Rasa.Controllers
{
///
/// You can post your training data to this endpoint to train a new model for a project.
/// This request will wait for the server answer: either the model was trained successfully or the training exited with an error.
///
[Route("[controller]")]
public class TrainController : ControllerBase
{
private RasaAi builder;
private readonly IPlatformSettings settings;
public TrainController(RasaAi configuration, IPlatformSettings settings)
{
builder = configuration;
this.settings = settings;
}
///
/// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project.
///
/// Model name
/// Agent name or agent id
///
[HttpPost]
public async Task> Train([FromQuery] string project, [FromQuery] string model)
{
string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project);
if (!Directory.Exists(agentDir))
{
Directory.CreateDirectory(agentDir);
}
string body = "";
using (var reader = new StreamReader(Request.Body))
{
body = reader.ReadToEnd();
}
var agent = await ImportAgent(project, body);
var corpus = await builder.ExtractorCorpus(agent);
var meta = await builder.Train(agent, corpus, new BotTrainOptions { Model = model });
return meta;
}
private async Task ImportAgent(string project, string body)
{
Console.WriteLine($"Update agent from http post, data length: {body.Length}");
// save to file
// save corpus to agent dir
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project);
var rawPath = Path.Combine(projectPath, "tmp");
// clear tmp dir
if (Directory.Exists(rawPath))
{
Directory.Delete(rawPath, true);
}
Directory.CreateDirectory(rawPath);
// Save raw data to file, then parse it to Agent instance.
var metaFileName = Path.Combine(rawPath, "meta.json");
System.IO.File.WriteAllText(metaFileName, JsonConvert.SerializeObject(new AgentImportHeader
{
Name = project,
Platform = PlatformType.Rasa,
Id = Guid.NewGuid().ToString()
}, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
// in order to unify the process.
var fileName = Path.Combine(rawPath, "corpus.json");
System.IO.File.WriteAllText(fileName, body);
/*string lang = Regex.Match(body, @"language:.+")?.Value;
if (!String.IsNullOrEmpty(lang))
{
lang = lang.Substring(11, 2);
}
string data = Regex.Match(body, @"data:([\s\S]*)")?.Value;
if (String.IsNullOrEmpty(data))
{
data = body;
}
else
{
data = data.Substring(6);
}*/
/*var agent = builder.GetAgentById(project);
if (agent == null)
{
agent = builder.GetAgentByName(project);
}
var corpus = builder.ExtractorCorpus(agent);
var meta = await builder.Train(agent, corpus);*/
// var rasa_nlu_data = JsonConvert.DeserializeObject(data);
//rasa_nlu_data.Model = model;
//rasa_nlu_data.Project = project;
var agent = await builder.LoadAgentFromFile>(rawPath);
await builder.SaveAgent(agent);
return agent;
}
private async Task> Train([FromBody] RasaTrainRequestViewModel request, [FromQuery] string project)
{
var trainer = new BotTrainer(settings);
if (String.IsNullOrEmpty(request.Project))
{
request.Project = project;
}
// save corpus to agent dir
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project);
var modelPath = Path.Combine(projectPath, request.Model);
if (!Directory.Exists(modelPath))
{
Directory.CreateDirectory(modelPath);
}
// Save raw data to file, then parse it to Agent instance.
var metaFileName = Path.Combine(modelPath, "meta.json");
System.IO.File.WriteAllText(metaFileName, JsonConvert.SerializeObject(new AgentImportHeader
{
Name = project
}));
// in order to unify the process.
var fileName = Path.Combine(modelPath, "corpus.json");
System.IO.File.WriteAllText(fileName, JsonConvert.SerializeObject(request, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
var agent = await builder.GetAgentByName(project);
var info = await trainer.Train(agent, new BotTrainOptions
{
AgentDir = projectPath,
Model = request.Model
});
return Ok(new { info = info.Model });
}
}
}