migrate RasaAI

This commit is contained in:
Oceania2018 2018-10-03 22:39:39 -05:00
parent fdd0cb6778
commit 98e846dd38
29 changed files with 25 additions and 1411 deletions

View file

@ -1,198 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using BotSharp.Core.Agents;
using BotSharp.Core.Entities;
using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using DotNetToolkit;
using Newtonsoft.Json;
namespace BotSharp.Core.Engines.Rasa
{
public class AgentImporterInRasa<TAgent> : IAgentImporter<TAgent>
{
public string AgentDir { get; set; }
public Agent LoadAgent(AgentImportHeader agentHeader)
{
var agent = new Agent();
agent.ClientAccessToken = Guid.NewGuid().ToString("N");
agent.DeveloperAccessToken = Guid.NewGuid().ToString("N");
agent.Id = agentHeader.Id;
agent.Name = agentHeader.Name;
return agent;
}
public void LoadBuildinEntities(Agent agent)
{
agent.Intents.ForEach(intent =>
{
if (intent.UserSays != null)
{
intent.UserSays.ForEach(us =>
{
us.Data.Where(data => data.Meta != null)
.ToList()
.ForEach(data =>
{
LoadBuildinEntityTypePerUserSay(agent, data);
});
});
}
});
}
private void LoadBuildinEntityTypePerUserSay(Agent agent, IntentExpressionPart data)
{
var existedEntityType = agent.Entities.FirstOrDefault(x => x.Name == data.Meta);
if (existedEntityType == null)
{
existedEntityType = new EntityType
{
Name = data.Meta,
Entries = new List<EntityEntry>(),
IsOverridable = true
};
agent.Entities.Add(existedEntityType);
}
var entries = existedEntityType.Entries.Select(x => x.Value.ToLower()).ToList();
if (!entries.Contains(data.Text.ToLower()))
{
existedEntityType.Entries.Add(new EntityEntry
{
Value = data.Text,
Synonyms = new List<EntrySynonym>
{
new EntrySynonym
{
Synonym = data.Text
}
}
});
}
}
public void LoadCustomEntities(Agent agent)
{
agent.Entities = new List<EntityType>();
}
public void LoadIntents(Agent agent)
{
string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json"));
var rasa = JsonConvert.DeserializeObject<RasaAgentImportModel>(data);
agent.Intents = rasa.Data.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
agent.Intents.ForEach(intent => {
ImportIntentUserSays(intent, rasa.Data.UserSays);
});
}
private void ImportIntentUserSays(Intent intent, List<RasaIntentExpression> sentences)
{
intent.UserSays = new List<IntentExpression>();
var userSays = sentences.Where(x => x.Intent == intent.Name).ToList();
userSays.ForEach(say =>
{
var expression = new IntentExpression();
say.Entities = say.Entities.OrderBy(x => x.Start).ToList();
expression.Data = new List<IntentExpressionPart>();
int pos = 0;
for (int entityIdx = 0; entityIdx < say.Entities.Count; entityIdx++)
{
var entity = say.Entities[entityIdx];
// previous
if (entity.Start > 0)
{
expression.Data.Add(new IntentExpressionPart
{
Text = say.Text.Substring(pos, entity.Start - pos),
Start = pos
});
}
// self
expression.Data.Add(new IntentExpressionPart
{
Alias = entity.Entity,
Meta = entity.Entity,
Text = say.Text.Substring(entity.Start, entity.Value.Length),
Start = entity.Start
});
pos = entity.End + 1;
if (pos < say.Text.Length && entityIdx == say.Entities.Count - 1)
{
// end
expression.Data.Add(new IntentExpressionPart
{
Text = say.Text.Substring(pos),
Start = pos
});
}
}
if (say.Entities.Count == 0)
{
expression.Data.Add(new IntentExpressionPart
{
Text = say.Text.Substring(pos)
});
}
int second = 0;
expression.Data.ForEach(x => x.UpdatedTime = DateTime.UtcNow.AddSeconds(second++));
intent.UserSays.Add(expression);
});
}
public void AssembleTrainData(Agent agent)
{
// convert agent to training corpus
agent.Corpus = new TrainingCorpus
{
Entities = new List<TrainingEntity>(),
UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>()
};
agent.Intents.ForEach(intent =>
{
intent.UserSays.ForEach(say => {
agent.Corpus.UserSays.Add(new TrainingIntentExpression<TrainingIntentExpressionPart>
{
Intent = intent.Name,
Text = String.Join("", say.Data.Select(x => x.Text)),
Entities = say.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.Select(x => new TrainingIntentExpressionPart
{
Value = x.Text,
Entity = x.Meta,
Start = x.Start
})
.ToList()
});
});
});
}
}
}

View file

@ -1,30 +0,0 @@
using BotSharp.Core.Adapters.Rasa;
using BotSharp.Core.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines.Rasa
{
public class RasaAgent
{
public String Id { get; set; }
public String Name { get; set; }
[JsonProperty("common_examples")]
public List<RasaIntentExpression> UserSays { get; set; }
[JsonProperty("entity_synonyms")]
public List<RasaTrainingEntity> Entities { get; set; }
[JsonProperty("regex_features")]
public List<RasaTrainingRegex> Regex { get; set; }
}
public class RasaAgentImportModel
{
[JsonProperty("rasa_nlu_data")]
public RasaAgent Data { get; set; }
}
}

View file

@ -1,223 +0,0 @@
using BotSharp.Core.Adapters.Rasa;
using BotSharp.Core.Agents;
using BotSharp.Core.Entities;
using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using RestSharp;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace BotSharp.Core.Engines
{
/// <summary>
/// Rasa nlu >= 0.12
/// </summary>
public class RasaAi : BotEngineBase, IBotPlatform
{
public AIResponse TextRequest(AIRequest request)
{
AIResponse aiResponse = new AIResponse();
string model = RasaRequestExtension.GetModelPerContexts(agent, AiConfig, request, dc);
var result = CallRasa(agent.Id, request.Query.First(), model);
result.Content.Log();
RasaResponse response = result.Data;
aiResponse.Id = Guid.NewGuid().ToString();
aiResponse.Lang = agent.Language;
aiResponse.Status = new AIResponseStatus { };
aiResponse.SessionId = AiConfig.SessionId;
aiResponse.Timestamp = DateTime.UtcNow;
var intentResponse = RasaRequestExtension.HandleIntentPerContextIn(agent, AiConfig, request, result.Data, dc);
RasaRequestExtension.HandleParameter(agent, intentResponse, response, request);
RasaRequestExtension.HandleMessage(intentResponse);
aiResponse.Result = new AIResponseResult
{
Source = "agent",
ResolvedQuery = request.Query.First(),
Action = intentResponse?.Action,
Parameters = intentResponse?.Parameters?.ToDictionary(x => x.Name, x => (object)x.Value),
Score = response.Intent.Confidence,
Metadata = new AIResponseMetadata { IntentId = intentResponse?.IntentId, IntentName = intentResponse?.IntentName },
Fulfillment = new AIResponseFulfillment
{
Messages = intentResponse?.Messages?.Select(x => {
if (x.Type == AIResponseMessageType.Custom)
{
return (new
{
x.Type,
Payload = JsonConvert.DeserializeObject(x.PayloadJson)
}) as Object;
}
else
{
return (new { x.Type, x.Speech }) as Object;
}
}).ToList()
}
};
RasaRequestExtension.HandleContext(dc, AiConfig, intentResponse, aiResponse);
Console.WriteLine(JsonConvert.SerializeObject(aiResponse.Result));
return aiResponse;
}
private IRestResponse<RasaResponse> CallRasa(string projectId, string text, string model)
{
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var client = new RestClient($"{config.GetSection("RasaNlu:url").Value}");
var rest = new RestRequest("parse", Method.POST);
string json = JsonConvert.SerializeObject(new { Project = projectId, Q = text, Model = model },
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
rest.AddParameter("application/json", json, ParameterType.RequestBody);
return client.Execute<RasaResponse>(rest);
}
public void Train()
{
var trainingData = new RasaTrainingData
{
Entities = new List<RasaTrainingEntity>(),
UserSays = new List<RasaIntentExpression>()
};
var corpus = GetIntentExpressions();
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var client = new RestClient($"{config.GetSection("RasaNlu:url").Value}");
var contextHashs = corpus.UserSays
.Select(x => x.ContextHash)
.Distinct()
.ToList();
contextHashs.ForEach(ctx =>
{
var common_examples = corpus.UserSays.Where(x => x.ContextHash == ctx || x.ContextHash == Guid.Empty.ToString("N")).ToList();
// assemble entity and synonyms
var usedEntities = new List<String>();
common_examples.ForEach(x =>
{
if (x.Entities != null)
{
usedEntities.AddRange(x.Entities.Select(y => y.Entity));
}
});
usedEntities = usedEntities.Distinct().ToList();
var entity_synonyms = corpus.Entities.Where(x => usedEntities.Contains(x.Entity)).ToList();
var data = new RasaTrainingData
{
Entities = entity_synonyms.Select(x => x.ToObject<RasaTrainingEntity>()).ToList(),
UserSays = common_examples.Select(x => x.ToObject<RasaIntentExpression>()).ToList()
};
// meet minimal requirement
// at least 2 different classes
int count = data.UserSays
.Select(x => x.Intent)
.Distinct().Count();
if (count < 2)
{
data.UserSays.Add(new RasaIntentExpression
{
Intent = "Intent2",
Text = Guid.NewGuid().ToString("N")
});
data.UserSays.Add(new RasaIntentExpression
{
Intent = "Intent2",
Text = Guid.NewGuid().ToString("N")
});
}
// at least 2 corpus per intent
data.UserSays.Select(x => x.Intent)
.Distinct()
.ToList()
.ForEach(intent =>
{
if(data.UserSays.Count(x => x.Intent == intent) < 2)
{
data.UserSays.Add(new RasaIntentExpression
{
Intent = intent,
Text = Guid.NewGuid().ToString("N")
});
}
});
// set empty synonym to null
/*data.Entities
.Where(x => x.Entity != null)
.ToList()
.ForEach(entity =>
{
if (entity.Synonyms.Count == 0)
{
entity.Synonyms = null;
}
});*/
string json = JsonConvert.SerializeObject(new { rasa_nlu_data = data },
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
NullValueHandling = NullValueHandling.Ignore,
});
var rest = new RestRequest("train", Method.POST);
rest.AddQueryParameter("project", agent.Id);
rest.AddQueryParameter("model", ctx);
string trainingConfig = agent.Language == "zh" ? "config_jieba_mitie_sklearn.yml" : "config_mitie_sklearn.yml";
var contentRootPatch = AppDomain.CurrentDomain.GetData("ContentRootPath").ToString();
string body = File.ReadAllText(Path.Combine(contentRootPatch, "Settings", trainingConfig));
body = $"{body}\r\ndata: {json}";
rest.AddParameter("application/x-yml", body, ParameterType.RequestBody);
var response = client.Execute(rest);
if (response.IsSuccessful)
{
var result = JObject.Parse(response.Content);
string modelName = result["info"].Value<String>().Split(new string[] { ": " }, StringSplitOptions.None)[1];
}
else
{
var result = JObject.Parse(response.Content);
Console.WriteLine(result["error"]);
result["error"].Log();
}
});
}
}
}

View file

@ -1,15 +0,0 @@
using BotSharp.Core.Adapters.Rasa;
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Models
{
public class RasaIntentExpression : TrainingIntentExpression<TrainingIntentExpressionPart>
{
}
}

View file

@ -1,12 +0,0 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Models
{
public class RasaIntentExpressionPart : TrainingIntentExpressionPart
{
}
}

View file

@ -1,15 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public class RasaOptions
{
public string HostUrl { get; set; }
public String[] Assembles { get; set; }
public string ContentRootPath { get; set; }
public String DbName { get; set; }
public String DbConnectionString { get; set; }
}
}

View file

@ -1,275 +0,0 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using BotSharp.Core.Conversations;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using RestSharp;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.Core.Engines
{
public static class RasaRequestExtension
{
public static AIResponse TextRequest(this RasaAi rasa, string text, RequestExtras requestExtras)
{
return rasa.TextRequest(new AIRequest(text, requestExtras));
}
public static IntentResponse HandleIntentPerContextIn(Agent agent, AIConfiguration aiConfig, AIRequest request, RasaResponse response, Database dc)
{
// Merge input contexts
var contexts = dc.Table<ConversationContext>()
.Where(x => x.ConversationId == aiConfig.SessionId && x.Lifespan > 0)
.ToList()
.Select(x => new AIContext { Name = x.Context.ToLower(), Lifespan = x.Lifespan })
.ToList();
contexts.AddRange(request.Contexts.Select(x => new AIContext { Name = x.Name.ToLower(), Lifespan = x.Lifespan }));
contexts = contexts.OrderBy(x => x.Name).ToList();
// search all potential intents which input context included in contexts
var intents = agent.Intents.Where(it =>
{
if (contexts.Count == 0)
{
return it.Contexts.Count() == 0;
}
else
{
return it.Contexts.Count() == 0 ||
it.Contexts.Count(x => contexts.Select(ctx => ctx.Name).Contains(x.Name.ToLower())) == it.Contexts.Count;
}
}).OrderByDescending(x => x.Contexts.Count).ToList();
if (response.IntentRanking == null)
{
response.IntentRanking = new List<RasaResponseIntent>
{
response.Intent
};
}
response.IntentRanking = response.IntentRanking.Where(x => x.Confidence > agent.MlConfig.MinConfidence).ToList();
response.IntentRanking = response.IntentRanking.Where(x => intents.Select(i => i.Name).Contains(x.Name)).ToList();
// add Default Fallback Intent
if (response.IntentRanking.Count == 0)
{
var defaultFallbackIntent = agent.Intents.FirstOrDefault(x => x.Name == "Default Fallback Intent");
response.IntentRanking.Add(new RasaResponseIntent
{
Name = defaultFallbackIntent.Name,
Confidence = decimal.Parse("0.8")
});
}
response.Intent = response.IntentRanking.First();
var intent = (dc.Table<Intent>().Where(x => x.AgentId == agent.Id && x.Name == response.Intent.Name)
.Include(x => x.Responses).ThenInclude(x => x.Contexts)
.Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts)
.Include(x => x.Responses).ThenInclude(x => x.Messages)).First();
var intentResponse = ArrayHelper.GetRandom(intent.Responses);
intentResponse.IntentName = intent.Name;
return intentResponse;
}
/// <summary>
///
/// </summary>
/// <param name="agent"></param>
/// <param name="intentResponse"></param>
/// <param name="response"></param>
/// <param name="request"></param>
/// <returns>Required field is missed</returns>
public static void HandleParameter(Agent agent, IntentResponse intentResponse, RasaResponse response, AIRequest request)
{
if (intentResponse == null) return;
intentResponse.Parameters.ForEach(p => {
string query = request.Query.First();
var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name || x.Entity.Split(':').Contains(p.Name));
if (entity != null)
{
p.Value = query.Substring(entity.Start, entity.End - entity.Start);
}
// convert to Standard entity value
if (!String.IsNullOrEmpty(p.Value) && !p.DataType.StartsWith("sys."))
{
p.Value = agent.Entities
.FirstOrDefault(x => x.Name == p.DataType)
.Entries
.FirstOrDefault((entry) =>
{
return entry.Value.ToLower() == p.Value.ToLower() ||
entry.Synonyms.Select(synonym => synonym.Synonym.ToLower()).Contains(p.Value.ToLower());
})?.Value;
}
// fixed entity per request
if (request.Entities != null)
{
var fixedEntity = request.Entities.FirstOrDefault(x => x.Name == p.Name);
if (fixedEntity != null)
{
if (query.ToLower().Contains(fixedEntity.Entries.First().Value.ToLower()))
{
p.Value = fixedEntity.Entries.First().Value;
}
}
}
});
}
public static void HandleMessage(IntentResponse intentResponse)
{
if (intentResponse == null) return;
var missingRequiredParameter = intentResponse.Parameters.FirstOrDefault(x => x.Required && String.IsNullOrEmpty(x.Value));
if (missingRequiredParameter != null)
{
intentResponse.Messages = new List<IntentResponseMessage> {
new IntentResponseMessage {
Type = AIResponseMessageType.Text,
Speech = ArrayHelper.GetRandom(missingRequiredParameter.Prompts).Prompt,
IntentResponseId = intentResponse.Id,
UpdatedTime = DateTime.UtcNow
}
};
}
else
{
intentResponse.Messages = intentResponse.Messages.OrderBy(x => x.UpdatedTime).ToList();
}
intentResponse.Messages.ToList()
.ForEach(msg =>
{
if (msg.Type == AIResponseMessageType.Custom)
{
}
else
{
if (msg.Speech != "[]")
{
msg.Speech = msg.Speech.StartsWith("[") ?
ArrayHelper.GetRandom(msg.Speech.Substring(2, msg.Speech.Length - 4).Split(new string[] { "\",\"" }, StringSplitOptions.None).ToList()) :
msg.Speech;
msg.Speech = ReplaceParameters4Response(intentResponse.Parameters, msg.Speech);
}
}
});
}
private static string ReplaceParameters4Response(List<IntentResponseParameter> parameters, string text)
{
var reg = new Regex(@"\$\w+");
reg.Matches(text).Cast<Match>().ToList().ForEach(token => {
var parameter = parameters.FirstOrDefault(x => x.Name == token.Value.Substring(1));
if(parameter != null)
{
text = text.Replace(token.Value, parameter?.Value?.ToString());
}
});
return text;
}
public static void HandleContext(Database dc, AIConfiguration AiConfig, IntentResponse intentResponse, AIResponse aiResponse)
{
if (intentResponse == null) return;
// Merge context lifespan
// override if exists, otherwise add, delete if lifespan is zero
dc.DbTran(() =>
{
var sessionContexts = dc.Table<ConversationContext>().Where(x => x.ConversationId == AiConfig.SessionId).ToList();
// minus 1 round
sessionContexts.Where(x => !intentResponse.Contexts.Select(ctx => ctx.Name).Contains(x.Context))
.ToList()
.ForEach(ctx => ctx.Lifespan = ctx.Lifespan - 1);
intentResponse.Contexts.ForEach(ctx =>
{
var session1 = sessionContexts.FirstOrDefault(x => x.Context == ctx.Name);
if (session1 != null)
{
if (ctx.Lifespan == 0)
{
dc.Table<ConversationContext>().Remove(session1);
}
else
{
session1.Lifespan = ctx.Lifespan;
}
}
else
{
dc.Table<ConversationContext>().Add(new ConversationContext
{
ConversationId = AiConfig.SessionId,
Context = ctx.Name,
Lifespan = ctx.Lifespan
});
}
});
});
aiResponse.Result.Contexts = dc.Table<ConversationContext>()
.Where(x => x.Lifespan > 0 && x.ConversationId == AiConfig.SessionId)
.Select(x => new AIContext { Name = x.Context.ToLower(), Lifespan = x.Lifespan })
.ToArray();
}
public static string GetModelPerContexts(Agent agent, AIConfiguration aiConfig, AIRequest request, Database dc)
{
// Merge input contexts
var contexts = dc.Table<ConversationContext>()
.Where(x => x.ConversationId == aiConfig.SessionId && x.Lifespan > 0)
.ToList()
.Select(x => new AIContext { Name = x.Context.ToLower(), Lifespan = x.Lifespan })
.ToList();
contexts.AddRange(request.Contexts.Select(x => new AIContext { Name = x.Name.ToLower(), Lifespan = x.Lifespan }));
contexts = contexts.OrderBy(x => x.Name).ToList();
// search all potential intents which input context included in contexts
var intents = agent.Intents.Where(it =>
{
if (contexts.Count == 0)
{
return it.Contexts.Count() == 0;
}
else
{
return it.Contexts.Count() > 0 &&
it.Contexts.Count(x => contexts.Select(ctx => ctx.Name).Contains(x.Name.ToLower())) == it.Contexts.Count;
}
}).OrderByDescending(x => x.Contexts.Count).ToList();
// query per request contexts
var contextHashs = intents.Select(x => x.ContextHash).Distinct().ToList();
return contextHashs.FirstOrDefault();
}
}
}

View file

@ -1,32 +0,0 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Models
{
public class RasaResponse
{
public RasaResponseIntent Intent { get; set; }
public AIResponseFulfillment Fullfillment { get; set; }
[JsonProperty("intent_ranking")]
public List<RasaResponseIntent> IntentRanking { get; set; }
public List<RasaResponseEntity> Entities { get; set; }
public String Text { get; set; }
public String Project { get; set; }
public String Model { get; set; }
}
public class RasaResponseIntent
{
public String Name { get; set; }
public Decimal Confidence { get; set; }
}
}

View file

@ -1,11 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Models
{
public class RasaResponseEntity : RasaIntentExpressionPart
{
public string Extractor { get; set; }
}
}

View file

@ -1,21 +0,0 @@
using BotSharp.Core.Adapters.Rasa;
using BotSharp.Core.Engines.Rasa;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Models
{
public class RasaTrainingData
{
[JsonProperty("common_examples")]
public List<RasaIntentExpression> UserSays { get; set; }
[JsonProperty("entity_synonyms")]
public List<RasaTrainingEntity> Entities { get; set; }
[JsonProperty("regex_features")]
public List<RasaTrainingRegex> Regex { get; set; }
}
}

View file

@ -1,15 +0,0 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Adapters.Rasa
{
public sealed class RasaTrainingEntity : TrainingEntity
{
[JsonProperty("value")]
public override String Entity { get; set; }
}
}

View file

@ -1,13 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines.Rasa
{
public class RasaTrainingRegex
{
public String Name { get; set; }
public String Pattern { get; set; }
}
}

View file

@ -1,10 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Platform.Models
{
public class DialogRequestOptions
{
}
}

View file

@ -1,53 +0,0 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace BotSharp.Platform.Models
{
/// <summary>
/// Standard agent data structure
/// All other platform agent has to align with this standard data structure.
/// </summary>
public class StandardAgent : AgentBase
{
public StandardAgent()
{
CreatedDate = DateTime.UtcNow;
Entities = new List<EntityBase>();
Intents = new List<IntentBase>();
}
/// <summary>
/// Is the chatbot public or private
/// </summary>
public Boolean Published { get; set; }
/// <summary>
/// Only access text/ audio rquest
/// </summary>
[StringLength(32)]
public String ClientAccessToken { get; set; }
/// <summary>
/// Developer can access more APIs
/// </summary>
[StringLength(32)]
public String DeveloperAccessToken { get; set; }
public List<IntentBase> Intents { get; set; }
public List<EntityBase> Entities { get; set; }
public String Birthday
{
get
{
return CreatedDate.ToShortDateString();
}
}
public DateTime CreatedDate { get; set; }
}
}

View file

@ -1,28 +0,0 @@
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
#if RASA
[Route("[controller]")]
public class ConfigController : ControllerBase
{
[HttpGet]
public ActionResult<RasaVersionModel> Get()
{
var status = new RasaStatusModel();
status.AvailableProjects = JObject.FromObject(new RasaProjectModel
{
Status = "ready",
AvailableModels = new List<string> { "model_XXXXXX" },
LoadedModels = new List<string> { "model_XXXXXX" }
});
return Ok(status);
}
}
#endif
}

View file

@ -1,109 +0,0 @@
using BotSharp.Core.Engines;
using BotSharp.Core.Engines.Rasa;
using BotSharp.Core.Models;
using BotSharp.NLP;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using Console = Colorful.Console;
namespace BotSharp.RestApi.Rasa
{
#if RASA
/// <summary>
/// send a text request
/// </summary>
[Route("[controller]")]
public class ParseController : ControllerBase
{
private readonly IBotPlatform _platform;
/// <summary>
/// Initialize dialog controller and get a platform instance
/// </summary>
/// <param name="platform"></param>
public ParseController(IBotPlatform platform)
{
_platform = platform;
}
/// <summary>
/// parse request
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
[HttpPost, HttpGet]
public ActionResult<RasaResponse> Parse(RasaRequestModel request)
{
var config = new AIConfiguration("", SupportedLanguage.English);
config.SessionId = "rasa nlu";
string body = "";
using (var reader = new StreamReader(Request.Body))
{
body = reader.ReadToEnd();
}
Console.WriteLine($"Got message from {Request.Host}: {body}", Color.Green);
if(request.Project ==null && !String.IsNullOrEmpty(body))
{
request = JsonConvert.DeserializeObject<RasaRequestModel>(body);
}
// Load agent
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project);
if (String.IsNullOrEmpty(request.Model))
{
request.Model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last();
}
var modelPath = Path.Combine(projectPath, request.Model);
var agent = _platform.LoadAgentFromFile(modelPath);
var aIResponse = _platform.TextRequest(new AIRequest
{
AgentDir = projectPath,
Model = request.Model,
Query = new String[] { request.Text }
});
var rasaResponse = new RasaResponse
{
Intent = new RasaResponseIntent
{
Name = aIResponse.Result.Metadata.IntentName,
Confidence = aIResponse.Result.Score
},
Entities = aIResponse.Result.Entities.Select(x => new RasaResponseEntity
{
Extractor = x.Extrator,
Start = x.Start,
Entity = x.Entity,
Value = x.Value
}).ToList(),
Text = request.Text,
Model = request.Model,
Project = agent.Name,
IntentRanking = new List<RasaResponseIntent>
{
new RasaResponseIntent
{
Name = aIResponse.Result.Metadata.IntentName,
Confidence = aIResponse.Result.Score
}
},
Fullfillment = aIResponse.Result.Fulfillment
};
return rasaResponse;
}
}
#endif
}

View file

@ -1,13 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
public class RasaConfigModel
{
public string Config { get; set; }
public string Data { get; set; }
}
}

View file

@ -1,17 +0,0 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
public class RasaRequestModel
{
[JsonProperty("q")]
public string Text { get; set; }
public string Project { get; set; }
public string Model { get; set; }
}
}

View file

@ -1,35 +0,0 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
public class RasaStatusModel
{
[JsonProperty("available_projects")]
public JObject AvailableProjects { get; set; }
[JsonProperty("current_training_processes")]
public int CurrentTrainingProcesses { get; set; }
[JsonProperty("max_training_processes")]
public int MaxTrainingProcesses { get; set; }
}
public class RasaProjectModel
{
[JsonProperty("status")]
public string Status { get; set; }
[JsonProperty("current_training_processes")]
public int CurrentTrainingProcesses { get; set; }
[JsonProperty("available_models")]
public List<string> AvailableModels { get; set; }
[JsonProperty("loaded_models")]
public List<string> LoadedModels { get; set; }
}
}

View file

@ -1,18 +0,0 @@
using BotSharp.Core.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
public class RasaTrainRequestModel
{
public string Project { get; set; }
public string Model { get; set; }
[JsonProperty("rasa_nlu_data")]
public RasaTrainingData Corpus { get; set; }
}
}

View file

@ -1,15 +0,0 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
public class RasaVersionModel
{
public string Version { get; set; }
[JsonProperty("minimum_compatible_version")]
public string MinimumCompatibleVersion { get; set; }
}
}

View file

@ -1,77 +0,0 @@
using BotSharp.Core.Engines;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
#if RASA
/// <summary>
/// This returns all the currently available projects.
/// </summary>
[Route("[controller]")]
public class StatusController : ControllerBase
{
private readonly IBotPlatform _platform;
/// <summary>
/// Initialize status controller and get a platform instance
/// </summary>
/// <param name="platform"></param>
public StatusController(IBotPlatform platform)
{
_platform = platform;
}
/// <summary>
/// Returns a list of available projects the server can use to fulfill /parse requests.
/// </summary>
/// <returns></returns>
[HttpGet]
public ActionResult<RasaVersionModel> Get()
{
var status = new RasaStatusModel();
status.AvailableProjects = JObject.FromObject(new { });
status.MaxTrainingProcesses = 1;
// scan dir, get all models
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects");
if (!Directory.Exists(projectPath))
{
Directory.CreateDirectory(projectPath);
}
var projectDirs = Directory.GetDirectories(projectPath);
for(int idx = 0; idx < projectDirs.Length; idx++)
{
string project = projectDirs[idx].Split('\\').Last();
var modelDirs = Directory.GetDirectories(projectDirs[idx]);
List<string> availableModels = new List<string>();
for (int mIdx = 0; mIdx < modelDirs.Length; mIdx++)
{
string model = modelDirs[mIdx].Split('\\').Last();
if (model.StartsWith(project + "_"))
{
availableModels.Add(model);
}
}
status.AvailableProjects.Add(project, JObject.FromObject(new RasaProjectModel
{
Status = "ready",
AvailableModels = availableModels
}));
}
return Ok(status);
}
}
#endif
}

View file

@ -1,140 +0,0 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines.Rasa;
using BotSharp.Platform.Models;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace BotSharp.RestApi.Rasa
{
#if RASA
/// <summary>
/// You can post your training data to this endpoint to train a new model for a project.
/// This request will wait for the server answer: either the model was trained successfully or the training exited with an error.
/// </summary>
[Route("[controller]")]
public class TrainController : ControllerBase
{
private readonly IBotPlatform _platform;
/// <summary>
/// Initialize dialog controller and get a platform instance
/// </summary>
/// <param name="platform"></param>
public TrainController(IBotPlatform platform)
{
_platform = platform;
}
/// <summary>
/// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project.
/// </summary>
/// <param name="model">Model name</param>
/// <param name="project">Agent name or agent id</param>
/// <returns></returns>
[HttpPost]
public async Task<ActionResult<String>> Train([FromQuery] string project, [FromQuery] string model)
{
string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project);
if (!Directory.Exists(agentDir))
{
Directory.CreateDirectory(agentDir);
}
if (string.IsNullOrEmpty(model))
{
string dest = Directory.GetDirectories(agentDir).Where(x => x.Contains("model_")).Last();
var agent = _platform.LoadAgentFromFile(dest);
model = dest.Split(Path.DirectorySeparatorChar).Last();
await _platform.Train(new BotTrainOptions { AgentDir = agentDir, Model = model });
return Ok(new { info = model });
}
else
{
string body = "";
using (var reader = new StreamReader(Request.Body))
{
body = reader.ReadToEnd();
}
string lang = Regex.Match(body, @"language:.+")?.Value;
if (!String.IsNullOrEmpty(lang))
{
lang = lang.Substring(11, 2);
}
string data = Regex.Match(body, @"data:([\s\S]*)")?.Value;
if (String.IsNullOrEmpty(data))
{
data = body;
}
else
{
data = data.Substring(6);
}
var rasa_nlu_data = JsonConvert.DeserializeObject<RasaTrainRequestModel>(data);
rasa_nlu_data.Model = model;
rasa_nlu_data.Project = project;
var trainResult = await Train(rasa_nlu_data, project);
return trainResult;
}
}
private async Task<ActionResult<String>> Train([FromBody] RasaTrainRequestModel request, [FromQuery] string project)
{
var trainer = new BotTrainer();
if (String.IsNullOrEmpty(request.Project))
{
request.Project = project;
}
// save corpus to agent dir
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project);
var modelPath = Path.Combine(projectPath, request.Model);
if (!Directory.Exists(modelPath))
{
Directory.CreateDirectory(modelPath);
}
// Save raw data to file, then parse it to Agent instance.
var metaFileName = Path.Combine(modelPath, "meta.json");
System.IO.File.WriteAllText(metaFileName, JsonConvert.SerializeObject(new AgentImportHeader
{
Name = project,
Platform = PlatformType.Rasa
}));
// in order to unify the process.
var fileName = Path.Combine(modelPath, "corpus.json");
System.IO.File.WriteAllText(fileName, JsonConvert.SerializeObject(request, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
var agent = _platform.LoadAgentFromFile(modelPath);
var info = await trainer.Train(agent, new BotTrainOptions
{
AgentDir = projectPath,
Model = request.Model
});
return Ok(new { info = info.Model });
}
}
#endif
}

View file

@ -1,23 +0,0 @@
using Microsoft.AspNetCore.Mvc;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Rasa
{
#if RASA
[Route("[controller]")]
public class VersionController : ControllerBase
{
[HttpGet]
public ActionResult<RasaVersionModel> Get()
{
return Ok(new RasaVersionModel
{
Version = "0.13.0",
MinimumCompatibleVersion = "0.13.0"
});
}
}
#endif
}

View file

@ -75,13 +75,16 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\botsharp-articulate\BotSharp.Platform.Articulate\BotSharp.Platform.Articulate.csproj" /> <ProjectReference Include="..\..\botsharp-dialogflow\BotSharp.Platform.Dialogflow\BotSharp.Platform.Dialogflow.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Content Update="Settings\app.json"> <Content Update="Settings\app.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content> </Content>
<Content Update="Settings\RasaAi.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Update="Settings\DialogflowAi.json"> <Content Update="Settings\DialogflowAi.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content> </Content>

View file

@ -44,7 +44,7 @@ namespace BotSharp.WebHost
config.AddJsonFile(setting, optional: false, reloadOnChange: true); config.AddJsonFile(setting, optional: false, reloadOnChange: true);
}); });
}) })
.UseUrls("http://0.0.0.0:7500") .UseUrls("http://0.0.0.0:3112")
.UseStartup<Startup>() .UseStartup<Startup>()
.Build(); .Build();
} }

View file

@ -0,0 +1,7 @@
{
"rasaAi": {
"botEngine": "BotSharpNLU",
"agentStorage": "AgentStorageInRedis"
}
}

View file

@ -94,7 +94,7 @@ namespace BotSharp.WebHost
c.DocumentTitle = info.Title; c.DocumentTitle = info.Title;
c.InjectStylesheet(Configuration.GetValue<String>("Swagger:Stylesheet")); c.InjectStylesheet(Configuration.GetValue<String>("Swagger:Stylesheet"));
Console.WriteLine($"{info.Title} {info.Version} {info.License.Name}", Color.Gray); Console.WriteLine($"{info.Title} [{info.Version}] {info.License.Name}", Color.Gray);
Console.WriteLine($"{info.Description}", Color.Gray); Console.WriteLine($"{info.Description}", Color.Gray);
Console.WriteLine($"{info.Contact.Name}", Color.Gray); Console.WriteLine($"{info.Contact.Name}", Color.Gray);
}); });
@ -141,7 +141,9 @@ namespace BotSharp.WebHost
new Formatter(engine, Color.Yellow), new Formatter(engine, Color.Yellow),
}; };
Console.WriteLineFormatted("Platform Emulator: {0} powered by {1} NLU engine.", Color.White, settings); Console.WriteLine();
Console.WriteLineFormatted("Platform Emulator: {0} powered by {1} engine.", Color.White, settings);
Console.WriteLine();
} }
} }
} }

View file

@ -24,7 +24,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Platform.Models",
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Platform.Dialogflow", "..\botsharp-dialogflow\BotSharp.Platform.Dialogflow\BotSharp.Platform.Dialogflow.csproj", "{B865F070-9693-47C6-B901-40121F659C6F}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Platform.Dialogflow", "..\botsharp-dialogflow\BotSharp.Platform.Dialogflow\BotSharp.Platform.Dialogflow.csproj", "{B865F070-9693-47C6-B901-40121F659C6F}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Platform.Articulate", "..\botsharp-articulate\BotSharp.Platform.Articulate\BotSharp.Platform.Articulate.csproj", "{17BAE152-DDCF-4605-8DCB-9F14D403002F}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Platform.Rasa", "..\botsharp-rasa\BotSharp.Platform.Rasa\BotSharp.Platform.Rasa.csproj", "{87C265EF-3A8A-4290-8AD2-33504168F653}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -98,14 +98,14 @@ Global
{B865F070-9693-47C6-B901-40121F659C6F}.Release|Any CPU.Build.0 = Release|Any CPU {B865F070-9693-47C6-B901-40121F659C6F}.Release|Any CPU.Build.0 = Release|Any CPU
{B865F070-9693-47C6-B901-40121F659C6F}.Release|x64.ActiveCfg = Release|Any CPU {B865F070-9693-47C6-B901-40121F659C6F}.Release|x64.ActiveCfg = Release|Any CPU
{B865F070-9693-47C6-B901-40121F659C6F}.Release|x64.Build.0 = Release|Any CPU {B865F070-9693-47C6-B901-40121F659C6F}.Release|x64.Build.0 = Release|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Debug|Any CPU.Build.0 = Debug|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Debug|Any CPU.Build.0 = Debug|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Debug|x64.ActiveCfg = Debug|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Debug|x64.ActiveCfg = Debug|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Debug|x64.Build.0 = Debug|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Debug|x64.Build.0 = Debug|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Release|Any CPU.ActiveCfg = Release|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Release|Any CPU.ActiveCfg = Release|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Release|Any CPU.Build.0 = Release|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Release|Any CPU.Build.0 = Release|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Release|x64.ActiveCfg = Release|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Release|x64.ActiveCfg = Release|Any CPU
{17BAE152-DDCF-4605-8DCB-9F14D403002F}.Release|x64.Build.0 = Release|Any CPU {87C265EF-3A8A-4290-8AD2-33504168F653}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE