1. Added context in/ out, lifespan.

2. Import agent in Dialogflow format.
3. Lazy training per input contexts.
This commit is contained in:
haiping008@gmail.com 2018-03-21 16:07:43 -05:00
parent 3d4134764a
commit 33d42ae035
59 changed files with 1380 additions and 208 deletions

1
.gitignore vendored
View file

@ -288,3 +288,4 @@ __pycache__/
*.xsd.cs
/App_Data
/Bot.WebStarter/App_Data/bot-rasa.db
/Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant

View file

@ -14,17 +14,17 @@ namespace Bot.Rasa.RestApi
[HttpGet("id")]
public Agent Get([FromRoute] String id)
{
var console = new RasaConsole(dc);
var console = new RasaAi(dc);
return console.LoadAgent(id);
}
[HttpPost]
public String Create([FromBody] Agent agent)
{
var console = new RasaConsole(dc);
var console = new RasaAi(dc);
dc.DbTran(() => {
console.CreateAgent(agent);
console.SaveAgent(agent);
});
return agent.Id;

View file

@ -9,7 +9,7 @@ namespace Bot.Rasa.RestApi
public class EntityController : EssentialController
{
[HttpPost]
public string CreateEntity([FromBody] EntityType entity)
public string CreateEntity([FromBody] Entity entity)
{
return entity.Id;
}

View file

@ -11,7 +11,7 @@ namespace Bot.Rasa.RestApi
public class EntityTypeController : EssentialController
{
[HttpPost]
public string CreateType([FromBody] EntityType entityType)
public string CreateType([FromBody] Entity entityType)
{
var agent = dc.Table<Agent>().Find(entityType.AgentId);
dc.DbTran(() => agent.CreateEntityType(dc, entityType));

View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowAgent
{
public String Id { get; set; }
public String Name { get; set; }
public String Description { get; set; }
public Boolean Published { get; set; }
public String Language { get; set; }
}
}

View file

@ -0,0 +1,42 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
[JsonObject]
public class DialogflowEntity
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("entries")]
public List<DialogflowEntityEntry> Entries { get; set; }
public DialogflowEntity()
{
}
public DialogflowEntity(string name)
{
this.Name = name;
}
public DialogflowEntity(string name, List<DialogflowEntityEntry> entries)
{
this.Name = name;
this.Entries = entries;
}
public void AddEntry(DialogflowEntityEntry entry)
{
if (Entries == null)
{
Entries = new List<DialogflowEntityEntry>();
}
Entries.Add(entry);
}
}
}

View file

@ -0,0 +1,32 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
[JsonObject]
public class DialogflowEntityEntry
{
[JsonProperty("value")]
public string Value { get; set; }
[JsonProperty("synonyms")]
public List<String> Synonyms { get; set; }
public DialogflowEntityEntry()
{
}
public DialogflowEntityEntry(string value, List<string> synonyms)
{
this.Value = value;
this.Synonyms = synonyms;
}
public DialogflowEntityEntry(string value, string[] synonyms) : this(value, new List<string>(synonyms))
{
}
}
}

View file

@ -0,0 +1,25 @@
using Bot.Rasa.Intents;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntent
{
public string Id { get; set; }
public string Name { get; set; }
public bool Auto { get; set; }
/// <summary>
/// Input Contexts
/// </summary>
public List<String> ContextList { get; set; }
public List<DialogflowIntentExpression> UserSays { get; set; }
public List<DialogflowIntentResponse> Responses { get; set; }
public int Priority { get; set; }
public bool WebhookUsed { get; set; }
public bool FallbackIntent { get; set; }
public List<DialogflowIntentEvent> Events { get; set; }
}
}

View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntentEvent
{
public string Name { get; set; }
}
}

View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntentExpression
{
public String Id { get; set; }
public List<DialogflowIntentExpressionPart> Data { get; set; }
public Boolean IsTemplate { get; set; }
}
}

View file

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntentExpressionPart
{
public String Text { get; set; }
public String Alias { get; set; }
public String Meta { get; set; }
public Boolean UserDefined { get; set; }
}
}

View file

@ -0,0 +1,26 @@
using Bot.Rasa.Intents;
using Bot.Rasa.Models;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntentResponse
{
public string Id { get; set; }
public bool ResetContexts { get; set; }
public string Action { get; set; }
public List<AIContext> AffectedContexts { get; set; }
public List<DialogflowIntentResponseParameter> Parameters { get; set; }
public List<DialogflowIntentResponseMessage> MessageList { get; set; }
public DialogflowIntentResponse()
{
Id = Guid.NewGuid().ToString();
}
}
}

View file

@ -0,0 +1,14 @@
using Bot.Rasa.Models;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntentResponseMessage : AIResponseMessageBase
{
public string Lang { get; set; }
public Object Speech { get; set; }
public Object Payload { get; set; }
}
}

View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Adapters.Dialogflow
{
public class DialogflowIntentResponseParameter
{
public string Id { get; set; }
public bool Required { get; set; }
public string DataType { get; set; }
public string Name { get; set; }
public string Value { get; set; }
public bool IsList { get; set; }
}
}

View file

@ -16,11 +16,17 @@ namespace Bot.Rasa.Agents
[MaxLength(64)]
public String Name { get; set; }
public String Description { get; set; }
public Boolean Published { get; set; }
public String Language { get; set; }
[ForeignKey("AgentId")]
public List<Intent> Intents { get; set; }
[ForeignKey("AgentId")]
[JsonProperty("entity_types")]
public List<EntityType> EntityTypes { get; set; }
public List<Entity> Entities { get; set; }
}
}

View file

@ -23,29 +23,45 @@ namespace Bot.Rasa.Agents
return dc.Table<Agent>().Find(agentId);
}
public static String CreateEntity(this Agent agent, EntityType entity, Database dc)
public static String CreateEntity(this Agent agent, Entity entity, Database dc)
{
return entity.Id;
}
public static RasaTrainingData GrabCorpus(this Agent agent, Database dc)
public static RasaTrainingData GrabCorpus(this Agent agent, Database dc, List<AIContext> ctx)
{
var trainingData = new RasaTrainingData
{
UserSays = new List<UserSay>()
};
var intents = dc.Table<Intent>().Include(x => x.Expressions).ToList();
var intents = dc.Table<Intent>()
.Include(x => x.Contexts)
.Include(x => x.UserSays).ThenInclude(say => say.Data).ToList();
intents.ForEach(intent => {
var contexts = ctx.OrderBy(x => x.Name).Select(x => x.Name.ToLower()).ToList();
trainingData.UserSays.AddRange(intent.Expressions
// search all potential intents which input context included in contexts
intents = intents.Where(it =>
{
if (contexts.Count == 0)
{
return it.Contexts.Count() == 0;
}
else
{
return it.Contexts.Count() > 0 && it.Contexts.Count(x => contexts.Contains(x.Name.ToLower())) == it.Contexts.Count;
}
}).OrderByDescending(x => x.Contexts.Count).ToList();
intents.ForEach(intent =>
{
trainingData.UserSays.AddRange(intent.UserSays
.Select(exp => new UserSay
{
Intent = intent.Name,
Text = exp.Text
Text = String.Join("", exp.Data.OrderBy(x => x.UpdatedTime).Select(x => x.Text))
}));
});
return trainingData;

View file

@ -1,22 +0,0 @@
using Bot.Rasa.Intents;
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Agents
{
public class AgentResponse
{
public AgentResponseIntent Intent { get; set; }
public String Text { get; set; }
}
public class AgentResponseIntent
{
public String Name { get; set; }
public Decimal Confidence { get; set; }
}
}

View file

@ -12,4 +12,9 @@
<PackageReference Include="RestSharp" Version="106.2.1" />
</ItemGroup>
<ItemGroup>
<Folder Include="Adapters\Luis\" />
<Folder Include="Adapters\Lex\" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,128 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Bot.Rasa.Adapters.Dialogflow;
using Bot.Rasa.Agents;
using Bot.Rasa.Entities;
using Bot.Rasa.Intents;
using Bot.Rasa.Models;
using DotNetToolkit;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace Bot.Rasa.Consoles
{
public class AgentImporterInDialogflow : IAgentImporter
{
public Agent LoadAgent(string agentId, string agentDir)
{
// load agent profile
string data = File.ReadAllText($"{agentDir}\\Dialogflow\\{agentId}\\agent.json");
var agent = JsonConvert.DeserializeObject<DialogflowAgent>(data);
agent.Id = Guid.NewGuid().ToString();
agent.Name = agentId;
return agent.ToObject<Agent>();
}
public void LoadEntities(Agent agent, string agentDir)
{
agent.Entities = new List<Entity>();
Directory.EnumerateFiles($"{agentDir}\\Dialogflow\\{agent.Name}\\entities")
.ToList()
.ForEach(fileName =>
{
string entityName = fileName.Split('\\').Last();
if (!entityName.Contains("_"))
{
string entityJson = File.ReadAllText($"{fileName}");
var entity = JsonConvert.DeserializeObject<DialogflowEntity>(entityJson);
// load entries
string entriesFileName = fileName.Replace(entity.Name, $"{entity.Name}_entries_{agent.Language}");
if (File.Exists(entriesFileName))
{
string entriesJson = File.ReadAllText($"{entriesFileName}");
entity.Entries = JsonConvert.DeserializeObject<List<DialogflowEntityEntry>>(entriesJson);
}
agent.Entities.Add(entity.ToObject<Entity>());
}
});
}
public void LoadIntents(Agent agent, string agentDir)
{
agent.Intents = new List<Intent>();
Directory.EnumerateFiles($"{agentDir}\\Dialogflow\\{agent.Name}\\intents")
.ToList()
.ForEach(fileName =>
{
if (!fileName.Contains("_usersays_" + agent.Language))
{
string intentJson = File.ReadAllText($"{fileName}");
// avoid confict data structure
intentJson = intentJson.Replace("\"contexts\":", "\"contextList\":");
intentJson = intentJson.Replace("\"messages\":", "\"messageList\":");
var intent = JsonConvert.DeserializeObject<DialogflowIntent>(intentJson);
// load user expressions
string expressionFileName = fileName.Replace(intent.Name, $"{intent.Name}_usersays_{agent.Language}");
if (File.Exists(expressionFileName))
{
string expressionJson = File.ReadAllText($"{expressionFileName}");
intent.UserSays = JsonConvert.DeserializeObject<List<DialogflowIntentExpression>>(expressionJson);
}
var newIntent = intent.ToObject<Intent>();
intent.Responses.ForEach(res =>
{
var newResponse = newIntent.Responses.First(x => x.Id == res.Id);
newResponse.Contexts = res.AffectedContexts.Select(x => new IntentResponseContext {
Name = x.Name,
Lifespan = x.Lifespan
}).ToList();
newResponse.Messages = res.MessageList.Where(x => x.Speech != null || x.Payload != null)
.Select(x =>
{
if(x.Type == AIResponseMessageType.Custom)
{
return new IntentResponseMessage
{
Lang = x.Lang,
Payload = JsonConvert.SerializeObject(x.Payload),
Type = x.Type
};
} else
{
var speech = JsonConvert.SerializeObject(x.Speech.GetType().Equals(typeof(String)) ?
new List<String> { x.Speech.ToString() } :
(x.Speech as JArray).Select(s => s.Value<String>()).ToList());
return new IntentResponseMessage
{
Lang = x.Lang,
Speech = speech,
Type = x.Type
};
}
}).ToList();
});
newIntent.Contexts = intent.ContextList.Select(x => new IntentInputContext { Name = x }).ToList();
agent.Intents.Add(newIntent);
}
});
}
}
}

View file

@ -0,0 +1,22 @@
using Bot.Rasa.Agents;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Consoles
{
public interface IAgentImporter
{
/// <summary>
/// Load agent summary
/// </summary>
/// <param name="agentId">agent guid or name</param>
/// <param name="agentDir"></param>
/// <returns></returns>
Agent LoadAgent(string agentId, string agentDir);
void LoadEntities(Agent agent, string agentDir);
void LoadIntents(Agent agent, string agentDir);
}
}

View file

@ -0,0 +1,86 @@
using Bot.Rasa.Agents;
using Bot.Rasa.Entities;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Bot.Rasa.Consoles
{
public class RasaAi
{
private Database dc { get; set; }
public static RasaOptions Options { get; set; }
public static IConfiguration Configuration { get; set; }
public Agent agent { get; set; }
public String SessionId { get; set; }
public RasaAi(Database dc)
{
this.dc = dc;
SessionId = Guid.NewGuid().ToString();
}
/// <summary>
/// Restore a agent instance from backup json files
/// </summary>
/// <param name="importor"></param>
/// <param name="agentId"></param>
/// <returns></returns>
public Agent RestoreAgent(IAgentImporter importer, String agentId)
{
string dataDir = $"{Options.ContentRootPath}\\App_Data\\DbInitializer\\Agents\\";
// Load agent summary
agent = importer.LoadAgent(agentId, dataDir);
// Load agent entities
importer.LoadEntities(agent, dataDir);
// Load agent intents
importer.LoadIntents(agent, dataDir);
return agent;
}
/// <summary>
/// Dump agent train data to json file
/// </summary>
/// <param name="agentId"></param>
/// <returns></returns>
public bool DumpAgent(String agentId)
{
return true;
}
public Agent LoadAgent(String agentId)
{
return dc.Table<Agent>()
.Include(x => x.Intents).ThenInclude(x => x.Contexts)
.FirstOrDefault(x => x.Id == agentId);
}
public String SaveAgent(Agent agent)
{
var existedAgent = dc.Table<Agent>().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name);
if (existedAgent == null)
{
dc.Table<Agent>().Add(agent);
return agent.Id;
}
else
{
agent.Id = existedAgent.Id;
return existedAgent.Id;
}
}
}
}

View file

@ -1,81 +0,0 @@
using Bot.Rasa.Agents;
using Bot.Rasa.Entities;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Bot.Rasa.Consoles
{
public class RasaConsole
{
private Database dc { get; set; }
public static RasaOptions Options { get; set; }
public static IConfiguration Configuration { get; set; }
public RasaConsole(Database dc)
{
this.dc = dc;
}
/// <summary>
/// Restore a agent instance from json file
/// </summary>
/// <param name="agentId"></param>
/// <returns></returns>
public Agent RestoreAgent(String agentId)
{
string json = File.ReadAllText($"{Options.ContentRootPath}\\App_Data\\DbInitializer\\Agents\\{agentId}.json");
var agent = JsonConvert.DeserializeObject<Agent>(json);
agent.Id = agentId;
agent.EntityTypes.ForEach(entityType =>
{
entityType.Items = entityType.Values.Select(x => new EntityItem
{
Value = x
}).ToList();
});
agent.Intents.ForEach(intent => {
intent.Expressions.ForEach(expression =>
{
});
});
return agent;
}
/// <summary>
/// Dump agent train data to json file
/// </summary>
/// <param name="agentId"></param>
/// <returns></returns>
public bool DumpAgent(String agentId)
{
return true;
}
public Agent LoadAgent(String agentId)
{
return dc.Table<Agent>().Include(x => x.Intents).FirstOrDefault(x => x.Id == agentId);
}
public String CreateAgent(Agent agent)
{
if (dc.Table<Agent>().Any(x => x.Id == agent.Id)) return String.Empty;
dc.Table<Agent>().Add(agent);
return agent.Id;
}
}
}

View file

@ -1,33 +1,167 @@
using Bot.Rasa.Agents;
using Bot.Rasa.Intents;
using Bot.Rasa.Models;
using Bot.Rasa.Sessions;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using RestSharp;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Bot.Rasa.Consoles
{
public static class RequestExtension
{
public static AgentResponse TextRequest(this RasaConsole console, String agentId, String text)
public static AIResponse TextRequest(this RasaAi rasa, Database dc, AIRequest request)
{
var client = new RestClient($"{RasaConsole.Options.HostUrl}");
AIResponse aiResponse = new AIResponse();
RasaResponse response = null;
var request = new RestRequest("parse", Method.POST);
string json = JsonConvert.SerializeObject(new { Project = agentId, Q = text },
new JsonSerializerSettings
// Merge input contexts
var contexts = dc.Table<SessionContext>()
.Where(x => x.SessionId == rasa.SessionId && x.Lifespan > 0)
.ToList()
.Select(x => new AIContext { Name = x.Context.ToLower(), Lifespan = x.Lifespan })
.ToList();
contexts.AddRange(request.Contexts.Select(x => new AIContext { Name = x.Name.ToLower(), Lifespan = x.Lifespan }));
contexts = contexts.OrderBy(x => x.Name).ToList();
// search all potential intents which input context included in contexts
var intents = rasa.agent.Intents.Where(it =>
{
if (contexts.Count == 0)
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
return it.Contexts.Count() == 0;
}
else
{
return it.Contexts.Count() > 0 &&
it.Contexts.Count(x => contexts.Select(ctx => ctx.Name).Contains(x.Name.ToLower())) == it.Contexts.Count;
}
}).OrderByDescending(x => x.Contexts.Count).ToList();
// Max contexts match
foreach(var it in intents)
{
request.Contexts = it.Contexts.Select(x => new AIContext { Name = x.Name.ToLower() })
.OrderBy(x => x.Name)
.ToList();
string contextId = $"{String.Join(',', request.Contexts.Select(x => x.Name))}".GetMd5Hash();
string modelName = dc.Table<ContextModelMapping>().FirstOrDefault(x => x.ContextId == contextId)?.ModelName;
// need training
if (String.IsNullOrEmpty(modelName))
{
dc.DbTran(() =>
{
modelName = TrainWithContexts(rasa, dc, request, contextId);
});
}
var client = new RestClient($"{RasaAi.Options.HostUrl}");
var rest = new RestRequest("parse", Method.POST);
string json = JsonConvert.SerializeObject(new { Project = rasa.agent.Id, Q = request.Query.First(), Model = modelName },
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
rest.AddParameter("application/json", json, ParameterType.RequestBody);
var result = client.Execute<RasaResponse>(rest);
if(result.Data.Intent != null)
{
response = result.Data;
break;
}
};
var intent = (dc.Table<Intent>().Where(x => x.Name == response.Intent.Name)
.Include(x => x.Responses).ThenInclude(x => x.Contexts)
.Include(x => x.Responses).ThenInclude(x => x.Parameters)
.Include(x => x.Responses).ThenInclude(x => x.Messages)).First();
var intentResponse = ArrayHelper.GetRandom(intent.Responses);
aiResponse.Id = Guid.NewGuid().ToString();
aiResponse.Lang = rasa.agent.Language;
aiResponse.Status = new AIResponseStatus { };
aiResponse.SessionId = rasa.SessionId;
aiResponse.Timestamp = DateTime.UtcNow;
intentResponse.Messages.Where(x => x.Type == AIResponseMessageType.Text)
.ToList()
.ForEach(msg =>
{
msg.Speech = ArrayHelper.GetRandom(msg.Speech.Substring(2, msg.Speech.Length - 4).Split("\",\"").ToList());
});
request.AddParameter("application/json", json, ParameterType.RequestBody);
var response = client.Execute<AgentResponse>(request);
aiResponse.Result = new AIResponseResult
{
Source = "agent",
ResolvedQuery = request.Query.First(),
Action = intentResponse.Action,
Parameters = new Dictionary<string, object>(),
Score = response.Intent.Confidence,
Metadata = new AIResponseMetadata { IntentId = intent.Id, IntentName = intent.Name },
Fulfillment = new AIResponseFulfillment
{
Messages = intentResponse.Messages.Select(x => (object)x).ToList()
}
};
return response.Data;
// Merge context lifespan
// override if exists, otherwise add, delete if lifespan is zero
dc.DbTran(() =>
{
var sessionContexts = dc.Table<SessionContext>().Where(x => x.SessionId == rasa.SessionId).ToList();
// minus 1 round
sessionContexts.Where(x => !intentResponse.Contexts.Select(ctx => ctx.Name).Contains(x.Context))
.ToList()
.ForEach(ctx => ctx.Lifespan = ctx.Lifespan - 1);
intentResponse.Contexts.ForEach(ctx =>
{
var session1 = sessionContexts.FirstOrDefault(x => x.Context == ctx.Name);
if (session1 != null)
{
if (ctx.Lifespan == 0)
{
dc.Table<SessionContext>().Remove(session1);
}
else
{
session1.Lifespan = ctx.Lifespan;
}
}
else
{
dc.Table<SessionContext>().Add(new SessionContext
{
SessionId = rasa.SessionId,
Context = ctx.Name,
Lifespan = ctx.Lifespan
});
}
});
});
aiResponse.Result.Contexts = dc.Table<SessionContext>()
.Where(x => x.SessionId == rasa.SessionId)
.Select(x => new AIContext { Name = x.Context, Lifespan = x.Lifespan })
.ToArray();
return aiResponse;
}
/// <summary>
@ -35,12 +169,30 @@ namespace Bot.Rasa.Consoles
/// </summary>
/// <param name="console"></param>
/// <param name="dc"></param>
/// <param name="agentId"></param>
/// <param name="request"></param>
/// <param name="contextId"></param>
/// <returns></returns>
public static bool Train(this RasaConsole console, Database dc, String agentId)
public static string TrainWithContexts(this RasaAi console, Database dc, AIRequest request, String contextId)
{
var agent = dc.Table<Agent>().Find(agentId);
var corpus = agent.GrabCorpus(dc);
var corpus = console.agent.GrabCorpus(dc, request.Contexts);
corpus.UserSays.Add(new UserSay
{
Intent = "Welcome",
Text = "Hi"
});
corpus.UserSays.Add(new UserSay
{
Intent = "Welcome",
Text = "Hey"
});
corpus.UserSays.Add(new UserSay
{
Intent = "Welcome",
Text = "Hello"
});
string json = JsonConvert.SerializeObject(new { rasa_nlu_data = corpus },
new JsonSerializerSettings
@ -48,14 +200,33 @@ namespace Bot.Rasa.Consoles
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
var client = new RestClient($"{RasaConsole.Options.HostUrl}");
var request = new RestRequest("train", Method.POST);
request.AddQueryParameter("project", agentId);
request.AddParameter("application/json", json, ParameterType.RequestBody);
var client = new RestClient($"{RasaAi.Options.HostUrl}");
var rest = new RestRequest("train", Method.POST);
rest.AddQueryParameter("project", console.agent.Id);
rest.AddParameter("application/json", json, ParameterType.RequestBody);
var response = client.Execute(request);
var response = client.Execute(rest);
var result = JObject.Parse(response.Content);
return response.IsSuccessful;
if (response.IsSuccessful)
{
string modelName = result["info"].Value<String>().Split(": ")[1];
dc.Table<ContextModelMapping>().Add(new ContextModelMapping
{
AgentId = console.agent.Id,
ModelName = modelName,
ContextId = contextId
});
return modelName;
}
else
{
Console.WriteLine(result["error"]);
return String.Empty;
}
}
}
}

View file

@ -7,8 +7,8 @@ using System.Text;
namespace Bot.Rasa.Entities
{
[Table("Bot_EntityType")]
public class EntityType : DbRecord, IDbRecord
[Table("Bot_Entity")]
public class Entity : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
@ -21,7 +21,7 @@ namespace Bot.Rasa.Entities
[NotMapped]
public List<String> Values { get; set; }
[ForeignKey("EntityTypeId")]
public List<EntityItem> Items { get; set; }
[ForeignKey("EntityId")]
public List<EntityEntry> Entries { get; set; }
}
}

View file

@ -7,12 +7,12 @@ using System.Text;
namespace Bot.Rasa.Entities
{
[Table("Bot_EntityItem")]
public class EntityItem : DbRecord, IDbRecord
[Table("Bot_EntityEntry")]
public class EntityEntry : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String EntityTypeId { get; set; }
public String EntityId { get; set; }
[MaxLength(128)]
public String Value { get; set; }

View file

@ -9,21 +9,21 @@ namespace Bot.Rasa.Entities
{
public static class EntityTypeExtension
{
public static string CreateEntityType(this Agent agent, Database dc, EntityType entityType)
public static string CreateEntityType(this Agent agent, Database dc, Entity entityType)
{
if (dc.Table<EntityType>().Any(x => x.Name == entityType.Name && x.AgentId == agent.Id)) return agent.Id;
if (dc.Table<Entity>().Any(x => x.Name == entityType.Name && x.AgentId == agent.Id)) return agent.Id;
dc.Table<EntityType>().Add(entityType);
dc.Table<Entity>().Add(entityType);
return entityType.Id;
}
public static void DeleteEntityType(this Agent agent, Database dc, String entityTypeId)
{
var entityType = dc.Table<EntityType>().FirstOrDefault(x => x.Id == entityTypeId);
var entityType = dc.Table<Entity>().FirstOrDefault(x => x.Id == entityTypeId);
if (entityType == null) return;
dc.Table<EntityType>().Remove(entityType);
dc.Table<Entity>().Remove(entityType);
}
}
}

View file

@ -7,21 +7,23 @@ using System.Text;
namespace Bot.Rasa.Expressions
{
[Table("Bot_EntityOfSpeech")]
public class EntitiyOfSpeech : DbRecord, IDbRecord
[Table("Bot_IntentExpressionPart")]
public class IntentExpressionPart : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String ExpressionId { get; set; }
public int Start { get; set; }
[Required]
[MaxLength(128)]
public String Value { get; set; }
public String Text { get; set; }
[Required]
[MaxLength(64)]
public String Entity { get; set; }
public String Alias { get; set; }
[MaxLength(64)]
public String Meta { get; set; }
public Boolean UserDefined { get; set; }
}
}

View file

@ -0,0 +1,25 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Intents
{
[Table("Bot_ContextModelMapping")]
public class ContextModelMapping : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String AgentId { get; set; }
[Required]
[StringLength(32)]
public string ContextId { get; set; }
[Required]
[StringLength(21)]
public string ModelName { get; set; }
}
}

View file

@ -24,6 +24,12 @@ namespace Bot.Rasa.Intents
public String Description { get; set; }
[ForeignKey("IntentId")]
public List<IntentExpression> Expressions { get; set; }
public List<IntentInputContext> Contexts { get; set; }
[ForeignKey("IntentId")]
public List<IntentExpression> UserSays { get; set; }
[ForeignKey("IntentId")]
public List<IntentResponse> Responses { get; set; }
}
}

View file

@ -15,23 +15,19 @@ namespace Bot.Rasa.Intents
{
public IntentExpression()
{
Entities = new List<EntitiyOfSpeech>();
Data = new List<IntentExpressionPart>();
}
[Required]
[StringLength(36)]
public String IntentId { get; set; }
[Required]
[MaxLength(128)]
public String Text { get; set; }
[ForeignKey("ExpressionId")]
public List<EntitiyOfSpeech> Entities { get; set; }
public List<IntentExpressionPart> Data { get; set; }
public bool IsExist(Database dc)
{
return dc.Table<IntentExpression>().Any(x => x.IntentId == IntentId && x.Text == Text);
return dc.Table<IntentExpression>().Any(x => x.IntentId == IntentId);
}
}
}

View file

@ -0,0 +1,21 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Intents
{
[Table("Bot_IntentInputContext")]
public class IntentInputContext : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String IntentId { get; set; }
[Required]
[MaxLength(64)]
public string Name { get; set; }
}
}

View file

@ -0,0 +1,31 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Intents
{
[Table("Bot_IntentResponse")]
public class IntentResponse : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String IntentId { get; set; }
[MaxLength(128)]
public String Action { get; set; }
public Boolean ResetContexts { get; set; }
[ForeignKey("IntentResponseId")]
public List<IntentResponseContext> Contexts { get; set; }
[ForeignKey("IntentResponseId")]
public List<IntentResponseParameter> Parameters { get; set; }
[ForeignKey("IntentResponseId")]
public List<IntentResponseMessage> Messages { get; set; }
}
}

View file

@ -0,0 +1,22 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Intents
{
[Table("Bot_IntentResponseContext")]
public class IntentResponseContext : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String IntentResponseId { get; set; }
[MaxLength(64)]
public string Name { get; set; }
public int Lifespan { get; set; }
}
}

View file

@ -0,0 +1,36 @@
using Bot.Rasa.Models;
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Intents
{
[Table("Bot_IntentResponseMessage")]
public class IntentResponseMessage : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String IntentResponseId { get; set; }
public AIResponseMessageType Type { get; set; }
[Required]
[MaxLength(3)]
public String Lang { get; set; }
/// <summary>
/// json list data
/// </summary>
[MaxLength(1024)]
public String Speech { get; set; }
/// <summary>
/// custom json payload
/// </summary>
[MaxLength(1024)]
public String Payload { get; set; }
}
}

View file

@ -0,0 +1,22 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Intents
{
[Table("Bot_IntentResponseParameter")]
public class IntentResponseParameter : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String IntentResponseId { get; set; }
public bool Required { get; set; }
public string DataType { get; set; }
public string Name { get; set; }
public string Value { get; set; }
public bool IsList { get; set; }
}
}

View file

@ -0,0 +1,27 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
[JsonObject]
public class AIContext
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("parameters")]
public Dictionary<string, string> Parameters { get; set; }
/// <summary>
/// Lifespan of the context measured in requests````
/// </summary>
[JsonProperty("lifespan")]
public int Lifespan { get; set; }
public AIContext()
{
}
}
}

View file

@ -0,0 +1,46 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
[JsonObject]
public class AIRequest : QuestionMetadata
{
[JsonProperty("query")]
public string[] Query { get; set; }
[JsonProperty("confidence")]
public float[] Confidence { get; set; }
[JsonProperty("contexts")]
public List<AIContext> Contexts { get; set; }
[JsonProperty("resetContexts")]
public bool? ResetContexts { get; set; }
[JsonProperty("originalRequest")]
public OriginalRequest OriginalRequest { get; set; }
public AIRequest()
{
Contexts = new List<AIContext>();
}
public AIRequest(string text)
{
Query = new string[] { text };
Confidence = new float[] { 1.0f };
}
public AIRequest(string text, RequestExtras requestExtras) : this(text)
{
if (requestExtras != null)
{
requestExtras.CopyTo(this);
}
}
}
}

View file

@ -0,0 +1,34 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponse
{
public string Id { get; set; }
public DateTime Timestamp { get; set; }
public string Lang { get; set; }
public AIResponseResult Result { get; set; }
public AIResponseStatus Status { get; set; }
public string SessionId { get; set; }
public bool IsError
{
get
{
if (Status != null && Status.Code >= 400)
{
return true;
}
return false;
}
}
}
}

View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponseCustomPayload : AIResponseMessageBase
{
public string Task { get; set; }
public Object Body { get; set; }
}
}

View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponseFulfillment
{
public string Speech { get; set; }
public List<object> Messages { get; set; }
}
}

View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponseMessageBase
{
public AIResponseMessageType Type { get; set; }
}
}

View file

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public enum AIResponseMessageType
{
Text = 0,
Custom = 4
}
}

View file

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponseMetadata
{
public string IntentId { get; set; }
public string IntentName { get; set; }
}
}

View file

@ -0,0 +1,148 @@
using Bot.Rasa.Intents;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponseResult
{
String action;
public Boolean ActionIncomplete { get; set; }
public String Action
{
get
{
if (string.IsNullOrEmpty(action))
{
return string.Empty;
}
return action;
}
set
{
action = value;
}
}
public Dictionary<string, object> Parameters { get; set; }
public AIContext[] Contexts { get; set; }
public AIResponseMetadata Metadata { get; set; }
public String ResolvedQuery { get; set; }
public AIResponseFulfillment Fulfillment { get; set; }
public string Source { get; set; }
public decimal Score { get; set; }
[JsonIgnore]
public bool HasParameters
{
get
{
return Parameters != null && Parameters.Count > 0;
}
}
public string GetStringParameter(string name, string defaultValue = "")
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (Parameters.ContainsKey(name))
{
return Parameters[name].ToString();
}
return defaultValue;
}
public int GetIntParameter(string name, int defaultValue = 0)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (Parameters.ContainsKey(name))
{
var parameterValue = Parameters[name].ToString();
int result;
if (int.TryParse(parameterValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out result))
{
return result;
}
float floatResult;
if (float.TryParse(parameterValue, NumberStyles.Float, CultureInfo.InvariantCulture, out floatResult))
{
result = Convert.ToInt32(floatResult);
return result;
}
}
return defaultValue;
}
public float GetFloatParameter(string name, float defaultValue = 0)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (Parameters.ContainsKey(name))
{
var parameterValue = Parameters[name].ToString();
float result;
if (float.TryParse(parameterValue, NumberStyles.Float, CultureInfo.InvariantCulture, out result))
{
return result;
}
}
return defaultValue;
}
public JObject GetJsonParameter(string name, JObject defaultValue = null)
{
if (string.IsNullOrEmpty("name"))
{
throw new ArgumentNullException(nameof(name));
}
if (Parameters.ContainsKey(name))
{
var parameter = Parameters[name] as JObject;
if (parameter != null)
{
return parameter;
}
}
return defaultValue;
}
public AIContext GetContext(string name)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentException("Name must be not empty", nameof(name));
}
return Contexts?.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.CurrentCultureIgnoreCase));
}
}
}

View file

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class AIResponseStatus
{
public int Code { get; set; }
public string ErrorType { get; set; }
public string ErrorDetails { get; set; }
public string ErrorID { get; set; }
}
}

View file

@ -0,0 +1,17 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
[JsonObject]
public class OriginalRequest
{
[JsonProperty("source")]
public string Source { get; set; }
[JsonProperty("data")]
public object Data { get; set; }
}
}

View file

@ -0,0 +1,24 @@
using Bot.Rasa.Entities;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
[JsonObject]
public class QuestionMetadata
{
[JsonProperty("timezone")]
public string Timezone { get; set; }
[JsonProperty("lang")]
public string Language { get; set; }
[JsonProperty("sessionId")]
internal string SessionId { get; set; }
[JsonProperty("entities")]
public List<Entity> Entities { get; set; }
}
}

View file

@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class RasaResponse
{
public RasaResponseIntent Intent { get; set; }
public String Text { get; set; }
}
public class RasaResponseIntent
{
public String Name { get; set; }
public Decimal Confidence { get; set; }
}
}

View file

@ -0,0 +1,63 @@
using Bot.Rasa.Entities;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bot.Rasa.Models
{
public class RequestExtras
{
public List<AIContext> Contexts { get; set; }
public List<Entity> Entities { get; set; }
public bool HasContexts
{
get
{
if (Contexts != null && Contexts.Count > 0)
{
return true;
}
return false;
}
}
public bool HasEntities
{
get
{
if (Entities != null && Entities.Count > 0)
{
return true;
}
return false;
}
}
public RequestExtras()
{
}
public RequestExtras(List<AIContext> contexts, List<Entity> entities)
{
this.Contexts = contexts;
this.Entities = entities;
}
public void CopyTo(AIRequest request)
{
if (HasContexts)
{
request.Contexts = Contexts;
}
if (HasEntities)
{
request.Entities = Entities;
}
}
}
}

View file

@ -9,11 +9,11 @@ namespace Bot.Rasa.Models
{
public UserSay()
{
Entities = new List<EntitiyOfSpeech>();
Entities = new List<IntentExpressionPart>();
}
public String Text { get; set; }
public String Intent { get; set; }
public List<EntitiyOfSpeech> Entities { get; set; }
public List<IntentExpressionPart> Entities { get; set; }
}
}

View file

@ -0,0 +1,23 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace Bot.Rasa.Sessions
{
[Table("Bot_SessionContext")]
public class SessionContext : DbRecord, IDbRecord
{
[Required]
[StringLength(36)]
public String SessionId { get; set; }
[Required]
[MaxLength(64)]
public String Context { get; set; }
public int Lifespan { get; set; }
}
}

View file

@ -1,6 +1,7 @@
using Bot.Rasa;
using Bot.Rasa.Agents;
using Bot.Rasa.Consoles;
using Bot.Rasa.Models;
using EntityFrameworkCore.BootKit;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
@ -13,43 +14,40 @@ namespace Bot.UnitTest
[TestClass]
public class AgentTest : TestEssential
{
public static String PIZZA_BOT_ID = "2b6a288e-d891-40c6-96ce-6a0cf324545c";
public static String BOT_ID = "2b6a288e-d891-40c6-96ce-6a0cf324545c";
public static String BOT_NAME = "VirtualAssistant";
[TestMethod]
public void CreateAgent()
{
var rasa = new RasaConsole(dc);
var rasa = new RasaAi(dc);
var importer = new AgentImporterInDialogflow();
var agent = rasa.RestoreAgent(importer, BOT_NAME);
agent.Id = BOT_ID;
var agent = rasa.RestoreAgent(PIZZA_BOT_ID);
int row = dc.DbTran(() => rasa.CreateAgent(agent));
if(row > 0)
{
var result = rasa.Train(dc, agent.Id);
}
int row = dc.DbTran(() => rasa.SaveAgent(agent));
var loadedAgent = rasa.LoadAgent(agent.Id);
Assert.IsTrue(loadedAgent.Intents.Count == agent.Intents.Count);
var response = rasa.TextRequest(agent.Id, "weather in Chicago tomorrow");
Assert.IsTrue(response.Intent.Name == "weather");
}
[TestMethod]
public void TextRequest()
{
var rasa = new RasaConsole(dc);
var response = rasa.TextRequest(PIZZA_BOT_ID, "how old are you");
response = rasa.TextRequest(PIZZA_BOT_ID, "where are you from");
response = rasa.TextRequest(PIZZA_BOT_ID, "would you like some cookie");
var rasa = new RasaAi(dc);
rasa.agent = rasa.LoadAgent(BOT_ID);
var response = rasa.TextRequest(dc, new AIRequest { Query = new String[] { "Create a work order for PetSmart" } });
Assert.IsTrue(response.Result.Metadata.IntentName == "Create Work Order");
response = rasa.TextRequest(dc, new AIRequest { Query = new String[] { "1010" } });
Assert.IsTrue(response.Result.Metadata.IntentName == "Telling Store Number");
}
[TestMethod]
public void Train()
{
var rasa = new RasaConsole(dc);
rasa.Train(dc, PIZZA_BOT_ID);
var rasa = new RasaAi(dc);
}
}
}

View file

@ -30,20 +30,12 @@ namespace Bot.UnitTest
dc = new DefaultDataContextLoader().GetDefaultDc();
RasaConsole.Options = new RasaOptions
RasaAi.Options = new RasaOptions
{
HostUrl = "http://192.168.56.101:5000",
HostUrl = Database.Configuration.GetSection("Rasa:Host").Value,
ContentRootPath = contentRoot,
Assembles = new String[] { "Bot.Rasa" }
};
dc = new Database();
dc.BindDbContext<IDbRecord, DbContext4Sqlite>(new DatabaseBind
{
MasterConnection = new SqliteConnection($"Data Source={RasaConsole.Options.ContentRootPath}\\App_Data\\bot-rasa.db"),
CreateDbIfNotExist = true
});
}
}

View file

@ -1,9 +1,9 @@
{
"TokenAuthentication": {
"SecretKey": "tK5aNR0FTEm4htInv+HA3A==",
"Subject": "Voicecoin",
"Issuer": "Voicecoin",
"Audience": "Voicecoin",
"SecretKey": "",
"Subject": "OpenBotKit",
"Issuer": "Haiping Chen",
"Audience": "Haiping Chen",
"TokenPath": "/token",
"CookieName": "token",
"LoginPath": "/login"

View file

@ -1,10 +1,10 @@
{
"AWS": {
"AWSRegionEndPoint": "us-east-1",
"AWSSecretKey": "38+UqlSqOVbKfL3LrS6hSmgW/ZSPgZzUa/Hom7ip",
"AWSAccessKey": "AKIAIPXFRDXOTKZWOVXQ",
"AWSSecretKey": "",
"AWSAccessKey": "",
"AWSEncoding": "utf-8",
"SESVerifiedEmail": "haiping008@gmail.com",
"AWSBucketPrefix": "voicecoin.ico"
"SESVerifiedEmail": "",
"AWSBucketPrefix": ""
}
}

View file

@ -1,10 +1,10 @@
{
"Database": {
"Default": "Sqlite",
"Default": "SqlServer",
"ConnectionStrings": {
"InMemory": "DataSource=:memory:",
"Sqlite": "Data Source=|DataDirectory|\\bot-rasa.db;",
"SqlServer": "Data Source=(localdb)\\MSSQLLocalDB;Initial Catalog=bot-rasa;Integrated Security=True;Connect Timeout=15;Encrypt=False;TrustServerCertificate=True;ApplicationIntent=ReadWrite;MultiSubnetFailover=False"
"Sqlite": "Data Source=|DataDirectory|\\RasaBot.db;",
"SqlServer": "Data Source=(localdb)\\MSSQLLocalDB;Initial Catalog=RasaBot;Integrated Security=True;Connect Timeout=15;Encrypt=False;TrustServerCertificate=True;ApplicationIntent=ReadWrite;MultiSubnetFailover=False"
}
}
}

View file

@ -1,5 +1,5 @@
{
"Rasa": {
"Host": "http://192.168.56.101:5000"
"Host": "http://81a1f75c.ngrok.io"
}
}

View file

@ -1,8 +1,8 @@
{
"Swagger": {
"Version": "v1",
"Title": "RasaBot API",
"Description": "RasaBot API",
"Title": "OpenBotKit API",
"Description": "OpenBotKit API",
"TermsOfService": "MIT",
"Contact": {
"Name": "Haiping Chen",