From 083f2640addc0b068cf50b882fa3bdc67d8f8191 Mon Sep 17 00:00:00 2001 From: "haiping008@gmail.com" Date: Mon, 21 May 2018 19:39:33 -0500 Subject: [PATCH] add entity and synonym api --- .../Dialogflow/DialogflowEntityEntry.cs | 2 +- BotSharp.Core/Agents/Agent.cs | 2 +- .../{AgentExtension.cs => AgentDriver.cs} | 58 ++++++++++++++----- .../Engines/AgentImporterInDialogflow.cs | 6 +- BotSharp.Core/Engines/IBotEngine.cs | 10 ++++ BotSharp.Core/Engines/RasaAi.cs | 57 +----------------- BotSharp.Core/Entities/EntityEntry.cs | 2 +- BotSharp.Core/Entities/EntityEntryDriver.cs | 28 +++++++++ .../Entities/{Entity.cs => EntityType.cs} | 4 +- BotSharp.Core/Entities/EntityTypeDriver.cs | 41 +++++++++++++ BotSharp.Core/Entities/EntityTypeExtension.cs | 29 ---------- ...{EntityEntrySynonym.cs => EntrySynonym.cs} | 2 +- BotSharp.Core/Entities/EntrySynonymDriver.cs | 17 ++++++ BotSharp.Core/Models/QuestionMetadata.cs | 2 +- BotSharp.Core/Models/RequestExtras.cs | 4 +- BotSharp.UnitTest/AgentTest.cs | 9 +-- BotSharp.UnitTest/TestEssential.cs | 2 +- 17 files changed, 159 insertions(+), 116 deletions(-) rename BotSharp.Core/Agents/{AgentExtension.cs => AgentDriver.cs} (80%) create mode 100644 BotSharp.Core/Engines/IBotEngine.cs create mode 100644 BotSharp.Core/Entities/EntityEntryDriver.cs rename BotSharp.Core/Entities/{Entity.cs => EntityType.cs} (91%) create mode 100644 BotSharp.Core/Entities/EntityTypeDriver.cs delete mode 100644 BotSharp.Core/Entities/EntityTypeExtension.cs rename BotSharp.Core/Entities/{EntityEntrySynonym.cs => EntrySynonym.cs} (88%) create mode 100644 BotSharp.Core/Entities/EntrySynonymDriver.cs diff --git a/BotSharp.Core/Adapters/Dialogflow/DialogflowEntityEntry.cs b/BotSharp.Core/Adapters/Dialogflow/DialogflowEntityEntry.cs index 31b81e23..e8e65322 100644 --- a/BotSharp.Core/Adapters/Dialogflow/DialogflowEntityEntry.cs +++ b/BotSharp.Core/Adapters/Dialogflow/DialogflowEntityEntry.cs @@ -14,7 +14,7 @@ namespace BotSharp.Core.Adapters.Dialogflow public List RawSynonyms { get; set; } - public List Synonyms { get; set; } + public List Synonyms { get; set; } public DialogflowEntityEntry() { diff --git a/BotSharp.Core/Agents/Agent.cs b/BotSharp.Core/Agents/Agent.cs index 79617723..651187f9 100644 --- a/BotSharp.Core/Agents/Agent.cs +++ b/BotSharp.Core/Agents/Agent.cs @@ -53,7 +53,7 @@ namespace BotSharp.Core.Agents [ForeignKey("AgentId")] [JsonProperty("entity_types")] - public List Entities { get; set; } + public List Entities { get; set; } public String Birthday { diff --git a/BotSharp.Core/Agents/AgentExtension.cs b/BotSharp.Core/Agents/AgentDriver.cs similarity index 80% rename from BotSharp.Core/Agents/AgentExtension.cs rename to BotSharp.Core/Agents/AgentDriver.cs index cceebb39..fe37d28d 100644 --- a/BotSharp.Core/Agents/AgentExtension.cs +++ b/BotSharp.Core/Agents/AgentDriver.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Adapters.Rasa; +using BotSharp.Core.Engines; using BotSharp.Core.Entities; using BotSharp.Core.Expressions; using BotSharp.Core.Intents; @@ -12,22 +13,49 @@ using System.Text; namespace BotSharp.Core.Agents { - public static class AgentExtension + public static class AgentDriver { - /// - /// Get agent header row from Agent table - /// - /// - /// - /// - public static Agent Agent(this Database dc, string agentId) + public static Agent LoadAgent(this IBotEngine engine, Database dc, AIConfiguration aiConfig) { - return dc.Table().Find(agentId); + return dc.Table() + .Include(x => x.Intents).ThenInclude(x => x.Contexts) + .Include(x => x.Entities).ThenInclude(x => x.Entries).ThenInclude(x => x.Synonyms) + .FirstOrDefault(x => x.ClientAccessToken == aiConfig.ClientAccessToken || x.DeveloperAccessToken == aiConfig.ClientAccessToken); } - public static String CreateEntity(this Agent agent, Entity entity, Database dc) + /// + /// Restore a agent instance from backup json files + /// + /// + /// + /// + public static Agent RestoreAgent(this IBotEngine engine, IAgentImporter importer, String agentId, string dataDir) { - return entity.Id; + // Load agent summary + var agent = importer.LoadAgent(agentId, dataDir); + + // Load agent entities + importer.LoadEntities(agent, dataDir); + + // Load agent intents + importer.LoadIntents(agent, dataDir); + + return agent; + } + + public static String SaveAgent(this Agent agent, Database dc) + { + var existedAgent = dc.Table().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name); + if (existedAgent == null) + { + dc.Table().Add(agent); + return agent.Id; + } + else + { + agent.Id = existedAgent.Id; + return existedAgent.Id; + } } public static RasaTrainingData GrabCorpus(this Agent agent, Database dc) @@ -77,9 +105,9 @@ namespace BotSharp.Core.Agents // assemble entity synonmus if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text)) { - var allSynonyms = (from e in dc.Table() + var allSynonyms = (from e in dc.Table() join ee in dc.Table() on e.Id equals ee.EntityId - join ees in dc.Table() on ee.Id equals ees.EntityEntryId + join ees in dc.Table() on ee.Id equals ees.EntityEntryId where e.Name == x.Alias && ee.Value == x.Text & ees.Synonym != x.Text select ees.Synonym).ToList(); @@ -163,9 +191,9 @@ namespace BotSharp.Core.Agents // assemble entity synonmus if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text)) { - var allSynonyms = (from e in dc.Table() + var allSynonyms = (from e in dc.Table() join ee in dc.Table() on e.Id equals ee.EntityId - join ees in dc.Table() on ee.Id equals ees.EntityEntryId + join ees in dc.Table() on ee.Id equals ees.EntityEntryId where e.Name == x.Alias && ee.Value == x.Text & ees.Synonym != x.Text select ees.Synonym ).ToList(); diff --git a/BotSharp.Core/Engines/AgentImporterInDialogflow.cs b/BotSharp.Core/Engines/AgentImporterInDialogflow.cs index ff8301d0..b715e0a9 100644 --- a/BotSharp.Core/Engines/AgentImporterInDialogflow.cs +++ b/BotSharp.Core/Engines/AgentImporterInDialogflow.cs @@ -29,7 +29,7 @@ namespace BotSharp.Core.Engines public void LoadEntities(Agent agent, string agentDir) { - agent.Entities = new List(); + agent.Entities = new List(); string entityDir = $"{agentDir}{Path.DirectorySeparatorChar}Dialogflow{Path.DirectorySeparatorChar}{agent.Name}{Path.DirectorySeparatorChar}entities"; if (!Directory.Exists(entityDir)) return; @@ -50,13 +50,13 @@ namespace BotSharp.Core.Engines string entriesJson = File.ReadAllText($"{entriesFileName}"); entriesJson = entriesJson.Replace("\"synonyms\":", "\"rawSynonyms\":"); entity.Entries = JsonConvert.DeserializeObject>(entriesJson); - entity.Entries.ForEach(x => x.Synonyms = x.RawSynonyms.Select(s => new EntityEntrySynonym + entity.Entries.ForEach(x => x.Synonyms = x.RawSynonyms.Select(s => new EntrySynonym { Synonym = s }).ToList()); } - agent.Entities.Add(entity.ToObject()); + agent.Entities.Add(entity.ToObject()); } }); } diff --git a/BotSharp.Core/Engines/IBotEngine.cs b/BotSharp.Core/Engines/IBotEngine.cs new file mode 100644 index 00000000..77bec918 --- /dev/null +++ b/BotSharp.Core/Engines/IBotEngine.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public interface IBotEngine + { + } +} diff --git a/BotSharp.Core/Engines/RasaAi.cs b/BotSharp.Core/Engines/RasaAi.cs index 0f2555be..55f4edeb 100644 --- a/BotSharp.Core/Engines/RasaAi.cs +++ b/BotSharp.Core/Engines/RasaAi.cs @@ -17,7 +17,7 @@ namespace BotSharp.Core.Engines /// /// Rasa nlu 0.11.x /// - public class RasaAi + public class RasaAi : IBotEngine { public Database dc { get; set; } public AIConfiguration AiConfig { get; set; } @@ -35,61 +35,8 @@ namespace BotSharp.Core.Engines this.dc = dc; AiConfig = aiConfig; - agent = LoadAgent(); + agent = this.LoadAgent(dc, aiConfig); aiConfig.DevMode = agent.DeveloperAccessToken == aiConfig.ClientAccessToken; } - - /// - /// Restore a agent instance from backup json files - /// - /// - /// - /// - public Agent RestoreAgent(IAgentImporter importer, String agentId, string dataDir) - { - // Load agent summary - agent = importer.LoadAgent(agentId, dataDir); - - // Load agent entities - importer.LoadEntities(agent, dataDir); - - // Load agent intents - importer.LoadIntents(agent, dataDir); - - return agent; - } - - /// - /// Dump agent train data to json file - /// - /// - /// - public bool DumpAgent(String agentId) - { - return true; - } - - public Agent LoadAgent() - { - return dc.Table() - .Include(x => x.Intents).ThenInclude(x => x.Contexts) - .Include(x => x.Entities).ThenInclude(x => x.Entries).ThenInclude(x => x.Synonyms) - .FirstOrDefault(x => x.ClientAccessToken == AiConfig.ClientAccessToken || x.DeveloperAccessToken == AiConfig.ClientAccessToken); - } - - public String SaveAgent(Agent agent) - { - var existedAgent = dc.Table().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name); - if (existedAgent == null) - { - dc.Table().Add(agent); - return agent.Id; - } - else - { - agent.Id = existedAgent.Id; - return existedAgent.Id; - } - } } } diff --git a/BotSharp.Core/Entities/EntityEntry.cs b/BotSharp.Core/Entities/EntityEntry.cs index 95abbaf4..5fdbc062 100644 --- a/BotSharp.Core/Entities/EntityEntry.cs +++ b/BotSharp.Core/Entities/EntityEntry.cs @@ -18,6 +18,6 @@ namespace BotSharp.Core.Entities public String Value { get; set; } [ForeignKey("EntityEntryId")] - public List Synonyms { get; set; } + public List Synonyms { get; set; } } } diff --git a/BotSharp.Core/Entities/EntityEntryDriver.cs b/BotSharp.Core/Entities/EntityEntryDriver.cs new file mode 100644 index 00000000..620982bc --- /dev/null +++ b/BotSharp.Core/Entities/EntityEntryDriver.cs @@ -0,0 +1,28 @@ +using BotSharp.Core.Agents; +using EntityFrameworkCore.BootKit; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Entities +{ + public static class EntityEntryDriver + { + public static void CreateEntity(this Agent agent, Database dc, EntityType entity) + { + dc.Table().Add(entity); + } + + public static void DeleteEntity(this Agent agent, Database dc, string entityId) + { + var entity = dc.Table().Find(entityId); + dc.Table().Remove(entity); + } + + public static void UpdateEntity(this Agent agent, Database dc, EntityType entity) + { + var oldEntity = dc.Table().Find(entity.Id); + oldEntity.Name = entity.Name; + } + } +} diff --git a/BotSharp.Core/Entities/Entity.cs b/BotSharp.Core/Entities/EntityType.cs similarity index 91% rename from BotSharp.Core/Entities/Entity.cs rename to BotSharp.Core/Entities/EntityType.cs index 6d5eaf2e..68a5d48e 100644 --- a/BotSharp.Core/Entities/Entity.cs +++ b/BotSharp.Core/Entities/EntityType.cs @@ -7,8 +7,8 @@ using System.Text; namespace BotSharp.Core.Entities { - [Table("Bot_Entity")] - public class Entity : DbRecord, IDbRecord + [Table("Bot_EntityType")] + public class EntityType : DbRecord, IDbRecord { [Required] [StringLength(36)] diff --git a/BotSharp.Core/Entities/EntityTypeDriver.cs b/BotSharp.Core/Entities/EntityTypeDriver.cs new file mode 100644 index 00000000..301da68f --- /dev/null +++ b/BotSharp.Core/Entities/EntityTypeDriver.cs @@ -0,0 +1,41 @@ +using BotSharp.Core.Agents; +using EntityFrameworkCore.BootKit; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Core.Entities +{ + public static class EntityTypeDriver + { + public static string CreateEntityType(this Agent agent, Database dc, EntityType entityType) + { + if (dc.Table().Any(x => x.Name == entityType.Name && x.AgentId == agent.Id)) return agent.Id; + + dc.Table().Add(entityType); + + return entityType.Id; + } + + public static void UpdateEntityType(this Agent agent, Database dc, String entityTypeId, EntityType entityType) + { + var existedEntityType = dc.Table().Find(entityTypeId); + if (entityType == null) return; + + existedEntityType.Name = entityType.Name; + existedEntityType.Description = entityType.Description; + existedEntityType.Color = entityType.Color; + existedEntityType.IsEnum = entityType.IsEnum; + existedEntityType.UpdatedTime = DateTime.UtcNow; + } + + public static void DeleteEntityType(this Agent agent, Database dc, String entityTypeId) + { + var entityType = dc.Table().FirstOrDefault(x => x.Id == entityTypeId); + if (entityType == null) return; + + dc.Table().Remove(entityType); + } + } +} diff --git a/BotSharp.Core/Entities/EntityTypeExtension.cs b/BotSharp.Core/Entities/EntityTypeExtension.cs deleted file mode 100644 index 12d6fde7..00000000 --- a/BotSharp.Core/Entities/EntityTypeExtension.cs +++ /dev/null @@ -1,29 +0,0 @@ -using BotSharp.Core.Agents; -using EntityFrameworkCore.BootKit; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace BotSharp.Core.Entities -{ - public static class EntityTypeExtension - { - public static string CreateEntityType(this Agent agent, Database dc, Entity entityType) - { - if (dc.Table().Any(x => x.Name == entityType.Name && x.AgentId == agent.Id)) return agent.Id; - - dc.Table().Add(entityType); - - return entityType.Id; - } - - public static void DeleteEntityType(this Agent agent, Database dc, String entityTypeId) - { - var entityType = dc.Table().FirstOrDefault(x => x.Id == entityTypeId); - if (entityType == null) return; - - dc.Table().Remove(entityType); - } - } -} diff --git a/BotSharp.Core/Entities/EntityEntrySynonym.cs b/BotSharp.Core/Entities/EntrySynonym.cs similarity index 88% rename from BotSharp.Core/Entities/EntityEntrySynonym.cs rename to BotSharp.Core/Entities/EntrySynonym.cs index e9a5cbe5..1b00f295 100644 --- a/BotSharp.Core/Entities/EntityEntrySynonym.cs +++ b/BotSharp.Core/Entities/EntrySynonym.cs @@ -8,7 +8,7 @@ using System.Text; namespace BotSharp.Core.Entities { [Table("Bot_EntityEntrySynonym")] - public class EntityEntrySynonym : DbRecord, IDbRecord + public class EntrySynonym : DbRecord, IDbRecord { [Required] [StringLength(36)] diff --git a/BotSharp.Core/Entities/EntrySynonymDriver.cs b/BotSharp.Core/Entities/EntrySynonymDriver.cs new file mode 100644 index 00000000..9a720fe9 --- /dev/null +++ b/BotSharp.Core/Entities/EntrySynonymDriver.cs @@ -0,0 +1,17 @@ +using EntityFrameworkCore.BootKit; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Entities +{ + public static class EntrySynonymDriver + { + public static void CreateEntitySynonym(this EntityType agent, Database dc, EntrySynonym entity) + { + + } + + + } +} diff --git a/BotSharp.Core/Models/QuestionMetadata.cs b/BotSharp.Core/Models/QuestionMetadata.cs index bbcb0ece..91af83cf 100644 --- a/BotSharp.Core/Models/QuestionMetadata.cs +++ b/BotSharp.Core/Models/QuestionMetadata.cs @@ -19,6 +19,6 @@ namespace BotSharp.Core.Models internal string SessionId { get; set; } [JsonProperty("entities")] - public List Entities { get; set; } + public List Entities { get; set; } } } diff --git a/BotSharp.Core/Models/RequestExtras.cs b/BotSharp.Core/Models/RequestExtras.cs index 08a66e42..ef321396 100644 --- a/BotSharp.Core/Models/RequestExtras.cs +++ b/BotSharp.Core/Models/RequestExtras.cs @@ -9,7 +9,7 @@ namespace BotSharp.Core.Models { public List Contexts { get; set; } - public List Entities { get; set; } + public List Entities { get; set; } public bool HasContexts { @@ -40,7 +40,7 @@ namespace BotSharp.Core.Models { } - public RequestExtras(List contexts, List entities) + public RequestExtras(List contexts, List entities) { this.Contexts = contexts; this.Entities = entities; diff --git a/BotSharp.UnitTest/AgentTest.cs b/BotSharp.UnitTest/AgentTest.cs index 34ff01f1..7a3f11f2 100644 --- a/BotSharp.UnitTest/AgentTest.cs +++ b/BotSharp.UnitTest/AgentTest.cs @@ -25,7 +25,7 @@ namespace BotSharp.UnitTest }; var rasa = new RasaAi(dc); - int row = dc.DbTran(() => rasa.SaveAgent(agent)); + int row = dc.DbTran(() => rasa.agent.SaveAgent(dc)); } [TestMethod] @@ -39,7 +39,7 @@ namespace BotSharp.UnitTest }; var rasa = new RasaAi(dc); - int row = dc.DbTran(() => rasa.SaveAgent(agent)); + int row = dc.DbTran(() => rasa.agent.SaveAgent(dc)); } [TestMethod] @@ -54,8 +54,9 @@ namespace BotSharp.UnitTest agent.ClientAccessToken = BOT_CLIENT_TOKEN; agent.DeveloperAccessToken = BOT_DEVELOPER_TOKEN; agent.UserId = Guid.NewGuid().ToString(); + rasa.agent = agent; - int row = dc.DbTran(() => rasa.SaveAgent(agent)); + int row = dc.DbTran(() => rasa.agent.SaveAgent(dc)); } [TestMethod] @@ -65,7 +66,7 @@ namespace BotSharp.UnitTest config.SessionId = Guid.NewGuid().ToString(); var rasa = new RasaAi(dc, config); - rasa.agent = rasa.LoadAgent(); + rasa.agent = rasa.LoadAgent(dc, config); string msg = rasa.Train(dc); Assert.IsTrue(!String.IsNullOrEmpty(msg)); diff --git a/BotSharp.UnitTest/TestEssential.cs b/BotSharp.UnitTest/TestEssential.cs index eff7a16e..6a48021a 100644 --- a/BotSharp.UnitTest/TestEssential.cs +++ b/BotSharp.UnitTest/TestEssential.cs @@ -13,7 +13,7 @@ namespace BotSharp.UnitTest public static String BOT_ID = "fd9f1b29-fed8-4c68-8fda-69ab463da126"; public static String BOT_CLIENT_TOKEN = "23a53c46d6244840bbb10c89c171d299"; public static String BOT_DEVELOPER_TOKEN = "d86103f446d049ff8d5f506e8dfe5f3f"; - public static String BOT_NAME = "Voicebot"; + public static String BOT_NAME = "VirtualAssistant"; protected Database dc { get; set; } protected string contentRoot;