From a5d90db1553da0cc284353731d3a2bbe7cbba95a Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 1 Oct 2018 08:36:39 -0500 Subject: [PATCH] agent, intent, entity persist completed. --- BotSharp.Core/AgentStorageInMemory.cs | 9 +++++++ BotSharp.Core/AgentStorageInRedis.cs | 27 +++++++++++++++---- BotSharp.Core/Agents/Agent.cs | 1 + BotSharp.Core/Engines/BotEngineBase.cs | 1 + .../Engines/BotSharp/BotSharpCRFNer.cs | 1 + .../Dialogflow/AgentImporterInDialogflow.cs | 1 + .../Engines/Rasa/AgentImporterInRasa.cs | 1 + BotSharp.Core/Engines/Rasa/RasaAi.cs | 2 +- .../Engines/Rasa/RasaIntentExpression.cs | 1 + .../Engines/Rasa/RasaIntentExpressionPart.cs | 1 + .../Engines/Rasa/RasaTrainingEntity.cs | 6 ++--- .../Engines/Sebis/AgentImporterInSebis.cs | 1 + .../Engines/Sebis/SebisIntentExpression.cs | 1 + .../Sebis/SebisIntentExpressionPart.cs | 1 + .../IAgentStorage.cs | 6 +++++ .../IPlatformBuilder.cs | 6 +++-- .../TrainingCorpus.cs | 8 +++--- .../TrainingEntity.cs | 6 ++--- .../TrainingIntentExpression.cs | 6 ++--- .../TrainingIntentExpressionPart.cs | 2 +- BotSharp.WebHost/Settings/db.json | 3 ++- Platform.Articulate/ArticulateAi.cs | 9 +++++-- .../Controllers/AgentController.cs | 2 +- .../Controllers/ParseControllercs.cs | 4 +-- .../Controllers/TrainController.cs | 4 ++- .../Models/IntentExampleModel.cs | 1 + 26 files changed, 80 insertions(+), 31 deletions(-) rename {BotSharp.Core/Engines => BotSharp.Platform.Models}/TrainingCorpus.cs (66%) rename {BotSharp.Core/Engines => BotSharp.Platform.Models}/TrainingEntity.cs (54%) rename {BotSharp.Core/Engines => BotSharp.Platform.Models}/TrainingIntentExpression.cs (78%) rename {BotSharp.Core/Engines => BotSharp.Platform.Models}/TrainingIntentExpressionPart.cs (91%) diff --git a/BotSharp.Core/AgentStorageInMemory.cs b/BotSharp.Core/AgentStorageInMemory.cs index a25b3d46..6c5d1368 100644 --- a/BotSharp.Core/AgentStorageInMemory.cs +++ b/BotSharp.Core/AgentStorageInMemory.cs @@ -54,6 +54,15 @@ namespace BotSharp.Core return true; } + public int PurgeAllAgents() + { + int count = agents.Count; + + agents.Clear(); + + return count; + } + public List Query() { return agents.Select(x => x.Value).ToList(); diff --git a/BotSharp.Core/AgentStorageInRedis.cs b/BotSharp.Core/AgentStorageInRedis.cs index bc636f06..4d00bb7d 100644 --- a/BotSharp.Core/AgentStorageInRedis.cs +++ b/BotSharp.Core/AgentStorageInRedis.cs @@ -1,6 +1,7 @@ using BotSharp.Platform.Abstraction; using BotSharp.Platform.Models; using CSRedis; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using System; using System.Collections.Generic; @@ -13,12 +14,19 @@ namespace BotSharp.Core where TAgent : AgentBase { private static CSRedisClient csredis; + private string prefix = String.Empty; public AgentStorageInRedis() { if (csredis == null) { - csredis = new CSRedisClient("127.0.0.1:6379,defaultDatabase=botsharp,poolsize=50,ssl=false,writeBuffer=10240,prefix=agent."); + IConfiguration config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration"); + var db = config.GetSection("Database:Default").Value; + var dbConnStr = config.GetSection($"Database:ConnectionStrings:{db}").Value; + + prefix = dbConnStr.Split(',').First(x => x.StartsWith("prefix=")).Split('=')[1]; + + csredis = new CSRedisClient(dbConnStr); } } @@ -39,10 +47,10 @@ namespace BotSharp.Core { var agents = new List(); - var keys = csredis.Keys("agent.*"); + var keys = csredis.Keys($"{prefix }*"); foreach (string key in keys) { - var data = csredis.Get(key.Split('.')[1]); + var data = csredis.Get(key.Substring(prefix.Length)); var agent = JsonConvert.DeserializeObject(data); if(agent.Name == agentName) @@ -66,14 +74,23 @@ namespace BotSharp.Core return true; } + public int PurgeAllAgents() + { + var keys = csredis.Keys($"{prefix}*"); + + csredis.Remove(keys.Select(x => x.Substring(prefix.Length)).ToArray()); + + return keys.Count(); + } + public List Query() { var agents = new List(); - var keys = csredis.Keys("agent.*"); + var keys = csredis.Keys($"{prefix}*"); foreach (string key in keys) { - var data = csredis.Get(key.Split('.')[1]); + var data = csredis.Get(key.Substring(prefix.Length)); var agent = JsonConvert.DeserializeObject(data); agents.Add(agent); } diff --git a/BotSharp.Core/Agents/Agent.cs b/BotSharp.Core/Agents/Agent.cs index e83027f5..42d52878 100644 --- a/BotSharp.Core/Agents/Agent.cs +++ b/BotSharp.Core/Agents/Agent.cs @@ -1,6 +1,7 @@ using BotSharp.Core.Engines; using BotSharp.Core.Entities; using BotSharp.Core.Intents; +using BotSharp.Platform.Models; using EntityFrameworkCore.BootKit; using Newtonsoft.Json; using System; diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index 8c989c74..5c5fdbac 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -5,6 +5,7 @@ using BotSharp.Core.Entities; using BotSharp.Core.Intents; using BotSharp.Core.Models; using BotSharp.Models.NLP; +using BotSharp.Platform.Models; using DotNetToolkit; using EntityFrameworkCore.BootKit; using Microsoft.EntityFrameworkCore; diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs index 525f9dd6..09d8bedd 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs @@ -13,6 +13,7 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using BotSharp.Platform.Models; namespace BotSharp.Core.Engines.BotSharp { diff --git a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs index 593429c4..6758fd36 100644 --- a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs +++ b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs @@ -8,6 +8,7 @@ using BotSharp.Core.Agents; using BotSharp.Core.Entities; using BotSharp.Core.Intents; using BotSharp.Core.Models; +using BotSharp.Platform.Models; using DotNetToolkit; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs index 7975823f..bc0d1a54 100644 --- a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs +++ b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs @@ -7,6 +7,7 @@ using BotSharp.Core.Agents; using BotSharp.Core.Entities; using BotSharp.Core.Intents; using BotSharp.Core.Models; +using BotSharp.Platform.Models; using DotNetToolkit; using Newtonsoft.Json; diff --git a/BotSharp.Core/Engines/Rasa/RasaAi.cs b/BotSharp.Core/Engines/Rasa/RasaAi.cs index fb650bcd..2480252a 100644 --- a/BotSharp.Core/Engines/Rasa/RasaAi.cs +++ b/BotSharp.Core/Engines/Rasa/RasaAi.cs @@ -129,7 +129,7 @@ namespace BotSharp.Core.Engines }); usedEntities = usedEntities.Distinct().ToList(); - var entity_synonyms = corpus.Entities.Where(x => usedEntities.Contains(x.EntityType)).ToList(); + var entity_synonyms = corpus.Entities.Where(x => usedEntities.Contains(x.Entity)).ToList(); var data = new RasaTrainingData { diff --git a/BotSharp.Core/Engines/Rasa/RasaIntentExpression.cs b/BotSharp.Core/Engines/Rasa/RasaIntentExpression.cs index 01c4863e..9bdda391 100644 --- a/BotSharp.Core/Engines/Rasa/RasaIntentExpression.cs +++ b/BotSharp.Core/Engines/Rasa/RasaIntentExpression.cs @@ -1,5 +1,6 @@ using BotSharp.Core.Adapters.Rasa; using BotSharp.Core.Engines; +using BotSharp.Platform.Models; using Newtonsoft.Json; using System; using System.Collections.Generic; diff --git a/BotSharp.Core/Engines/Rasa/RasaIntentExpressionPart.cs b/BotSharp.Core/Engines/Rasa/RasaIntentExpressionPart.cs index 4925df0e..a5bb1e2e 100644 --- a/BotSharp.Core/Engines/Rasa/RasaIntentExpressionPart.cs +++ b/BotSharp.Core/Engines/Rasa/RasaIntentExpressionPart.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Engines; +using BotSharp.Platform.Models; using System; using System.Collections.Generic; using System.Text; diff --git a/BotSharp.Core/Engines/Rasa/RasaTrainingEntity.cs b/BotSharp.Core/Engines/Rasa/RasaTrainingEntity.cs index 9b564a18..38a9c174 100644 --- a/BotSharp.Core/Engines/Rasa/RasaTrainingEntity.cs +++ b/BotSharp.Core/Engines/Rasa/RasaTrainingEntity.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Engines; +using BotSharp.Platform.Models; using Newtonsoft.Json; using System; using System.Collections.Generic; @@ -8,10 +9,7 @@ namespace BotSharp.Core.Adapters.Rasa { public sealed class RasaTrainingEntity : TrainingEntity { - [JsonIgnore] - public override String EntityType { get; set; } - [JsonProperty("value")] - public override String EntityValue { get; set; } + public override String Entity { get; set; } } } diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index 4ee9b6d6..a87fcbda 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -9,6 +9,7 @@ using BotSharp.Core.Agents; using BotSharp.Core.Entities; using BotSharp.Core.Intents; using BotSharp.Core.Models; +using BotSharp.Platform.Models; using DotNetToolkit; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/BotSharp.Core/Engines/Sebis/SebisIntentExpression.cs b/BotSharp.Core/Engines/Sebis/SebisIntentExpression.cs index c8af7165..ef7b04b9 100644 --- a/BotSharp.Core/Engines/Sebis/SebisIntentExpression.cs +++ b/BotSharp.Core/Engines/Sebis/SebisIntentExpression.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Text; using BotSharp.Core.Engines; +using BotSharp.Platform.Models; namespace BotSharp.Core.Adapters.Sebis { diff --git a/BotSharp.Core/Engines/Sebis/SebisIntentExpressionPart.cs b/BotSharp.Core/Engines/Sebis/SebisIntentExpressionPart.cs index 85c8dfe5..f3100086 100644 --- a/BotSharp.Core/Engines/Sebis/SebisIntentExpressionPart.cs +++ b/BotSharp.Core/Engines/Sebis/SebisIntentExpressionPart.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Text; using BotSharp.Core.Engines; +using BotSharp.Platform.Models; using Newtonsoft.Json; namespace BotSharp.Core.Adapters.Sebis diff --git a/BotSharp.Platform.Abstraction/IAgentStorage.cs b/BotSharp.Platform.Abstraction/IAgentStorage.cs index 4cdaa18d..f1a89977 100644 --- a/BotSharp.Platform.Abstraction/IAgentStorage.cs +++ b/BotSharp.Platform.Abstraction/IAgentStorage.cs @@ -37,5 +37,11 @@ namespace BotSharp.Platform.Abstraction /// /// List Query(); + + /// + /// Delete agents + /// + /// + int PurgeAllAgents(); } } diff --git a/BotSharp.Platform.Abstraction/IPlatformBuilder.cs b/BotSharp.Platform.Abstraction/IPlatformBuilder.cs index 60d5c6d3..216f28c5 100644 --- a/BotSharp.Platform.Abstraction/IPlatformBuilder.cs +++ b/BotSharp.Platform.Abstraction/IPlatformBuilder.cs @@ -19,11 +19,11 @@ namespace BotSharp.Platform.Abstraction // DialogRequestOptions RequestOptions { get; set; } /// - /// Convert platform specific agent to standard agent format + /// Convert platform specific data to standard training corpus format /// /// /// - StandardAgent StandardizeAgent(TAgent agent); + TrainingCorpus ExtractorCorpus(TAgent agent); /// /// @@ -32,5 +32,7 @@ namespace BotSharp.Platform.Abstraction /// /// bool SaveAgent(TAgent agent); + + bool Train(TrainingCorpus corpus); } } diff --git a/BotSharp.Core/Engines/TrainingCorpus.cs b/BotSharp.Platform.Models/TrainingCorpus.cs similarity index 66% rename from BotSharp.Core/Engines/TrainingCorpus.cs rename to BotSharp.Platform.Models/TrainingCorpus.cs index c2d9c69f..fe9e1d01 100644 --- a/BotSharp.Core/Engines/TrainingCorpus.cs +++ b/BotSharp.Platform.Models/TrainingCorpus.cs @@ -1,14 +1,16 @@ -using BotSharp.Core.Models; -using System; +using System; using System.Collections.Generic; using System.Text; -namespace BotSharp.Core.Engines +namespace BotSharp.Platform.Models { public class TrainingCorpus { public List> UserSays { get; set; } + /// + /// User custom entities + /// public List Entities { get; set; } } } diff --git a/BotSharp.Core/Engines/TrainingEntity.cs b/BotSharp.Platform.Models/TrainingEntity.cs similarity index 54% rename from BotSharp.Core/Engines/TrainingEntity.cs rename to BotSharp.Platform.Models/TrainingEntity.cs index 8c73135a..259255d2 100644 --- a/BotSharp.Core/Engines/TrainingEntity.cs +++ b/BotSharp.Platform.Models/TrainingEntity.cs @@ -2,13 +2,11 @@ using System.Collections.Generic; using System.Text; -namespace BotSharp.Core.Engines +namespace BotSharp.Platform.Models { public class TrainingEntity { - public virtual String EntityType { get; set; } - - public virtual String EntityValue { get; set; } + public virtual String Entity { get; set; } public List Synonyms { get; set; } } diff --git a/BotSharp.Core/Engines/TrainingIntentExpression.cs b/BotSharp.Platform.Models/TrainingIntentExpression.cs similarity index 78% rename from BotSharp.Core/Engines/TrainingIntentExpression.cs rename to BotSharp.Platform.Models/TrainingIntentExpression.cs index 1a8fa034..9e5c16a8 100644 --- a/BotSharp.Core/Engines/TrainingIntentExpression.cs +++ b/BotSharp.Platform.Models/TrainingIntentExpression.cs @@ -1,16 +1,14 @@ -using Newtonsoft.Json; -using System; +using System; using System.Collections.Generic; using System.Text; -namespace BotSharp.Core.Engines +namespace BotSharp.Platform.Models { public class TrainingIntentExpression where TPart : TrainingIntentExpressionPart { public String Text { get; set; } public String Intent { get; set; } - [JsonIgnore] public String ContextHash { get; set; } public List Entities { get; set; } diff --git a/BotSharp.Core/Engines/TrainingIntentExpressionPart.cs b/BotSharp.Platform.Models/TrainingIntentExpressionPart.cs similarity index 91% rename from BotSharp.Core/Engines/TrainingIntentExpressionPart.cs rename to BotSharp.Platform.Models/TrainingIntentExpressionPart.cs index 28739675..87cd9acd 100644 --- a/BotSharp.Core/Engines/TrainingIntentExpressionPart.cs +++ b/BotSharp.Platform.Models/TrainingIntentExpressionPart.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace BotSharp.Core.Engines +namespace BotSharp.Platform.Models { public class TrainingIntentExpressionPart { diff --git a/BotSharp.WebHost/Settings/db.json b/BotSharp.WebHost/Settings/db.json index 68bf73bf..63b7fcae 100644 --- a/BotSharp.WebHost/Settings/db.json +++ b/BotSharp.WebHost/Settings/db.json @@ -1,8 +1,9 @@ { "Database": { - "Default": "Sqlite", + "Default": "Redis", "ConnectionStrings": { "InMemory": "DataSource=:memory:", + "Redis": "127.0.0.1:6379,defaultDatabase=BotSharp,poolsize=50,ssl=false,writeBuffer=10240,prefix=agent_", "Sqlite": "Data Source=|DataDirectory|BotSharp.db;", "SqlServer": "Data Source=(localdb)\\MSSQLLocalDB;Initial Catalog=BotSharp;Integrated Security=True;Connect Timeout=15;Encrypt=False;TrustServerCertificate=True;ApplicationIntent=ReadWrite;MultiSubnetFailover=False" } diff --git a/Platform.Articulate/ArticulateAi.cs b/Platform.Articulate/ArticulateAi.cs index 2aec19bd..d13d55ec 100644 --- a/Platform.Articulate/ArticulateAi.cs +++ b/Platform.Articulate/ArticulateAi.cs @@ -80,7 +80,7 @@ namespace Platform.Articulate return intents; } - public StandardAgent StandardizeAgent(TAgent specificAgent) + public TrainingCorpus ExtractorCorpus(TAgent specificAgent) { var agent1 = specificAgent as AgentModel; @@ -91,7 +91,12 @@ namespace Platform.Articulate Description = agent1.Description }; - return standardAgent; + return new TrainingCorpus(); + } + + public bool Train(TrainingCorpus corpus) + { + throw new NotImplementedException(); } } } diff --git a/Platform.Articulate/Controllers/AgentController.cs b/Platform.Articulate/Controllers/AgentController.cs index 021ed7e2..75310101 100644 --- a/Platform.Articulate/Controllers/AgentController.cs +++ b/Platform.Articulate/Controllers/AgentController.cs @@ -57,7 +57,7 @@ namespace Platform.Articulate.Controllers public List GetAgent() { var results = builder.GetAllAgents(); - var agents = results.Select(x => x as AgentModel).ToList(); + var agents = results.ToList(); return agents; } diff --git a/Platform.Articulate/Controllers/ParseControllercs.cs b/Platform.Articulate/Controllers/ParseControllercs.cs index 2aa7db0b..b30880ca 100644 --- a/Platform.Articulate/Controllers/ParseControllercs.cs +++ b/Platform.Articulate/Controllers/ParseControllercs.cs @@ -9,8 +9,8 @@ namespace Platform.Articulate.Controllers [Route("[controller]")] public class ParseControllercs : ControllerBase { - [HttpGet("/agent/{agentId}/parse")] - public void ParseText([FromRoute] string agentId, [FromQuery] string text) + [HttpGet("/agent/{agentId}/converse")] + public void ParseText([FromRoute] string agentId, [FromQuery] string text, [FromQuery] string sessionId) { } diff --git a/Platform.Articulate/Controllers/TrainController.cs b/Platform.Articulate/Controllers/TrainController.cs index da6b32af..13ade32d 100644 --- a/Platform.Articulate/Controllers/TrainController.cs +++ b/Platform.Articulate/Controllers/TrainController.cs @@ -23,7 +23,9 @@ namespace Platform.Articulate.Controllers { var agent = builder.GetAgentById(agentId); - agent.Status = "ready"; + + + agent.Status = "Ready"; return agent; } diff --git a/Platform.Articulate/Models/IntentExampleModel.cs b/Platform.Articulate/Models/IntentExampleModel.cs index 825d9eaa..d3cbf398 100644 --- a/Platform.Articulate/Models/IntentExampleModel.cs +++ b/Platform.Articulate/Models/IntentExampleModel.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Engines; +using BotSharp.Platform.Models; using System; using System.Collections.Generic; using System.Text;