From e4fe4b2a1e00c499ff370cefb6663b46af248755 Mon Sep 17 00:00:00 2001 From: "haiping008@gmail.com" Date: Thu, 21 Jun 2018 14:14:06 -0500 Subject: [PATCH 1/3] fix missing parameter promote --- BotSharp.Core/Engines/Rasa/RasaAi.cs | 20 ++++++++++++--- BotSharp.Core/Engines/RequestExtension.cs | 30 +++++++++++++++++------ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/BotSharp.Core/Engines/Rasa/RasaAi.cs b/BotSharp.Core/Engines/Rasa/RasaAi.cs index 9856e5e6..42d23ab2 100644 --- a/BotSharp.Core/Engines/Rasa/RasaAi.cs +++ b/BotSharp.Core/Engines/Rasa/RasaAi.cs @@ -85,7 +85,6 @@ namespace BotSharp.Core.Engines public void TrainWithContexts() { var corpus = agent.GrabCorpus(dc); - var client = new RestClient($"{Database.Configuration.GetSection("Rasa:Nlu").Value}"); var contextHashs = corpus.UserSays @@ -95,10 +94,25 @@ namespace BotSharp.Core.Engines contextHashs.ForEach(ctx => { + var common_examples = corpus.UserSays.Where(x => x.ContextHash == ctx || x.ContextHash == Guid.Empty.ToString("N")).ToList(); + + // assemble entity and synonyms + var usedEntities = new List(); + common_examples.ForEach(x => + { + if (x.Entities != null) + { + usedEntities.AddRange(x.Entities.Select(y => y.Entity)); + } + }); + usedEntities = usedEntities.Distinct().ToList(); + + var entity_synonyms = corpus.Entities.Where(x => usedEntities.Contains(x.EntityType)).ToList(); + var data = new RasaTrainingData { - Entities = corpus.Entities, - UserSays = corpus.UserSays.Where(x => x.ContextHash == ctx).ToList() + Entities = entity_synonyms, + UserSays = common_examples }; // meet minimal requirement diff --git a/BotSharp.Core/Engines/RequestExtension.cs b/BotSharp.Core/Engines/RequestExtension.cs index b271f9ea..7ca82f5e 100644 --- a/BotSharp.Core/Engines/RequestExtension.cs +++ b/BotSharp.Core/Engines/RequestExtension.cs @@ -44,10 +44,10 @@ namespace BotSharp.Core.Engines aiResponse.Timestamp = DateTime.UtcNow; var intentResponse = HandleIntentPerContextIn(rasa, request, result.Data); - bool missedRequiredField = HandleParameter(rasa.agent, intentResponse, response, request); + HandleParameter(rasa.agent, intentResponse, response, request); HandleMessage(intentResponse); - + aiResponse.Result = new AIResponseResult { Source = "agent", @@ -137,7 +137,7 @@ namespace BotSharp.Core.Engines var intent = (dc.Table().Where(x => x.AgentId == rasa.agent.Id && x.Name == response.Intent.Name) .Include(x => x.Responses).ThenInclude(x => x.Contexts) - .Include(x => x.Responses).ThenInclude(x => x.Parameters) + .Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts) .Include(x => x.Responses).ThenInclude(x => x.Messages)).First(); var intentResponse = ArrayHelper.GetRandom(intent.Responses); @@ -155,9 +155,9 @@ namespace BotSharp.Core.Engines /// /// /// Required field is missed - private static bool HandleParameter(Agent agent, IntentResponse intentResponse, RasaResponse response, AIRequest request) + private static void HandleParameter(Agent agent, IntentResponse intentResponse, RasaResponse response, AIRequest request) { - if (intentResponse == null) return false; + if (intentResponse == null) return; intentResponse.Parameters.ForEach(p => { string query = request.Query.First(); @@ -193,15 +193,29 @@ namespace BotSharp.Core.Engines } } }); - - return intentResponse.Parameters.Any(x => x.Required && String.IsNullOrEmpty(x.Value)); } private static void HandleMessage(IntentResponse intentResponse) { if (intentResponse == null) return; - intentResponse.Messages = intentResponse.Messages.OrderBy(x => x.UpdatedTime).ToList(); + var missingRequiredParameter = intentResponse.Parameters.FirstOrDefault(x => x.Required && String.IsNullOrEmpty(x.Value)); + if (missingRequiredParameter != null) + { + intentResponse.Messages = new List { + new IntentResponseMessage { + Type = AIResponseMessageType.Text, + Speech = ArrayHelper.GetRandom(missingRequiredParameter.Prompts).Prompt, + IntentResponseId = intentResponse.Id, + UpdatedTime = DateTime.UtcNow + } + }; + } + else + { + intentResponse.Messages = intentResponse.Messages.OrderBy(x => x.UpdatedTime).ToList(); + } + intentResponse.Messages.ToList() .ForEach(msg => { From d2c4fa3f363f03de7e4a6e3fd258929dd8bd30fc Mon Sep 17 00:00:00 2001 From: Bo Peng Date: Thu, 21 Jun 2018 14:54:36 -0500 Subject: [PATCH 2/3] SpaCy Entity Recognizer --- .../Engines/SpaCy/SpaCyEntityRecognizer.cs | 44 +++++++++++++++++++ BotSharp.UnitTest/Settings/settings.bot.json | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs new file mode 100644 index 00000000..d0fd5587 --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs @@ -0,0 +1,44 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using RestSharp; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines.SpaCy +{ + public class SpaCyEntityRecognizer : INlpPipeline + { + List entitiesInTrainingSet = new List(); + public IConfiguration Configuration { get; set; } + + public bool Process(Agent agent, JObject data) + { + String modelPath = "./entity_rec_output"; + String newModelName = "test"; + String outputDir = "./entity_rec_output2"; + int iterTimes = 20; + + agent.Entities.ForEach(entity => entitiesInTrainingSet.Add(entity.Name)); + var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); + var request = new RestRequest("entityrecognizer", Method.POST); + request.RequestFormat = DataFormat.Json; + + request.AddParameter("application/json", JsonConvert.SerializeObject(new { ModelPath = modelPath, NewModelName = newModelName, OutputDir = outputDir, IterTimes = iterTimes, EntitiesInTrainingSet = entitiesInTrainingSet }), ParameterType.RequestBody); + + var response = client.Execute(request); + + data["EntityModelTrained"] = response.Data.EntityModelTrained; + + return true; + } + } + + public class Result + { + public Boolean EntityModelTrained { get; set; } + } +} diff --git a/BotSharp.UnitTest/Settings/settings.bot.json b/BotSharp.UnitTest/Settings/settings.bot.json index 1e115890..798a07b3 100644 --- a/BotSharp.UnitTest/Settings/settings.bot.json +++ b/BotSharp.UnitTest/Settings/settings.bot.json @@ -9,6 +9,6 @@ "SpaCyProvider": { "Url": "http://10.2.21.200:5005" }, - "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer" + "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer, SpaCyEntityRecognizer" } } From 253a99275fb180194447c5dbca2e96dc7bf6b2ce Mon Sep 17 00:00:00 2001 From: Deep-Blue-2013 Date: Thu, 21 Jun 2018 16:30:06 -0500 Subject: [PATCH 3/3] fix dir separator in UnitTest for Linux env --- BotSharp.UnitTest/AgentTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/BotSharp.UnitTest/AgentTest.cs b/BotSharp.UnitTest/AgentTest.cs index 766f0723..7853cbab 100644 --- a/BotSharp.UnitTest/AgentTest.cs +++ b/BotSharp.UnitTest/AgentTest.cs @@ -7,6 +7,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; @@ -50,7 +51,7 @@ namespace BotSharp.UnitTest var rasa = new RasaAi(dc); var importer = new AgentImporterInDialogflow(); - string dataDir = $"{Database.ContentRootPath}\\App_Data\\DbInitializer\\Agents\\"; + string dataDir = $"{Database.ContentRootPath}App_Data{Path.DirectorySeparatorChar}DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}"; var agent = rasa.RestoreAgent(importer, BOT_NAME, dataDir); agent.Id = BOT_ID; agent.ClientAccessToken = BOT_CLIENT_TOKEN;