Merge branch 'master' of https://github.com/Oceania2018/BotSharp

BotSharp.Core/Engines/Rasa/RasaAi.cs

@@ -85,7 +85,6 @@ namespace BotSharp.Core.Engines
         public void TrainWithContexts()
         {
             var corpus = agent.GrabCorpus(dc);
-
             var client = new RestClient($"{Database.Configuration.GetSection("Rasa:Nlu").Value}");
 
             var contextHashs = corpus.UserSays
@@ -95,10 +94,25 @@ namespace BotSharp.Core.Engines
 
             contextHashs.ForEach(ctx =>
             {
+                var common_examples = corpus.UserSays.Where(x => x.ContextHash == ctx || x.ContextHash == Guid.Empty.ToString("N")).ToList();
+
+                // assemble entity and synonyms
+                var usedEntities = new List<String>();
+                common_examples.ForEach(x =>
+                {
+                    if (x.Entities != null)
+                    {
+                        usedEntities.AddRange(x.Entities.Select(y => y.Entity));
+                    }
+                });
+                usedEntities = usedEntities.Distinct().ToList();
+
+                var entity_synonyms = corpus.Entities.Where(x => usedEntities.Contains(x.EntityType)).ToList();
+
                 var data = new RasaTrainingData
                 {
-                    Entities = corpus.Entities,
-                    UserSays = corpus.UserSays.Where(x => x.ContextHash == ctx).ToList()
+                    Entities = entity_synonyms,
+                    UserSays = common_examples
                 };
 
                 // meet minimal requirement

BotSharp.Core/Engines/RequestExtension.cs

@@ -46,10 +46,10 @@ namespace BotSharp.Core.Engines
             aiResponse.Timestamp = DateTime.UtcNow;
 
             var intentResponse = HandleIntentPerContextIn(rasa, request, result.Data);
-            bool missedRequiredField = HandleParameter(rasa.agent, intentResponse, response, request);
+            HandleParameter(rasa.agent, intentResponse, response, request);
 
             HandleMessage(intentResponse);
-
+            
             aiResponse.Result = new AIResponseResult
             {
                 Source = "agent",
@@ -139,7 +139,7 @@ namespace BotSharp.Core.Engines
 
             var intent = (dc.Table<Intent>().Where(x => x.AgentId == rasa.agent.Id && x.Name == response.Intent.Name)
                 .Include(x => x.Responses).ThenInclude(x => x.Contexts)
-                .Include(x => x.Responses).ThenInclude(x => x.Parameters)
+                .Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts)
                 .Include(x => x.Responses).ThenInclude(x => x.Messages)).First();
 
             var intentResponse = ArrayHelper.GetRandom(intent.Responses);
@@ -157,9 +157,9 @@ namespace BotSharp.Core.Engines
         /// <param name="response"></param>
         /// <param name="request"></param>
         /// <returns>Required field is missed</returns>
-        private static bool HandleParameter(Agent agent, IntentResponse intentResponse, RasaResponse response, AIRequest request)
+        private static void HandleParameter(Agent agent, IntentResponse intentResponse, RasaResponse response, AIRequest request)
         {
-            if (intentResponse == null) return false;
+            if (intentResponse == null) return;
 
             intentResponse.Parameters.ForEach(p => {
                 string query = request.Query.First();
@@ -195,15 +195,29 @@ namespace BotSharp.Core.Engines
                     }
                 }
             });
-
-            return intentResponse.Parameters.Any(x => x.Required && String.IsNullOrEmpty(x.Value));
         }
 
         private static void HandleMessage(IntentResponse intentResponse)
         {
             if (intentResponse == null) return;
 
-            intentResponse.Messages = intentResponse.Messages.OrderBy(x => x.UpdatedTime).ToList();
+            var missingRequiredParameter = intentResponse.Parameters.FirstOrDefault(x => x.Required && String.IsNullOrEmpty(x.Value));
+            if (missingRequiredParameter != null)
+            {
+                intentResponse.Messages = new List<IntentResponseMessage> {
+                    new IntentResponseMessage {
+                        Type = AIResponseMessageType.Text,
+                        Speech = ArrayHelper.GetRandom(missingRequiredParameter.Prompts).Prompt,
+                        IntentResponseId = intentResponse.Id,
+                        UpdatedTime = DateTime.UtcNow
+                    }
+                };
+            }
+            else
+            {
+                intentResponse.Messages = intentResponse.Messages.OrderBy(x => x.UpdatedTime).ToList();
+            }
+            
             intentResponse.Messages.ToList()
                 .ForEach(msg =>
                 {

BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs

@@ -0,0 +1,44 @@
+using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using RestSharp;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines.SpaCy
+{
+    public class SpaCyEntityRecognizer : INlpPipeline
+    {
+        List<String> entitiesInTrainingSet = new List<string>();
+        public IConfiguration Configuration { get; set; }
+
+        public bool Process(Agent agent, JObject data)
+        {
+            String modelPath = "./entity_rec_output";
+            String newModelName = "test";
+            String outputDir = "./entity_rec_output2";
+            int iterTimes = 20;
+
+            agent.Entities.ForEach(entity => entitiesInTrainingSet.Add(entity.Name));
+            var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
+            var request = new RestRequest("entityrecognizer", Method.POST);
+            request.RequestFormat = DataFormat.Json;
+
+            request.AddParameter("application/json", JsonConvert.SerializeObject(new { ModelPath = modelPath, NewModelName = newModelName, OutputDir = outputDir, IterTimes = iterTimes, EntitiesInTrainingSet = entitiesInTrainingSet }), ParameterType.RequestBody);
+
+            var response = client.Execute<Result>(request);
+
+            data["EntityModelTrained"] = response.Data.EntityModelTrained;
+
+            return true;
+        }
+    }
+
+    public class Result
+    {
+        public Boolean EntityModelTrained { get; set; }
+    }
+}

BotSharp.UnitTest/AgentTest.cs

@@ -7,6 +7,7 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Text;
 
@@ -50,7 +51,7 @@ namespace BotSharp.UnitTest
             var rasa = new RasaAi(dc);
             var importer = new AgentImporterInDialogflow();
 
-            string dataDir =  $"{Database.ContentRootPath}\\App_Data\\DbInitializer\\Agents\\";
+            string dataDir =  $"{Database.ContentRootPath}App_Data{Path.DirectorySeparatorChar}DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}";
             var agent = rasa.RestoreAgent(importer, BOT_NAME, dataDir);
             agent.Id = BOT_ID;
             agent.ClientAccessToken = BOT_CLIENT_TOKEN;

BotSharp.UnitTest/Settings/settings.bot.json

@@ -9,6 +9,6 @@
     "SpaCyProvider": {
       "Url": "http://10.2.21.200:5005"
     },
-    "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer"
+    "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer, SpaCyEntityRecognizer"
   }
 }