Add SpaCy TextCategorizer component

.gitignore

@@ -291,3 +291,4 @@ __pycache__/
 /Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
 /BotSharp.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
 /BotSharp.UnitTest/App_Data/DbInitializer/Agents
+/BotSharp.UnitTest/App_Data/BotSharp.db

BotSharp.Core/Abstractions/INlpPipeline.cs

@@ -1,4 +1,5 @@
-using Microsoft.Extensions.Configuration;
+using BotSharp.Core.Agents;
+using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -13,6 +14,6 @@ namespace BotSharp.Core.Abstractions
     {
         IConfiguration Configuration { get; set; }
 
-        bool Process(String text, JObject data);
+        bool Process(Agent agent, JObject data);
     }
 }

BotSharp.Core/Engines/BotTrainer.cs

@@ -3,8 +3,11 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using BotSharp.Core.Intents;
 using DotNetToolkit;
 using EntityFrameworkCore.BootKit;
+using Microsoft.EntityFrameworkCore;
 using Newtonsoft.Json.Linq;
 
 namespace BotSharp.Core.Engines
@@ -24,15 +27,26 @@ namespace BotSharp.Core.Engines
             this.config = config;
         }
 
-        public string Train()
+        public string Train(Agent agent)
         {
-            var data = JObject.FromObject(new { });
+            agent.Intents = dc.Table<Intent>()
+                .Include(x => x.Contexts)
+                .Include(x => x.Responses).ThenInclude(x => x.Contexts)
+                .Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts)
+                .Include(x => x.Responses).ThenInclude(x => x.Messages)
+                .Include(x => x.UserSays).ThenInclude(x => x.Data)
+                .Where(x => x.AgentId == agentId)
+                .ToList();
+
+            var data = JObject.FromObject(new
+            {
+            });
 
             // Get NLP Provider
             string providerName = Database.Configuration.GetSection($"{config}:Provider").Value;
             var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpPipeline;
             provider.Configuration = Database.Configuration.GetSection("BotSharpAi");
-            provider.Process("How are you today ?", data);
+            provider.Process(agent, data);
 
 
             // pipe process
@@ -45,9 +59,7 @@ namespace BotSharp.Core.Engines
             {
                 var pipe = TypeHelper.GetInstance(pipeName, Database.Assemblies) as INlpPipeline;
                 pipe.Configuration = provider.Configuration;
-                var tokens = pipe.Process("How are you today ?", data);
+                pipe.Process(agent, data);
-
-
             });
 
 

BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
 using BotSharp.Core.Models;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
@@ -13,14 +14,14 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool Process(string text, JObject data)
+        public bool Process(Agent agent, JObject data)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("entitize", Method.GET);
-            request.AddParameter("text", text);
+            request.AddParameter("text", "");
             var response = client.Execute<Result>(request);
 
-            data.Add("Entities", JToken.FromObject(response.Data.Entities));
+            //data.Add("Entities", JToken.FromObject(response.Data.Entities));
 
             return response.IsSuccessful;
         }

BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs

@@ -1,4 +1,5 @@
 using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
 using RestSharp;
@@ -13,7 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool Process(string text, JObject data)
+        public bool Process(Agent agent, JObject data)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("load", Method.GET);

BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs

@@ -0,0 +1,70 @@
+using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using BotSharp.Core.Intents;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using RestSharp;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.Core.Engines.SpaCy
+{
+    public class SpaCyTextCategorizer : INlpPipeline
+    {
+        public IConfiguration Configuration { get; set; }
+
+        public bool Process(Agent agent, JObject data)
+        {
+            //var input = new List<Tuple<String, JObject>>();
+
+            var texts = new List<String>();
+            var golds = new List<JObject>();
+
+            List<string> intentNames = agent.Intents.Select(x => x.Name).Distinct().ToList();
+
+            agent.Intents.ForEach(intent =>
+            {
+                intent.UserSays.ForEach(userSay => {
+
+                    var text = String.Join(string.Empty, userSay.Data.Select(say => say.Text));
+                    var dim = JObject.FromObject(new { });
+
+                    intentNames.ForEach(name =>
+                    {
+                        dim[name] = (intent.Name == name) ? 1 : 0;
+                    });
+
+                    //input.Add(new Tuple<string, JObject>(text, JObject.FromObject(new { Cats = dim })));
+                    texts.Add(text);
+                    golds.Add(JObject.FromObject(new { cats = dim }));
+                });
+
+            });
+
+            var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
+            var request = new RestRequest("textcategorizer", Method.POST);
+            request.RequestFormat = DataFormat.Json;
+
+            request.AddParameter("application/json", JsonConvert.SerializeObject(new { Texts = texts.Take(2), Golds = golds.Take(2), Labels = intentNames }), ParameterType.RequestBody);           
+
+            var response = client.Execute<Result>(request);
+
+            data["ModelName"] = response.Data.ModelName;
+            
+            //Predict
+            var request2 = new RestRequest("predict", Method.GET);
+            request2.AddParameter("text", "the roof is leaking");
+            var response2 = client.Execute(request2);
+            
+            return true;
+        }
+
+        public class Result
+        {
+            public String ModelName { get; set; }
+        }
+    }
+}

BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs

@@ -1,4 +1,5 @@
 using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
 using BotSharp.Core.Models;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
@@ -13,11 +14,11 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool Process(string text, JObject data)
+        public bool Process(Agent agent, JObject data)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("tokenize", Method.GET);
-            request.AddParameter("text", text);
+            request.AddParameter("text", "");
             var response = client.Execute<Result>(request);
 
             data.Add("Tokens", JToken.FromObject(response.Data.Tokens));

BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
 using RestSharp;
@@ -12,11 +13,11 @@ namespace BotSharp.Core.Engines.SpaCy
     {
         public IConfiguration Configuration { get; set; }
 
-        public bool Process(string text, JObject data)
+        public bool Process(Agent agent, JObject data)
         {
             var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
             var request = new RestRequest("featurize", Method.GET);
-            request.AddParameter("text", text);
+            request.AddParameter("text", "");
             var response = client.Execute<Result>(request);
 
             data.Add("Features", JToken.FromObject(response.Data.Vectors));

BotSharp.UnitTest/AgentTest.cs

@@ -1,7 +1,9 @@
 using BotSharp.Core.Agents;
 using BotSharp.Core.Engines;
+using BotSharp.Core.Intents;
 using BotSharp.Core.Models;
 using EntityFrameworkCore.BootKit;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
 using System.Collections.Generic;
@@ -66,6 +68,7 @@ namespace BotSharp.UnitTest
             config.SessionId = Guid.NewGuid().ToString();
 
             var rasa = new RasaAi(dc, config);
+
             string msg = rasa.Train();
 
             Assert.IsTrue(!String.IsNullOrEmpty(msg));

BotSharp.UnitTest/BotTrainerTest.cs

@@ -1,4 +1,5 @@
 using BotSharp.Core.Engines;
+using BotSharp.Core.Models;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
 using System.Collections.Generic;
@@ -12,8 +13,13 @@ namespace BotSharp.UnitTest
         [TestMethod]
         public void TrainingTest()
         {
+            var config = new AIConfiguration(BOT_CLIENT_TOKEN, SupportedLanguage.English);
+            config.SessionId = Guid.NewGuid().ToString();
+
+            var rasa = new RasaAi(dc, config);
+
             var trainer = new BotTrainer(BOT_ID, dc);
-            trainer.Train();
+            trainer.Train(rasa.agent);
         }
     }
 }

BotSharp.UnitTest/Settings/settings.bot.json

@@ -7,8 +7,8 @@
     "Lang": "en",
     "Provider": "SpaCyProvider",
     "SpaCyProvider": {
-      "Url": "http://gtx.local:5005"
+      "Url": "http://10.2.21.200:5005"
     },
-    "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer"
+    "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer"
   }
 }

BotSharp.UnitTest/Settings/settings.db.json

@@ -1,6 +1,6 @@
 {
   "Database": {
-    "Default": "SqlServer",
+    "Default": "Sqlite",
     "ConnectionStrings": {
       "InMemory": "DataSource=:memory:",
       "Sqlite": "Data Source=|DataDirectory|BotSharp.db;",

BotSharp.UnitTest/TestEssential.cs

@@ -10,10 +10,10 @@ namespace BotSharp.UnitTest
 {
     public abstract class TestEssential
     {
-        public static String BOT_ID = "fd9f1b29-fed8-4c68-8fda-69ab463da126";
+        public static String BOT_ID = "5f98a0fd-e7e9-4155-9610-d3f40d026162";
-        public static String BOT_CLIENT_TOKEN = "23a53c46d6244840bbb10c89c171d299";
+        public static String BOT_CLIENT_TOKEN = "2fffb9a1a9214144ab2717a37fa43c33";
-        public static String BOT_DEVELOPER_TOKEN = "d86103f446d049ff8d5f506e8dfe5f3f";
+        public static String BOT_DEVELOPER_TOKEN = "2c7d224cf7274f9d93b4c65c31ca82fe";
-        public static String BOT_NAME = "Voicebot";
+        public static String BOT_NAME = "Handybot";
 
         protected Database dc { get; set; }
         protected string contentRoot;