From bbbcf93ca664bf657832a2896825dd5357cfd572 Mon Sep 17 00:00:00 2001 From: Bo Peng Date: Fri, 15 Jun 2018 12:39:24 -0500 Subject: [PATCH] Add SpaCy TextCategorizer component --- .gitignore | 1 + BotSharp.Core/Abstractions/INlpPipeline.cs | 5 +- BotSharp.Core/Engines/BotTrainer.cs | 24 +++++-- BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs | 7 +- BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs | 3 +- .../Engines/SpaCy/SpaCyTextCategorizer.cs | 70 +++++++++++++++++++ BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs | 5 +- .../Engines/SpaCy/SpacyFeaturizer.cs | 5 +- BotSharp.UnitTest/AgentTest.cs | 3 + BotSharp.UnitTest/BotTrainerTest.cs | 8 ++- BotSharp.UnitTest/Settings/settings.bot.json | 4 +- BotSharp.UnitTest/Settings/settings.db.json | 2 +- BotSharp.UnitTest/TestEssential.cs | 8 +-- 13 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs diff --git a/.gitignore b/.gitignore index 07eaa50d..343c8bec 100644 --- a/.gitignore +++ b/.gitignore @@ -291,3 +291,4 @@ __pycache__/ /Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant /BotSharp.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant /BotSharp.UnitTest/App_Data/DbInitializer/Agents +/BotSharp.UnitTest/App_Data/BotSharp.db diff --git a/BotSharp.Core/Abstractions/INlpPipeline.cs b/BotSharp.Core/Abstractions/INlpPipeline.cs index 8b9c7d36..774be45e 100644 --- a/BotSharp.Core/Abstractions/INlpPipeline.cs +++ b/BotSharp.Core/Abstractions/INlpPipeline.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Configuration; +using BotSharp.Core.Agents; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; @@ -13,6 +14,6 @@ namespace BotSharp.Core.Abstractions { IConfiguration Configuration { get; set; } - bool Process(String text, JObject data); + bool Process(Agent agent, JObject data); } } diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index d3ec7dcc..7de0110f 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -3,8 +3,11 @@ using System.Collections.Generic; using System.Linq; using System.Text; using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.Core.Intents; using DotNetToolkit; using EntityFrameworkCore.BootKit; +using Microsoft.EntityFrameworkCore; using Newtonsoft.Json.Linq; namespace BotSharp.Core.Engines @@ -24,15 +27,26 @@ namespace BotSharp.Core.Engines this.config = config; } - public string Train() + public string Train(Agent agent) { - var data = JObject.FromObject(new { }); + agent.Intents = dc.Table() + .Include(x => x.Contexts) + .Include(x => x.Responses).ThenInclude(x => x.Contexts) + .Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts) + .Include(x => x.Responses).ThenInclude(x => x.Messages) + .Include(x => x.UserSays).ThenInclude(x => x.Data) + .Where(x => x.AgentId == agentId) + .ToList(); + + var data = JObject.FromObject(new + { + }); // Get NLP Provider string providerName = Database.Configuration.GetSection($"{config}:Provider").Value; var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpPipeline; provider.Configuration = Database.Configuration.GetSection("BotSharpAi"); - provider.Process("How are you today ?", data); + provider.Process(agent, data); // pipe process @@ -45,9 +59,7 @@ namespace BotSharp.Core.Engines { var pipe = TypeHelper.GetInstance(pipeName, Database.Assemblies) as INlpPipeline; pipe.Configuration = provider.Configuration; - var tokens = pipe.Process("How are you today ?", data); - - + pipe.Process(agent, data); }); diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs index 280d3c63..4862bb82 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; using BotSharp.Core.Models; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; @@ -13,14 +14,14 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool Process(string text, JObject data) + public bool Process(Agent agent, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("entitize", Method.GET); - request.AddParameter("text", text); + request.AddParameter("text", ""); var response = client.Execute(request); - data.Add("Entities", JToken.FromObject(response.Data.Entities)); + //data.Add("Entities", JToken.FromObject(response.Data.Entities)); return response.IsSuccessful; } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index a4eb8020..2d214b92 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using RestSharp; @@ -13,7 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool Process(string text, JObject data) + public bool Process(Agent agent, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("load", Method.GET); diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs new file mode 100644 index 00000000..bb8ccfde --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs @@ -0,0 +1,70 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.Core.Intents; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using RestSharp; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Core.Engines.SpaCy +{ + public class SpaCyTextCategorizer : INlpPipeline + { + public IConfiguration Configuration { get; set; } + + public bool Process(Agent agent, JObject data) + { + //var input = new List>(); + + var texts = new List(); + var golds = new List(); + + List intentNames = agent.Intents.Select(x => x.Name).Distinct().ToList(); + + agent.Intents.ForEach(intent => + { + intent.UserSays.ForEach(userSay => { + + var text = String.Join(string.Empty, userSay.Data.Select(say => say.Text)); + var dim = JObject.FromObject(new { }); + + intentNames.ForEach(name => + { + dim[name] = (intent.Name == name) ? 1 : 0; + }); + + //input.Add(new Tuple(text, JObject.FromObject(new { Cats = dim }))); + texts.Add(text); + golds.Add(JObject.FromObject(new { cats = dim })); + }); + + }); + + var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); + var request = new RestRequest("textcategorizer", Method.POST); + request.RequestFormat = DataFormat.Json; + + request.AddParameter("application/json", JsonConvert.SerializeObject(new { Texts = texts.Take(2), Golds = golds.Take(2), Labels = intentNames }), ParameterType.RequestBody); + + var response = client.Execute(request); + + data["ModelName"] = response.Data.ModelName; + + //Predict + var request2 = new RestRequest("predict", Method.GET); + request2.AddParameter("text", "the roof is leaking"); + var response2 = client.Execute(request2); + + return true; + } + + public class Result + { + public String ModelName { get; set; } + } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index 2ba3b6aa..87d3ed15 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; using BotSharp.Core.Models; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; @@ -13,11 +14,11 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool Process(string text, JObject data) + public bool Process(Agent agent, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tokenize", Method.GET); - request.AddParameter("text", text); + request.AddParameter("text", ""); var response = client.Execute(request); data.Add("Tokens", JToken.FromObject(response.Data.Tokens)); diff --git a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs index cb8af886..e1cdc6c3 100644 --- a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using RestSharp; @@ -12,11 +13,11 @@ namespace BotSharp.Core.Engines.SpaCy { public IConfiguration Configuration { get; set; } - public bool Process(string text, JObject data) + public bool Process(Agent agent, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("featurize", Method.GET); - request.AddParameter("text", text); + request.AddParameter("text", ""); var response = client.Execute(request); data.Add("Features", JToken.FromObject(response.Data.Vectors)); diff --git a/BotSharp.UnitTest/AgentTest.cs b/BotSharp.UnitTest/AgentTest.cs index de8063f0..87506f92 100644 --- a/BotSharp.UnitTest/AgentTest.cs +++ b/BotSharp.UnitTest/AgentTest.cs @@ -1,7 +1,9 @@ using BotSharp.Core.Agents; using BotSharp.Core.Engines; +using BotSharp.Core.Intents; using BotSharp.Core.Models; using EntityFrameworkCore.BootKit; +using Microsoft.EntityFrameworkCore; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; @@ -66,6 +68,7 @@ namespace BotSharp.UnitTest config.SessionId = Guid.NewGuid().ToString(); var rasa = new RasaAi(dc, config); + string msg = rasa.Train(); Assert.IsTrue(!String.IsNullOrEmpty(msg)); diff --git a/BotSharp.UnitTest/BotTrainerTest.cs b/BotSharp.UnitTest/BotTrainerTest.cs index 098cb3a3..4b006bf5 100644 --- a/BotSharp.UnitTest/BotTrainerTest.cs +++ b/BotSharp.UnitTest/BotTrainerTest.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Engines; +using BotSharp.Core.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; @@ -12,8 +13,13 @@ namespace BotSharp.UnitTest [TestMethod] public void TrainingTest() { + var config = new AIConfiguration(BOT_CLIENT_TOKEN, SupportedLanguage.English); + config.SessionId = Guid.NewGuid().ToString(); + + var rasa = new RasaAi(dc, config); + var trainer = new BotTrainer(BOT_ID, dc); - trainer.Train(); + trainer.Train(rasa.agent); } } } diff --git a/BotSharp.UnitTest/Settings/settings.bot.json b/BotSharp.UnitTest/Settings/settings.bot.json index c2593bfe..1e115890 100644 --- a/BotSharp.UnitTest/Settings/settings.bot.json +++ b/BotSharp.UnitTest/Settings/settings.bot.json @@ -7,8 +7,8 @@ "Lang": "en", "Provider": "SpaCyProvider", "SpaCyProvider": { - "Url": "http://gtx.local:5005" + "Url": "http://10.2.21.200:5005" }, - "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer" + "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer" } } diff --git a/BotSharp.UnitTest/Settings/settings.db.json b/BotSharp.UnitTest/Settings/settings.db.json index ebd2d785..68bf73bf 100644 --- a/BotSharp.UnitTest/Settings/settings.db.json +++ b/BotSharp.UnitTest/Settings/settings.db.json @@ -1,6 +1,6 @@ { "Database": { - "Default": "SqlServer", + "Default": "Sqlite", "ConnectionStrings": { "InMemory": "DataSource=:memory:", "Sqlite": "Data Source=|DataDirectory|BotSharp.db;", diff --git a/BotSharp.UnitTest/TestEssential.cs b/BotSharp.UnitTest/TestEssential.cs index eff7a16e..44ae611c 100644 --- a/BotSharp.UnitTest/TestEssential.cs +++ b/BotSharp.UnitTest/TestEssential.cs @@ -10,10 +10,10 @@ namespace BotSharp.UnitTest { public abstract class TestEssential { - public static String BOT_ID = "fd9f1b29-fed8-4c68-8fda-69ab463da126"; - public static String BOT_CLIENT_TOKEN = "23a53c46d6244840bbb10c89c171d299"; - public static String BOT_DEVELOPER_TOKEN = "d86103f446d049ff8d5f506e8dfe5f3f"; - public static String BOT_NAME = "Voicebot"; + public static String BOT_ID = "5f98a0fd-e7e9-4155-9610-d3f40d026162"; + public static String BOT_CLIENT_TOKEN = "2fffb9a1a9214144ab2717a37fa43c33"; + public static String BOT_DEVELOPER_TOKEN = "2c7d224cf7274f9d93b4c65c31ca82fe"; + public static String BOT_NAME = "Handybot"; protected Database dc { get; set; } protected string contentRoot;