SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add SpaCy TextCategorizer component

Browse source
This commit is contained in:
Bo Peng 2018-06-15 12:39:24 -05:00
parent 86561ed839
commit bbbcf93ca6
13 changed files with 121 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -291,3 +291,4 @@ __pycache__/
/Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
/BotSharp.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
/BotSharp.UnitTest/App_Data/DbInitializer/Agents
/BotSharp.UnitTest/App_Data/BotSharp.db

5
BotSharp.Core/Abstractions/INlpPipeline.cs
View file

@ -1,4 +1,5 @@
using Microsoft.Extensions.Configuration;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
@ -13,6 +14,6 @@ namespace BotSharp.Core.Abstractions
{
IConfiguration Configuration { get; set; }
bool Process(String text, JObject data);
bool Process(Agent agent, JObject data);
}
}

24
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -3,8 +3,11 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Intents;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq;
namespace BotSharp.Core.Engines
@ -24,15 +27,26 @@ namespace BotSharp.Core.Engines
this.config = config;
}
public string Train()
public string Train(Agent agent)
{
var data = JObject.FromObject(new { });
agent.Intents = dc.Table<Intent>()
.Include(x => x.Contexts)
.Include(x => x.Responses).ThenInclude(x => x.Contexts)
.Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts)
.Include(x => x.Responses).ThenInclude(x => x.Messages)
.Include(x => x.UserSays).ThenInclude(x => x.Data)
.Where(x => x.AgentId == agentId)
.ToList();
var data = JObject.FromObject(new
{
});
// Get NLP Provider
string providerName = Database.Configuration.GetSection($"{config}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpPipeline;
provider.Configuration = Database.Configuration.GetSection("BotSharpAi");
provider.Process("How are you today ?", data);
provider.Process(agent, data);
// pipe process
@ -45,9 +59,7 @@ namespace BotSharp.Core.Engines
{
var pipe = TypeHelper.GetInstance(pipeName, Database.Assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
var tokens = pipe.Process("How are you today ?", data);
pipe.Process(agent, data);
});

7
BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs
View file

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Text;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
@ -13,14 +14,14 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(string text, JObject data)
public bool Process(Agent agent, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("entitize", Method.GET);
request.AddParameter("text", text);
request.AddParameter("text", "");
var response = client.Execute<Result>(request);
data.Add("Entities", JToken.FromObject(response.Data.Entities));
//data.Add("Entities", JToken.FromObject(response.Data.Entities));
return response.IsSuccessful;
}

3
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
@ -13,7 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(string text, JObject data)
public bool Process(Agent agent, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("load", Method.GET);

70
BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs Normal file
View file

@ -0,0 +1,70 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Intents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyTextCategorizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
{
//var input = new List<Tuple<String, JObject>>();
var texts = new List<String>();
var golds = new List<JObject>();
List<string> intentNames = agent.Intents.Select(x => x.Name).Distinct().ToList();
agent.Intents.ForEach(intent =>
{
intent.UserSays.ForEach(userSay => {
var text = String.Join(string.Empty, userSay.Data.Select(say => say.Text));
var dim = JObject.FromObject(new { });
intentNames.ForEach(name =>
{
dim[name] = (intent.Name == name) ? 1 : 0;
});
//input.Add(new Tuple<string, JObject>(text, JObject.FromObject(new { Cats = dim })));
texts.Add(text);
golds.Add(JObject.FromObject(new { cats = dim }));
});
});
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("textcategorizer", Method.POST);
request.RequestFormat = DataFormat.Json;
request.AddParameter("application/json", JsonConvert.SerializeObject(new { Texts = texts.Take(2), Golds = golds.Take(2), Labels = intentNames }), ParameterType.RequestBody);
var response = client.Execute<Result>(request);
data["ModelName"] = response.Data.ModelName;
//Predict
var request2 = new RestRequest("predict", Method.GET);
request2.AddParameter("text", "the roof is leaking");
var response2 = client.Execute(request2);
return true;
}
public class Result
{
public String ModelName { get; set; }
}
}
}

5
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
@ -13,11 +14,11 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(string text, JObject data)
public bool Process(Agent agent, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tokenize", Method.GET);
request.AddParameter("text", text);
request.AddParameter("text", "");
var response = client.Execute<Result>(request);
data.Add("Tokens", JToken.FromObject(response.Data.Tokens));

5
BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs
View file

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Text;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
@ -12,11 +13,11 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(string text, JObject data)
public bool Process(Agent agent, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("featurize", Method.GET);
request.AddParameter("text", text);
request.AddParameter("text", "");
var response = client.Execute<Result>(request);
data.Add("Features", JToken.FromObject(response.Data.Vectors));

3
BotSharp.UnitTest/AgentTest.cs
View file

@ -1,7 +1,9 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
@ -66,6 +68,7 @@ namespace BotSharp.UnitTest
config.SessionId = Guid.NewGuid().ToString();
var rasa = new RasaAi(dc, config);
string msg = rasa.Train();
Assert.IsTrue(!String.IsNullOrEmpty(msg));

8
BotSharp.UnitTest/BotTrainerTest.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Engines;
using BotSharp.Core.Models;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
@ -12,8 +13,13 @@ namespace BotSharp.UnitTest
[TestMethod]
public void TrainingTest()
{
var config = new AIConfiguration(BOT_CLIENT_TOKEN, SupportedLanguage.English);
config.SessionId = Guid.NewGuid().ToString();
var rasa = new RasaAi(dc, config);
var trainer = new BotTrainer(BOT_ID, dc);
trainer.Train();
trainer.Train(rasa.agent);
}
}
}

4
BotSharp.UnitTest/Settings/settings.bot.json
View file

@ -7,8 +7,8 @@
"Lang": "en",
"Provider": "SpaCyProvider",
"SpaCyProvider": {
"Url": "http://gtx.local:5005"
"Url": "http://10.2.21.200:5005"
},
"Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer"
"Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer, SpaCyTextCategorizer"
}
}

2
BotSharp.UnitTest/Settings/settings.db.json
View file

@ -1,6 +1,6 @@
{
"Database": {
"Default": "SqlServer",
"Default": "Sqlite",
"ConnectionStrings": {
"InMemory": "DataSource=:memory:",
"Sqlite": "Data Source=|DataDirectory|BotSharp.db;",

8
BotSharp.UnitTest/TestEssential.cs
View file

@ -10,10 +10,10 @@ namespace BotSharp.UnitTest
{
public abstract class TestEssential
{
public static String BOT_ID = "fd9f1b29-fed8-4c68-8fda-69ab463da126";
public static String BOT_CLIENT_TOKEN = "23a53c46d6244840bbb10c89c171d299";
public static String BOT_DEVELOPER_TOKEN = "d86103f446d049ff8d5f506e8dfe5f3f";
public static String BOT_NAME = "Voicebot";
public static String BOT_ID = "5f98a0fd-e7e9-4155-9610-d3f40d026162";
public static String BOT_CLIENT_TOKEN = "2fffb9a1a9214144ab2717a37fa43c33";
public static String BOT_DEVELOPER_TOKEN = "2c7d224cf7274f9d93b4c65c31ca82fe";
public static String BOT_NAME = "Handybot";
protected Database dc { get; set; }
protected string contentRoot;