diff --git a/BotSharp.Core/Abstractions/INlpEntitizer.cs b/BotSharp.Core/Abstractions/INlpEntitizer.cs deleted file mode 100644 index 2c454cba..00000000 --- a/BotSharp.Core/Abstractions/INlpEntitizer.cs +++ /dev/null @@ -1,15 +0,0 @@ -using BotSharp.Core.Models; -using Microsoft.Extensions.Configuration; -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.Core.Engines -{ - public interface INlpEntitizer - { - IConfiguration Configuration { get; set; } - - List Entitize(string text); - } -} diff --git a/BotSharp.Core/Abstractions/INlpFeaturizer.cs b/BotSharp.Core/Abstractions/INlpFeaturizer.cs deleted file mode 100644 index 34d6fd0a..00000000 --- a/BotSharp.Core/Abstractions/INlpFeaturizer.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Microsoft.Extensions.Configuration; -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.Core.Engines -{ - public interface INlpFeaturizer - { - IConfiguration Configuration { get; set; } - - List Featurize(string text); - } -} diff --git a/BotSharp.Core/Abstractions/INlpPipeline.cs b/BotSharp.Core/Abstractions/INlpPipeline.cs new file mode 100644 index 00000000..8b9c7d36 --- /dev/null +++ b/BotSharp.Core/Abstractions/INlpPipeline.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Abstractions +{ + /// + /// NLP process pipeline interface + /// + public interface INlpPipeline + { + IConfiguration Configuration { get; set; } + + bool Process(String text, JObject data); + } +} diff --git a/BotSharp.Core/Abstractions/INlpProvider.cs b/BotSharp.Core/Abstractions/INlpProvider.cs deleted file mode 100644 index d5ce8ded..00000000 --- a/BotSharp.Core/Abstractions/INlpProvider.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Microsoft.Extensions.Configuration; -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.Core.Engines -{ - public interface INlpProvider - { - IConfiguration Configuration { get; set; } - - bool Load(); - } -} diff --git a/BotSharp.Core/Abstractions/INlpTokenizer.cs b/BotSharp.Core/Abstractions/INlpTokenizer.cs deleted file mode 100644 index 08b5c6d2..00000000 --- a/BotSharp.Core/Abstractions/INlpTokenizer.cs +++ /dev/null @@ -1,18 +0,0 @@ -using BotSharp.Core.Models; -using Microsoft.Extensions.Configuration; -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.Core.Engines -{ - /// - /// Segment text into words, punctuations marks etc. - /// - public interface INlpTokenizer - { - IConfiguration Configuration { get; set; } - - List Tokenize(string text); - } -} diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 775aee62..c854e0bd 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; +using BotSharp.Core.Abstractions; using DotNetToolkit; using EntityFrameworkCore.BootKit; +using Newtonsoft.Json.Linq; namespace BotSharp.Core.Engines { @@ -23,32 +26,30 @@ namespace BotSharp.Core.Engines public string Train() { + var data = JObject.FromObject(new { }); + // Get NLP Provider string providerName = Database.Configuration.GetSection($"{config}:Provider").Value; - var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpProvider; + var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpPipeline; provider.Configuration = Database.Configuration.GetSection("BotSharpAi"); + provider.Process("How are you doing?", data); - provider.Load(); // pipe process + var pipelines = Database.Configuration.GetSection($"{config}:Pipe").Value + .Split(',') + .Select(x => x.Trim()) + .ToList(); - // Tokenize - var tokenizerName = provider.Configuration.GetSection("Pipe:Tokenizer").Value; - var tokenizer = TypeHelper.GetInstance(tokenizerName, Database.Assemblies) as INlpTokenizer; - tokenizer.Configuration = provider.Configuration; - var tokens = tokenizer.Tokenize("How are you doing?"); + pipelines.ForEach(pipeName => + { + var pipe = TypeHelper.GetInstance(pipeName, Database.Assemblies) as INlpPipeline; + pipe.Configuration = provider.Configuration; + var tokens = pipe.Process("How are you doing?", data); - // Featurize - var featurizerName = provider.Configuration.GetSection("Pipe:Featurizer").Value; - var featurizer = TypeHelper.GetInstance(featurizerName, Database.Assemblies) as INlpFeaturizer; - featurizer.Configuration = provider.Configuration; - var features = featurizer.Featurize("How are you doing?"); - // Entitize - var entitizerName = provider.Configuration.GetSection("Pipe:Entitizer").Value; - var entitizer = TypeHelper.GetInstance(entitizerName, Database.Assemblies) as INlpEntitizer; - entitizer.Configuration = provider.Configuration; - var entities = entitizer.Entitize("Where are you going tomorrow ?"); + }); + return ""; } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs index 80f968f2..280d3c63 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs @@ -1,24 +1,28 @@ using System; using System.Collections.Generic; using System.Text; +using BotSharp.Core.Abstractions; using BotSharp.Core.Models; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; using RestSharp; namespace BotSharp.Core.Engines.SpaCy { - public class SpaCyEntitizer : INlpEntitizer + public class SpaCyEntitizer : INlpPipeline { public IConfiguration Configuration { get; set; } - public List Entitize(string text) + public bool Process(string text, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("entitize", Method.GET); request.AddParameter("text", text); var response = client.Execute(request); - return response.Data.Entities; + data.Add("Entities", JToken.FromObject(response.Data.Entities)); + + return response.IsSuccessful; } public class Result diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index 5651a36b..a4eb8020 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -1,4 +1,6 @@ -using Microsoft.Extensions.Configuration; +using BotSharp.Core.Abstractions; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; using RestSharp; using System; using System.Collections.Generic; @@ -7,11 +9,11 @@ using System.Text; namespace BotSharp.Core.Engines.SpaCy { - public class SpaCyProvider : INlpProvider + public class SpaCyProvider : INlpPipeline { public IConfiguration Configuration { get; set; } - public bool Load() + public bool Process(string text, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("load", Method.GET); diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index 1da3e7f6..3857974b 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -1,5 +1,7 @@ -using BotSharp.Core.Models; +using BotSharp.Core.Abstractions; +using BotSharp.Core.Models; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; using RestSharp; using System; using System.Collections.Generic; @@ -7,18 +9,20 @@ using System.Text; namespace BotSharp.Core.Engines.SpaCy { - public class SpaCyTokenizer : INlpTokenizer + public class SpaCyTokenizer : INlpPipeline { public IConfiguration Configuration { get; set; } - public List Tokenize(string text) + public bool Process(string text, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tokenize", Method.GET); request.AddParameter("text", text); var response = client.Execute(request); - return response.Data.Tokens; + data.Add("tokens", JToken.FromObject(response.Data.Tokens)); + + return response.IsSuccessful; } public class Result diff --git a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs index 67c0d472..2d04535b 100644 --- a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs @@ -1,23 +1,27 @@ using System; using System.Collections.Generic; using System.Text; +using BotSharp.Core.Abstractions; using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; using RestSharp; namespace BotSharp.Core.Engines.SpaCy { - public class SpacyFeaturizer : INlpFeaturizer + public class SpacyFeaturizer : INlpPipeline { public IConfiguration Configuration { get; set; } - public List Featurize(string text) + public bool Process(string text, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("featurize", Method.GET); request.AddParameter("text", text); var response = client.Execute(request); - return response.Data.Vectors; + data.Add("features", JToken.FromObject(response.Data.Vectors)); + + return response.IsSuccessful; } public class Result diff --git a/BotSharp.UnitTest/Settings/settings.bot.json b/BotSharp.UnitTest/Settings/settings.bot.json index f7d9a5c4..c2593bfe 100644 --- a/BotSharp.UnitTest/Settings/settings.bot.json +++ b/BotSharp.UnitTest/Settings/settings.bot.json @@ -9,10 +9,6 @@ "SpaCyProvider": { "Url": "http://gtx.local:5005" }, - "Pipe": { - "Tokenizer": "SpaCyTokenizer", - "Featurizer": "SpacyFeaturizer", - "Entitizer": "SpaCyEntitizer" - } + "Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer" } }