diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index cdee7c8e..775aee62 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -26,13 +26,29 @@ namespace BotSharp.Core.Engines // Get NLP Provider string providerName = Database.Configuration.GetSection($"{config}:Provider").Value; var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpProvider; + provider.Configuration = Database.Configuration.GetSection("BotSharpAi"); + provider.Load(); - - // tokenize - ITokenizer tokenizer; + // pipe process - INlpProvider nlpProvider; + // Tokenize + var tokenizerName = provider.Configuration.GetSection("Pipe:Tokenizer").Value; + var tokenizer = TypeHelper.GetInstance(tokenizerName, Database.Assemblies) as INlpTokenizer; + tokenizer.Configuration = provider.Configuration; + var tokens = tokenizer.Tokenize("How are you doing?"); + + // Featurize + var featurizerName = provider.Configuration.GetSection("Pipe:Featurizer").Value; + var featurizer = TypeHelper.GetInstance(featurizerName, Database.Assemblies) as INlpFeaturizer; + featurizer.Configuration = provider.Configuration; + var features = featurizer.Featurize("How are you doing?"); + + // Entitize + var entitizerName = provider.Configuration.GetSection("Pipe:Entitizer").Value; + var entitizer = TypeHelper.GetInstance(entitizerName, Database.Assemblies) as INlpEntitizer; + entitizer.Configuration = provider.Configuration; + var entities = entitizer.Entitize("Where are you going tomorrow ?"); return ""; } diff --git a/BotSharp.Core/Engines/INlpEntitizer.cs b/BotSharp.Core/Engines/INlpEntitizer.cs new file mode 100644 index 00000000..2c454cba --- /dev/null +++ b/BotSharp.Core/Engines/INlpEntitizer.cs @@ -0,0 +1,15 @@ +using BotSharp.Core.Models; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public interface INlpEntitizer + { + IConfiguration Configuration { get; set; } + + List Entitize(string text); + } +} diff --git a/BotSharp.Core/Engines/INlpFeaturizer.cs b/BotSharp.Core/Engines/INlpFeaturizer.cs new file mode 100644 index 00000000..34d6fd0a --- /dev/null +++ b/BotSharp.Core/Engines/INlpFeaturizer.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + public interface INlpFeaturizer + { + IConfiguration Configuration { get; set; } + + List Featurize(string text); + } +} diff --git a/BotSharp.Core/Engines/INlpProvider.cs b/BotSharp.Core/Engines/INlpProvider.cs index 14cd9008..d5ce8ded 100644 --- a/BotSharp.Core/Engines/INlpProvider.cs +++ b/BotSharp.Core/Engines/INlpProvider.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Configuration; +using System; using System.Collections.Generic; using System.Text; @@ -6,7 +7,8 @@ namespace BotSharp.Core.Engines { public interface INlpProvider { - void LoadModel(); - Object GetDoc(); + IConfiguration Configuration { get; set; } + + bool Load(); } } diff --git a/BotSharp.Core/Engines/INlpTokenizer.cs b/BotSharp.Core/Engines/INlpTokenizer.cs new file mode 100644 index 00000000..08b5c6d2 --- /dev/null +++ b/BotSharp.Core/Engines/INlpTokenizer.cs @@ -0,0 +1,18 @@ +using BotSharp.Core.Models; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines +{ + /// + /// Segment text into words, punctuations marks etc. + /// + public interface INlpTokenizer + { + IConfiguration Configuration { get; set; } + + List Tokenize(string text); + } +} diff --git a/BotSharp.Core/Engines/ITokenizer.cs b/BotSharp.Core/Engines/ITokenizer.cs deleted file mode 100644 index 82fbcafe..00000000 --- a/BotSharp.Core/Engines/ITokenizer.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BotSharp.Core.Engines -{ - /// - /// - /// - public interface ITokenizer - { - } -} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs new file mode 100644 index 00000000..80f968f2 --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Text; +using BotSharp.Core.Models; +using Microsoft.Extensions.Configuration; +using RestSharp; + +namespace BotSharp.Core.Engines.SpaCy +{ + public class SpaCyEntitizer : INlpEntitizer + { + public IConfiguration Configuration { get; set; } + + public List Entitize(string text) + { + var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); + var request = new RestRequest("entitize", Method.GET); + request.AddParameter("text", text); + var response = client.Execute(request); + + return response.Data.Entities; + } + + public class Result + { + public List Entities { get; set; } + } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs index a6e27937..51a6ab0b 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs @@ -1,4 +1,6 @@ using BotSharp.Core.Utilities; +using Microsoft.Extensions.Configuration; +using RestSharp; using System; using System.Collections.Generic; using System.Net.Http; @@ -8,17 +10,15 @@ namespace BotSharp.Core.Engines.SpaCy { public class SpaCyProvider : INlpProvider { - public async void LoadModel() - { - using (var client = new HttpClient()) - { - var response = await client.PostAsync("", new JsonContent(new { })); - } - } + public IConfiguration Configuration { get; set; } - public object GetDoc() + public bool Load() { - throw new NotImplementedException(); + var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); + var request = new RestRequest("load", Method.GET); + var response = client.Execute(request); + + return response.IsSuccessful; } } } diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs new file mode 100644 index 00000000..1da3e7f6 --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -0,0 +1,29 @@ +using BotSharp.Core.Models; +using Microsoft.Extensions.Configuration; +using RestSharp; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Engines.SpaCy +{ + public class SpaCyTokenizer : INlpTokenizer + { + public IConfiguration Configuration { get; set; } + + public List Tokenize(string text) + { + var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); + var request = new RestRequest("tokenize", Method.GET); + request.AddParameter("text", text); + var response = client.Execute(request); + + return response.Data.Tokens; + } + + public class Result + { + public List Tokens { get; set; } + } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs new file mode 100644 index 00000000..67c0d472 --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.Configuration; +using RestSharp; + +namespace BotSharp.Core.Engines.SpaCy +{ + public class SpacyFeaturizer : INlpFeaturizer + { + public IConfiguration Configuration { get; set; } + + public List Featurize(string text) + { + var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); + var request = new RestRequest("featurize", Method.GET); + request.AddParameter("text", text); + var response = client.Execute(request); + + return response.Data.Vectors; + } + + public class Result + { + public List Vectors { get; set; } + } + } +} diff --git a/BotSharp.Core/Engines/SpaCy/server.py b/BotSharp.Core/Engines/SpaCy/server.py new file mode 100644 index 00000000..1f80d48f --- /dev/null +++ b/BotSharp.Core/Engines/SpaCy/server.py @@ -0,0 +1,45 @@ +from bottle import route, run, request +from spacy.tokenizer import Tokenizer +from spacy.pipeline import EntityRecognizer +import spacy + +nlp = spacy.load('en') +tokenizer = Tokenizer(nlp.vocab) +ner = EntityRecognizer(nlp.vocab) + +@route('/load') +def load(): + pass + +@route('/tokenize') +def tokenize(): + tokens = tokenizer(request.query.text) + list = [] + for token in tokens: + print(token) + list.append({'text': token.text, 'offset': token.idx}) + return {'tokens': list} + +@route('/featurize') +def tokenize(): + doc = nlp(request.query.text) + list = [] + print(doc.vector.size) + for vec in doc.vector: + print(vec) + list.append(str(vec.real)) + return {'vectors': list} + +@route('/entitize') +def entitize(): + doc = nlp(request.query.text) + entities = ner(doc) + print(entities) + list = [] + print(doc.ents.size) + for entity in entities: + print(entity) + list.append(entity) + return {'entities': list} + +run(host='0.0.0.0', port=5005, debug=True) \ No newline at end of file diff --git a/BotSharp.Core/Models/NlpEntity.cs b/BotSharp.Core/Models/NlpEntity.cs new file mode 100644 index 00000000..93a36efd --- /dev/null +++ b/BotSharp.Core/Models/NlpEntity.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Models +{ + public class NlpEntity + { + /// + /// Entity label + /// + public string Entity { get; set; } + + public string Value { get; set; } + + public int Start { get; set; } + + public decimal Confidence { get; set; } + + public int End { get; set; } + } +} diff --git a/BotSharp.Core/Models/NlpToken.cs b/BotSharp.Core/Models/NlpToken.cs new file mode 100644 index 00000000..40518011 --- /dev/null +++ b/BotSharp.Core/Models/NlpToken.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Core.Models +{ + public class NlpToken + { + public string Text { get; set; } + public int Offset { get; set; } + public int End + { + get + { + return Offset + Text.Length; + } + } + } +} diff --git a/BotSharp.UnitTest/Settings/settings.bot.json b/BotSharp.UnitTest/Settings/settings.bot.json index a78d1f72..f7d9a5c4 100644 --- a/BotSharp.UnitTest/Settings/settings.bot.json +++ b/BotSharp.UnitTest/Settings/settings.bot.json @@ -4,13 +4,15 @@ }, "BotSharpAi": { - "Lang": "en", - "Provider": "SpaCyProvider", - "SpaCyProvider": { - "Url": "http://localhost:5005" - }, - "Pipe": [ - - ] + "Lang": "en", + "Provider": "SpaCyProvider", + "SpaCyProvider": { + "Url": "http://gtx.local:5005" + }, + "Pipe": { + "Tokenizer": "SpaCyTokenizer", + "Featurizer": "SpacyFeaturizer", + "Entitizer": "SpaCyEntitizer" + } } }