added SpaCy tokenizer and featurizer

BotSharp.Core/Engines/BotTrainer.cs

@@ -26,13 +26,29 @@ namespace BotSharp.Core.Engines
             // Get NLP Provider
             string providerName = Database.Configuration.GetSection($"{config}:Provider").Value;
             var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpProvider;
+            provider.Configuration = Database.Configuration.GetSection("BotSharpAi");
 
+            provider.Load();
 
-            
-            // tokenize
-            ITokenizer tokenizer;
+            // pipe process
 
-            INlpProvider nlpProvider;
+            // Tokenize
+            var tokenizerName = provider.Configuration.GetSection("Pipe:Tokenizer").Value;
+            var tokenizer = TypeHelper.GetInstance(tokenizerName, Database.Assemblies) as INlpTokenizer;
+            tokenizer.Configuration = provider.Configuration;
+            var tokens = tokenizer.Tokenize("How are you doing?");
+
+            // Featurize
+            var featurizerName = provider.Configuration.GetSection("Pipe:Featurizer").Value;
+            var featurizer = TypeHelper.GetInstance(featurizerName, Database.Assemblies) as INlpFeaturizer;
+            featurizer.Configuration = provider.Configuration;
+            var features = featurizer.Featurize("How are you doing?");
+
+            // Entitize
+            var entitizerName = provider.Configuration.GetSection("Pipe:Entitizer").Value;
+            var entitizer = TypeHelper.GetInstance(entitizerName, Database.Assemblies) as INlpEntitizer;
+            entitizer.Configuration = provider.Configuration;
+            var entities = entitizer.Entitize("Where are you going tomorrow ?");
 
             return "";
         }

BotSharp.Core/Engines/INlpEntitizer.cs

@@ -0,0 +1,15 @@
+using BotSharp.Core.Models;
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines
+{
+    public interface INlpEntitizer
+    {
+        IConfiguration Configuration { get; set; }
+
+        List<NlpEntity> Entitize(string text);
+    }
+}

BotSharp.Core/Engines/INlpFeaturizer.cs

@@ -0,0 +1,14 @@
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines
+{
+    public interface INlpFeaturizer
+    {
+        IConfiguration Configuration { get; set; }
+
+        List<decimal> Featurize(string text);
+    }
+}

BotSharp.Core/Engines/INlpProvider.cs

@@ -1,4 +1,5 @@
-using System;
+using Microsoft.Extensions.Configuration;
+using System;
 using System.Collections.Generic;
 using System.Text;
 
@@ -6,7 +7,8 @@ namespace BotSharp.Core.Engines
 {
     public interface INlpProvider
     {
-        void LoadModel();
-        Object GetDoc();
+        IConfiguration Configuration { get; set; }
+
+        bool Load();
     }
 }

BotSharp.Core/Engines/INlpTokenizer.cs

@@ -0,0 +1,18 @@
+using BotSharp.Core.Models;
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines
+{
+    /// <summary>
+    /// Segment text into words, punctuations marks etc.
+    /// </summary>
+    public interface INlpTokenizer
+    {
+        IConfiguration Configuration { get; set; }
+
+        List<NlpToken> Tokenize(string text);
+    }
+}

BotSharp.Core/Engines/ITokenizer.cs

@@ -1,13 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace BotSharp.Core.Engines
-{
-    /// <summary>
-    /// 
-    /// </summary>
-    public interface ITokenizer
-    {
-    }
-}

BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BotSharp.Core.Models;
+using Microsoft.Extensions.Configuration;
+using RestSharp;
+
+namespace BotSharp.Core.Engines.SpaCy
+{
+    public class SpaCyEntitizer : INlpEntitizer
+    {
+        public IConfiguration Configuration { get; set; }
+
+        public List<NlpEntity> Entitize(string text)
+        {
+            var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
+            var request = new RestRequest("entitize", Method.GET);
+            request.AddParameter("text", text);
+            var response = client.Execute<Result>(request);
+
+            return response.Data.Entities;
+        }
+
+        public class Result
+        {
+            public List<NlpEntity> Entities { get; set; }
+        }
+    }
+}

BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs

@@ -1,4 +1,6 @@
 using BotSharp.Core.Utilities;
+using Microsoft.Extensions.Configuration;
+using RestSharp;
 using System;
 using System.Collections.Generic;
 using System.Net.Http;
@@ -8,17 +10,15 @@ namespace BotSharp.Core.Engines.SpaCy
 {
     public class SpaCyProvider : INlpProvider
     {
-        public async void LoadModel()
-        {
-            using (var client = new HttpClient())
-            {
-                var response = await client.PostAsync("", new JsonContent(new { }));
-            }
-        }
+        public IConfiguration Configuration { get; set; }
 
-        public object GetDoc()
+        public bool Load()
         {
-            throw new NotImplementedException();
+            var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
+            var request = new RestRequest("load", Method.GET);
+            var response = client.Execute(request);
+
+            return response.IsSuccessful;
         }
     }
 }

BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs

@@ -0,0 +1,29 @@
+using BotSharp.Core.Models;
+using Microsoft.Extensions.Configuration;
+using RestSharp;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Engines.SpaCy
+{
+    public class SpaCyTokenizer : INlpTokenizer
+    {
+        public IConfiguration Configuration { get; set; }
+
+        public List<NlpToken> Tokenize(string text)
+        {
+            var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
+            var request = new RestRequest("tokenize", Method.GET);
+            request.AddParameter("text", text);
+            var response = client.Execute<Result>(request);
+
+            return response.Data.Tokens;
+        }
+
+        public class Result
+        {
+            public List<NlpToken> Tokens { get; set; }
+        }
+    }
+}

BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.Extensions.Configuration;
+using RestSharp;
+
+namespace BotSharp.Core.Engines.SpaCy
+{
+    public class SpacyFeaturizer : INlpFeaturizer
+    {
+        public IConfiguration Configuration { get; set; }
+
+        public List<decimal> Featurize(string text)
+        {
+            var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
+            var request = new RestRequest("featurize", Method.GET);
+            request.AddParameter("text", text);
+            var response = client.Execute<Result>(request);
+
+            return response.Data.Vectors;
+        }
+
+        public class Result
+        {
+            public List<decimal> Vectors { get; set; }
+        }
+    }
+}

BotSharp.Core/Engines/SpaCy/server.py

@@ -0,0 +1,45 @@
+from bottle import route, run, request
+from spacy.tokenizer import Tokenizer
+from spacy.pipeline import EntityRecognizer
+import spacy
+
+nlp = spacy.load('en')
+tokenizer = Tokenizer(nlp.vocab)
+ner = EntityRecognizer(nlp.vocab)
+
+@route('/load')
+def load():
+    pass
+
+@route('/tokenize')
+def tokenize():
+    tokens = tokenizer(request.query.text)
+    list = []
+    for token in tokens:
+        print(token)
+        list.append({'text': token.text, 'offset': token.idx})
+    return {'tokens': list}
+    
+@route('/featurize')
+def tokenize():
+    doc = nlp(request.query.text)
+    list = []
+    print(doc.vector.size)
+    for vec in doc.vector:
+        print(vec)
+        list.append(str(vec.real))
+    return {'vectors': list}
+    
+@route('/entitize')
+def entitize():
+    doc = nlp(request.query.text)
+    entities = ner(doc)
+    print(entities)
+    list = []
+    print(doc.ents.size)
+    for entity in entities:
+        print(entity)
+        list.append(entity)
+    return {'entities': list}    
+    
+run(host='0.0.0.0', port=5005, debug=True)

BotSharp.Core/Models/NlpEntity.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Models
+{
+    public class NlpEntity
+    {
+        /// <summary>
+        /// Entity label
+        /// </summary>
+        public string Entity { get; set; }
+
+        public string Value { get; set; }
+
+        public int Start { get; set; }
+
+        public decimal Confidence { get; set; }
+
+        public int End { get; set; }
+    }
+}

BotSharp.Core/Models/NlpToken.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Core.Models
+{
+    public class NlpToken
+    {
+        public string Text { get; set; }
+        public int Offset { get; set; }
+        public int End
+        {
+            get
+            {
+                return Offset + Text.Length;
+            }
+        }
+    }
+}

BotSharp.UnitTest/Settings/settings.bot.json

@@ -4,13 +4,15 @@
   },
 
   "BotSharpAi": {
-      "Lang": "en",
-      "Provider": "SpaCyProvider",
-      "SpaCyProvider": {
-        "Url": "http://localhost:5005"
-      },
-      "Pipe": [
-
-      ]
+    "Lang": "en",
+    "Provider": "SpaCyProvider",
+    "SpaCyProvider": {
+      "Url": "http://gtx.local:5005"
+    },
+    "Pipe": {
+      "Tokenizer": "SpaCyTokenizer",
+      "Featurizer": "SpacyFeaturizer",
+      "Entitizer": "SpaCyEntitizer"
+    }
   }
 }