From 97d644e5fe1fb2304c2b5f769b5de70b25d55f6c Mon Sep 17 00:00:00 2001
From: Bolo <36342729+PppBr@users.noreply.github.com>
Date: Mon, 13 Aug 2018 14:30:30 -0500
Subject: [PATCH] intergrate nltk tokenizerin pipeline

---
 BotSharp.Core/Engines/Nltk/NltkTokenizer.cs | 97 +++++++++++++++++++++
 BotSharp.WebHost/Settings/bot.json          |  8 +-
 2 files changed, 102 insertions(+), 3 deletions(-)
 create mode 100644 BotSharp.Core/Engines/Nltk/NltkTokenizer.cs

diff --git a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs
new file mode 100644
index 00000000..6191ae03
--- /dev/null
+++ b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs
@@ -0,0 +1,97 @@
+using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using BotSharp.Core.Models;
+using BotSharp.MachineLearning.NLP;
+using EntityFrameworkCore.BootKit;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using RestSharp;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Core.Engines.SpaCy
+{
+    public class NltkTokenizer : INlpTrain, INlpPredict
+    {
+        public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
+
+        public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
+        {
+            var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
+            var request = new RestRequest("nltktokenizesentences", Method.GET);
+            List<List<NlpToken>> tokens = new List<List<NlpToken>>();
+            Boolean res = true;
+            var dc = new DefaultDataContextLoader().GetDefaultDc();
+            var corpus = agent.Corpus;
+
+            doc.Sentences = new List<NlpDocSentence>();
+            List<string> sentencesList = new List<string>();
+            corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text));
+
+
+
+
+            request.RequestFormat = DataFormat.Json;
+
+            request.AddParameter("application/json", JsonConvert.SerializeObject(new { sentences = sentencesList}));
+
+            var response = client.Execute<Result>(request);
+
+
+
+
+
+            /*
+             corpus.UserSays.ForEach(usersay => {
+                Console.WriteLine(usersay.Text);
+                request.AddParameter("text", usersay.Text);
+                var response = client.Execute<Result>(request);
+                
+                tokens.Add(response.Data.Tokens);
+
+                doc.Sentences.Add(new NlpDocSentence
+                {
+                    Tokens = response.Data.Tokens,
+                    Text = usersay.Text
+                });
+
+                res = res && response.IsSuccessful;
+                
+            });
+             */
+           
+
+            return res;
+        }
+
+        public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
+        {
+            var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
+            var request = new RestRequest("nltktokenizesentences", Method.POST);
+            List<List<NlpToken>> tokens = new List<List<NlpToken>>();
+            Boolean res = true;
+            var corpus = agent.Corpus;
+
+            request.AddParameter("sentences", doc.Sentences[0].Text);
+            var response = client.Execute<Result>(request);
+            
+            //tokens.Add(response.Data.Tokens);
+
+            res = res && response.IsSuccessful;
+
+            doc.Sentences[0].Tokens = tokens[0];
+
+            return true;
+        }
+
+        private class Result
+        {
+            public List<List<NlpToken>> Tokens { get; set; }
+        }
+    }
+}
diff --git a/BotSharp.WebHost/Settings/bot.json b/BotSharp.WebHost/Settings/bot.json
index 7765b13d..cea88c28 100644
--- a/BotSharp.WebHost/Settings/bot.json
+++ b/BotSharp.WebHost/Settings/bot.json
@@ -9,10 +9,12 @@
     "SpaCyProvider": {
       "url": "http://localhost:5005"
     },
-
+    "NltkProvider": {
+      "url": "http://localhost:5005"
+    },
     "Pipe": {
-      "train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
-      "predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer"
+      "train": "NltkTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
+      "predict": "NltkTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer"
     },
 
     "SpaCyTokenizer": {