From 60844b94b26c9e9a76f2244c2a074c7ea56c0798 Mon Sep 17 00:00:00 2001 From: "haiping008@gmail.com" Date: Mon, 6 Aug 2018 10:51:49 -0500 Subject: [PATCH 1/3] Fix get corpus is empty for Sebis trainer. --- BotSharp.Core/Engines/BotTrainer.cs | 5 +---- .../Engines/Sebis/AgentImporterInSebis.cs | 6 ++++-- BotSharp.RestApi/AgentController.cs | 21 ++++++++++++++----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index a7812a44..9065a3f2 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -19,13 +19,10 @@ namespace BotSharp.Core.Engines private string agentId; - private string config; - - public BotTrainer(string agentId, Database dc, string config = "BotSharpAi") + public BotTrainer(string agentId, Database dc) { this.dc = dc; this.agentId = agentId; - this.config = config; } public string Train(Agent agent) diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index 32afb413..25e86fe0 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -58,11 +58,11 @@ namespace BotSharp.Core.Engines agent.Intents = sentences.Select(x => x.Name).Distinct().Select(x => new Intent{Name = x}).ToList(); agent.Intents.ForEach(intent => { - ImportIntentUserSays(agent, intent, sentences); + ImportIntentUserSays(intent, sentences); }); } - private void ImportIntentUserSays(Agent agent, Intent intent, List sentences) + private void ImportIntentUserSays(Intent intent, List sentences) { intent.UserSays = new List(); @@ -113,6 +113,8 @@ namespace BotSharp.Core.Engines }); } } + + intent.UserSays.Add(expression); }); } diff --git a/BotSharp.RestApi/AgentController.cs b/BotSharp.RestApi/AgentController.cs index 153b8bb8..10f74cf8 100644 --- a/BotSharp.RestApi/AgentController.cs +++ b/BotSharp.RestApi/AgentController.cs @@ -2,8 +2,10 @@ using BotSharp.Core.Engines; using BotSharp.Core.Engines.BotSharp; using BotSharp.Core.Models; +using DotNetToolkit; using EntityFrameworkCore.BootKit; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using System; using System.Collections.Generic; @@ -19,6 +21,17 @@ namespace BotSharp.RestApi [Route("v1/[controller]/[action]")] public class AgentController : ControllerBase { + private readonly IBotPlatform _platform; + + /// + /// Initialize dialog controller and get a platform instance + /// + /// + public AgentController(IBotPlatform platform) + { + _platform = platform; + } + /// /// Restore a agent from a uploaded zip file /// @@ -45,9 +58,8 @@ namespace BotSharp.RestApi [HttpGet("{agentId}")] public string Train([FromRoute] String agentId) { - var ai = new BotSharpAi(); - ai.LoadAgent("bff7605c-3db5-44dc-9ba7-1c9be2832318"); - ai.Train(); + _platform.LoadAgent(agentId); + _platform.Train(); return ""; } @@ -60,8 +72,7 @@ namespace BotSharp.RestApi [HttpGet("{agentId}")] public ActionResult Dump([FromRoute] String agentId) { - var rasa = new RasaAi(); - var agent = rasa.LoadAgent(agentId); + var agent = _platform.LoadAgent(agentId); return Ok(agent); } From 6b7f9671f32ef35832c8121daae47f342a2f07a3 Mon Sep 17 00:00:00 2001 From: "haiping008@gmail.com" Date: Mon, 6 Aug 2018 11:59:03 -0500 Subject: [PATCH 2/3] Fix space missing before entity. --- BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index 25e86fe0..1f59dd62 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -102,7 +102,7 @@ namespace BotSharp.Core.Engines Text = say.Text.Substring(entity.Start, entity.Value.Length) }); - pos = entity.End + 1; + pos = entity.End; if (pos < say.Text.Length && entityIdx == say.Entities.Count - 1) { From b389d837f25afa256b35d0418274967557fb06f9 Mon Sep 17 00:00:00 2001 From: "haiping008@gmail.com" Date: Mon, 6 Aug 2018 13:39:26 -0500 Subject: [PATCH 3/3] Fix Sebis import expression data sequence issue. --- BotSharp.Core/Engines/BotEngineBase.cs | 4 +++- .../CRFsuite/CRFsuiteEntityRecognizer.cs | 10 +++++++--- .../Engines/Sebis/AgentImporterInSebis.cs | 18 ++++++++++++------ BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs | 4 ++-- BotSharp.WebHost/BotSharp.WebHost.csproj | 1 + 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index 62251799..6445072b 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -120,10 +120,12 @@ namespace BotSharp.Core.Engines { intent.UserSays.ForEach(exp => { + exp.Data = exp.Data.OrderBy(x => x.UpdatedTime).ToList(); + var say = new TrainingIntentExpression { Intent = intent.Name, - Text = String.Join("", exp.Data.OrderBy(x => x.UpdatedTime).Select(x => x.Text)), + Text = String.Join("", exp.Data.Select(x => x.Text)), ContextHash = intent.ContextHash }; diff --git a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs index 7058db0e..d601eb33 100644 --- a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs @@ -27,9 +27,10 @@ namespace BotSharp.Core.Engines.CRFsuite List> tags = data["Tags"].ToObject>>(); List> tokens = data["Tokens"].ToObject>>(); List> userSays = corpus.UserSays; - List> list =new List>(); + List> list = new List>(); - FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt", FileMode.Create); + var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles"); + FileStream fs = new FileStream(Path.Join(dir, "rawTrain.txt"), FileMode.Create); StreamWriter sw = new StreamWriter(fs); for (int i = 0 ; i < tags.Count; i++) @@ -56,7 +57,10 @@ namespace BotSharp.Core.Engines.CRFsuite public void Runcmd () { - string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite/1.txt"; + var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); + var dataDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles"); + + string cmd = $"{algorithmDir}/crfsuite learn -m {dataDir}/crfsuite/bolo.model {dataDir}/crfsuite/1.txt"; System.Diagnostics.Process p = new System.Diagnostics.Process(); p.StartInfo.FileName = "sh"; p.StartInfo.UseShellExecute = false; diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index 1f59dd62..ba849bfd 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -78,7 +78,7 @@ namespace BotSharp.Core.Engines say.Entities.ForEach(x => { ConvertWordPosToCharPos(say.Text, x, pos); - pos = x.End; + pos = x.End + 1; }); expression.Data = new List(); @@ -89,10 +89,13 @@ namespace BotSharp.Core.Engines var entity = say.Entities[entityIdx]; // previous - expression.Data.Add(new IntentExpressionPart + if(entity.Start > 0) { - Text = say.Text.Substring(pos, entity.Start - pos) - }); + expression.Data.Add(new IntentExpressionPart + { + Text = say.Text.Substring(pos, entity.Start - pos) + }); + } // self expression.Data.Add(new IntentExpressionPart @@ -102,7 +105,7 @@ namespace BotSharp.Core.Engines Text = say.Text.Substring(entity.Start, entity.Value.Length) }); - pos = entity.End; + pos = entity.End + 1; if (pos < say.Text.Length && entityIdx == say.Entities.Count - 1) { @@ -114,6 +117,9 @@ namespace BotSharp.Core.Engines } } + int second = 0; + expression.Data.ForEach(x => x.UpdatedTime = DateTime.UtcNow.AddSeconds(second++)); + intent.UserSays.Add(expression); }); } @@ -121,7 +127,7 @@ namespace BotSharp.Core.Engines private void ConvertWordPosToCharPos(string text, SebisIntentExpressionPart entity, int start) { entity.Start = text.IndexOf(entity.Value, start); - entity.End = entity.Start + entity.Value.Length; + entity.End = entity.Start + entity.Value.Length - 1; } public void LoadBuildinEntities(Agent agent, string agentDir) diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs index 272cb843..46305e6f 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs @@ -11,12 +11,12 @@ using BotSharp.MachineLearning.NLP; namespace BotSharp.Core.Engines.SpaCy { - class SpaCyTagger : INlpPipeline + public class SpaCyTagger : INlpPipeline { public IConfiguration Configuration { get; set; } -public bool Process(Agent agent, JObject data) + public bool Process(Agent agent, JObject data) { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tagger", Method.GET); diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index 458f02fd..757b271d 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -57,6 +57,7 @@ +