From 47623056f76f0ef65b3ed729a03d28797036505c Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Thu, 30 Aug 2018 08:04:10 -0500 Subject: [PATCH] Add quick question and answer function. --- .gitignore | 1 + BotSharp.Core/Engines/BotEngineBase.cs | 15 ++- BotSharp.Core/Engines/BotTrainer.cs | 8 +- .../Engines/Classifiers/FasttextClassifier.cs | 4 +- .../Engines/NERs/CRFsuiteEntityRecognizer.cs | 4 +- .../Engines/QuickQA/AgentImporterInQuickQA.cs | 94 +++++++++++++++++++ BotSharp.Core/Engines/Rasa/RasaResponse.cs | 2 + BotSharp.RestApi/Rasa/ParseController.cs | 3 +- 8 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 BotSharp.Core/Engines/QuickQA/AgentImporterInQuickQA.cs diff --git a/.gitignore b/.gitignore index 8145526a..3a124b12 100644 --- a/.gitignore +++ b/.gitignore @@ -302,3 +302,4 @@ __pycache__/ /Data /docs/build /docs/_build +/BotSharp.WebHost/App_Data/AgentArchive/Smart Niraj.zip diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index 78f12967..292cfba3 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -1,8 +1,11 @@ using BotSharp.Core.Agents; +using BotSharp.Core.Engines.QuickQA; using BotSharp.Core.Engines.Rasa; using BotSharp.Core.Entities; using BotSharp.Core.Intents; using BotSharp.Core.Models; +using BotSharp.Models.NLP; +using DotNetToolkit; using EntityFrameworkCore.BootKit; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; @@ -36,6 +39,10 @@ namespace BotSharp.Core.Engines var preditor = new BotPredictor(); var doc = preditor.Predict(agent, request).Result; var parameters = new Dictionary(); + if(doc.Sentences[0].Entities == null) + { + doc.Sentences[0].Entities = new List(); + } doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value)); return new AIResponse @@ -48,7 +55,10 @@ namespace BotSharp.Core.Engines { Score = doc.Sentences[0].Intent == null ? 0 : doc.Sentences[0].Intent.Confidence, ResolvedQuery = doc.Sentences[0].Text, - Fulfillment = new AIResponseFulfillment { }, + Fulfillment = new AIResponseFulfillment + { + Speech = agent.Intents.FirstOrDefault(tnt => tnt.Name == doc.Sentences[0].Intent?.Label)?.Responses?.Random()?.Messages?.Random()?.Speech + }, Parameters = parameters, Entities = doc.Sentences[0].Entities, Metadata = new AIResponseMetadata @@ -111,6 +121,9 @@ namespace BotSharp.Core.Engines case "Sebis": importer = new AgentImporterInSebis(); break; + case "QuickQA": + importer = new AgentImporterInQuickQA(); + break; default: break; } diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index ab263971..d3933a13 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -94,21 +94,21 @@ namespace BotSharp.Core.Engines .Select(x => x.Trim()) .ToList(); - pipelines.ForEach(async pipeName => + for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++) { - var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpTrain; + var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain; pipe.Configuration = provider.Configuration; pipe.Settings = settings; pipeModel = new PipeModel { - Name = pipeName, + Name = pipelines[pipeIdx], Class = pipe.ToString(), Time = DateTime.UtcNow }; meta.Pipeline.Add(pipeModel); await pipe.Train(agent, data, pipeModel); - }); + } // save model meta data var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs index 7dcff604..1e8ae9a4 100644 --- a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs +++ b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs @@ -25,7 +25,7 @@ namespace BotSharp.Core.Engines.Classifiers string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt"); File.WriteAllText(predictFileName, doc.Sentences[0].Text); - var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}"); + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\""); File.Delete(predictFileName); @@ -52,7 +52,7 @@ namespace BotSharp.Core.Engines.Classifiers File.WriteAllText(parsedTrainingDataFileName, corpus.ToString()); - var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}", false); + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false); Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); diff --git a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs index 0d3d44ed..b1d85122 100644 --- a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs @@ -79,7 +79,7 @@ namespace BotSharp.Core.Engines.NERs var algorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); - CmdHelper.Run(Path.Combine(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}", false); // --split=3 -x + CmdHelper.Run(Path.Combine(algorithmDir, "crfsuite"), $"learn -m \"{modelFileName}\" \"{parsedTrainingDataFileName}\"", false); // --split=3 -x Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); meta.Meta["fields"] = fields; @@ -197,7 +197,7 @@ namespace BotSharp.Core.Engines.NERs new NLP.Models.CRFsuite.Ner() .NerStart(rawPredictingDataFileName, parsedPredictingDataFileName, field, uniFeatures.Split(' '), biFeatures.Split(' ')); - var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "crfsuite"), $"tag -i -m {modelFileName} {parsedPredictingDataFileName}", false); + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "crfsuite"), $"tag -i -m \"{modelFileName}\" \"{parsedPredictingDataFileName}\"", false); var entities = new List(); diff --git a/BotSharp.Core/Engines/QuickQA/AgentImporterInQuickQA.cs b/BotSharp.Core/Engines/QuickQA/AgentImporterInQuickQA.cs new file mode 100644 index 00000000..0bf54ba6 --- /dev/null +++ b/BotSharp.Core/Engines/QuickQA/AgentImporterInQuickQA.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using BotSharp.Core.Agents; +using BotSharp.Core.Intents; +using Newtonsoft.Json; + +namespace BotSharp.Core.Engines.QuickQA +{ + public class AgentImporterInQuickQA : IAgentImporter + { + public string AgentDir { get; set; } + + public Agent LoadAgent(AgentImportHeader agentHeader) + { + var agent = new Agent(); + agent.ClientAccessToken = Guid.NewGuid().ToString("N"); + agent.DeveloperAccessToken = Guid.NewGuid().ToString("N"); + agent.Id = agentHeader.Id; + agent.Name = agentHeader.Name; + + return agent; + } + + public void LoadBuildinEntities(Agent agent) + { + + } + + public void LoadCustomEntities(Agent agent) + { + + } + + public void LoadIntents(Agent agent) + { + string lines = File.ReadAllText(Path.Combine(AgentDir, "corpus.txt")); + + var questions = Regex.Matches(lines, @"^Q - .+\n", RegexOptions.Multiline).Cast().ToArray(); + var answers = Regex.Matches(lines, @"^A - ", RegexOptions.Multiline).Cast().ToArray(); + + int qNumber = 1; + agent.Intents = questions.Select(x => new Intent + { + Name = $"Q{qNumber++}", + UserSays = new List + { + new IntentExpression + { + Data = new List + { + new IntentExpressionPart + { + Text = x.Value.Substring(4).Trim() + } + } + } + } + }).ToList(); + + // assemble answers + for (int idx = 0; idx < agent.Intents.Count(); idx++) + { + var intent = agent.Intents[idx]; + var answer = answers[idx]; + + var start = answer.Index + 4; + var length = ((idx == agent.Intents.Count() - 1) ? lines.Length : questions[idx + 1].Index) - answer.Index - 4; + + intent.Responses = new List + { + new IntentResponse + { + Messages = new List + { + new IntentResponseMessage + { + Speech = lines.Substring(start, length).Trim() + } + } + } + }; + } + } + + public void AssembleTrainData(Agent agent) + { + + } + } +} diff --git a/BotSharp.Core/Engines/Rasa/RasaResponse.cs b/BotSharp.Core/Engines/Rasa/RasaResponse.cs index b9f5e374..6aedf5fe 100644 --- a/BotSharp.Core/Engines/Rasa/RasaResponse.cs +++ b/BotSharp.Core/Engines/Rasa/RasaResponse.cs @@ -9,6 +9,8 @@ namespace BotSharp.Core.Models { public RasaResponseIntent Intent { get; set; } + public AIResponseFulfillment Fullfillment { get; set; } + [JsonProperty("intent_ranking")] public List IntentRanking { get; set; } diff --git a/BotSharp.RestApi/Rasa/ParseController.cs b/BotSharp.RestApi/Rasa/ParseController.cs index b55a92ac..35659bae 100644 --- a/BotSharp.RestApi/Rasa/ParseController.cs +++ b/BotSharp.RestApi/Rasa/ParseController.cs @@ -95,7 +95,8 @@ namespace BotSharp.RestApi.Rasa Name = aIResponse.Result.Metadata.IntentName, Confidence = aIResponse.Result.Score } - } + }, + Fullfillment = aIResponse.Result.Fulfillment }; return rasaResponse;