From b4a8d4919cf9299639533cbc4eba24899a17f32c Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 1 Oct 2018 16:28:40 -0500 Subject: [PATCH] make IAgentStorage configurable in json. --- BotSharp.Core.UnitTest/BotTrainerTest.cs | 1 + .../BotSharp/BotSharpCBOWClassifier.cs | 65 ------------------- .../Engines/BotSharp/BotSharpCRFNer.cs | 2 +- BotSharp.Core/Engines/BotTrainer.cs | 8 +-- BotSharp.Core/Engines/PipeSettings.cs | 8 --- BotSharp.NLP/Classify/SVMClassifier.cs | 3 +- BotSharp.RestApi/Rasa/TrainController.cs | 1 + Platform.Articulate/ArticulateAi.cs | 49 +++++++++++++- .../Controllers/ParseControllercs.cs | 4 +- .../Controllers/TrainController.cs | 5 ++ 10 files changed, 60 insertions(+), 86 deletions(-) delete mode 100644 BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs diff --git a/BotSharp.Core.UnitTest/BotTrainerTest.cs b/BotSharp.Core.UnitTest/BotTrainerTest.cs index 2f232448..c1518ad0 100644 --- a/BotSharp.Core.UnitTest/BotTrainerTest.cs +++ b/BotSharp.Core.UnitTest/BotTrainerTest.cs @@ -1,6 +1,7 @@ using BotSharp.Core.Engines; using BotSharp.Core.Engines.BotSharp; using BotSharp.Core.Models; +using BotSharp.Platform.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs deleted file mode 100644 index b3db17a7..00000000 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs +++ /dev/null @@ -1,65 +0,0 @@ -using BotSharp.Core.Abstractions; -using BotSharp.Core.Agents; -using DotNetToolkit; -using Microsoft.Extensions.Configuration; -using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace BotSharp.Core.Engines.BotSharp -{ - public class BotSharpCBOWClassifier : INlpTrain, INlpPredict - { - public IConfiguration Configuration { get; set; } - - public PipeSettings Settings { get; set; } - - public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) - { - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt"); - File.WriteAllText(predictFileName, doc.Sentences[0].Text); - - var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\""); - - File.Delete(predictFileName); - - if (!String.IsNullOrEmpty(output)) - { - doc.Sentences[0].Intent = new TextClassificationResult - { - Classifier = "FasttextClassifier", - Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], - Confidence = decimal.Parse(output.Split(' ')[1]) - }; - } - - return true; - } - - public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) - { - meta.Model = "classification-fasttext.model"; - - string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt"); - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - - // assemble corpus - StringBuilder corpus = new StringBuilder(); - agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}")); - - File.WriteAllText(parsedTrainingDataFileName, corpus.ToString()); - - var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false); - - Console.WriteLine($"Saved model to {modelFileName}"); - - return true; - } - } -} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs index 09d8bedd..050dc9b5 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs @@ -31,7 +31,7 @@ namespace BotSharp.Core.Engines.BotSharp List> userSays = corpus.UserSays; List> list = new List>(); - string rawTrainingDataFileName = System.IO.Path.Combine(Settings.TempDir, "ner-crf.corpus.txt"); + string rawTrainingDataFileName = System.IO.Path.Combine(Settings.ModelDir, "ner-crf.corpus.txt"); string modelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model); using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 650092c1..c97a509a 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -58,8 +58,7 @@ namespace BotSharp.Core.Engines var settings = new PipeSettings { - ProjectDir = options.AgentDir, - AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") + ProjectDir = options.AgentDir }; settings.ModelDir = Path.Combine(options.AgentDir, options.Model); @@ -69,11 +68,6 @@ namespace BotSharp.Core.Engines Directory.CreateDirectory(settings.ProjectDir); } - if (!Directory.Exists(settings.TempDir)) - { - Directory.CreateDirectory(settings.TempDir); - } - if (!Directory.Exists(settings.ModelDir)) { Directory.CreateDirectory(settings.ModelDir); diff --git a/BotSharp.Core/Engines/PipeSettings.cs b/BotSharp.Core/Engines/PipeSettings.cs index afe29f18..e2881140 100644 --- a/BotSharp.Core/Engines/PipeSettings.cs +++ b/BotSharp.Core/Engines/PipeSettings.cs @@ -9,13 +9,5 @@ namespace BotSharp.Core.Engines { public string ProjectDir { get; set; } public string ModelDir { get; set; } - public string AlgorithmDir { get; set; } - public string TempDir - { - get - { - return Path.Combine(ProjectDir, "Temp"); - } - } } } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 9bcec55e..4b28ee9e 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -66,8 +66,9 @@ namespace BotSharp.NLP.Classify int numberOfClasses = train.Y.OrderBy(x => x).Distinct().Count(); if (numberOfClasses == 1) { - throw new ArgumentException("Number of classes can't be one!"); + Console.Write("Number of classes must greater than one!"); } + if (svm == SvmType.C_SVC) { for (int i = 0; i < numberOfClasses; i++) diff --git a/BotSharp.RestApi/Rasa/TrainController.cs b/BotSharp.RestApi/Rasa/TrainController.cs index 8412a139..a9d038f6 100644 --- a/BotSharp.RestApi/Rasa/TrainController.cs +++ b/BotSharp.RestApi/Rasa/TrainController.cs @@ -1,6 +1,7 @@ using BotSharp.Core.Agents; using BotSharp.Core.Engines; using BotSharp.Core.Engines.Rasa; +using BotSharp.Platform.Models; using Microsoft.AspNetCore.Mvc; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/Platform.Articulate/ArticulateAi.cs b/Platform.Articulate/ArticulateAi.cs index 62057895..baac23d0 100644 --- a/Platform.Articulate/ArticulateAi.cs +++ b/Platform.Articulate/ArticulateAi.cs @@ -1,6 +1,9 @@ using BotSharp.Core; using BotSharp.Core.Agents; using BotSharp.Core.Engines; +using BotSharp.Core.Entities; +using BotSharp.Core.Intents; +using BotSharp.Core.Models; using BotSharp.Platform.Abstraction; using BotSharp.Platform.Models; using DotNetToolkit; @@ -139,11 +142,53 @@ namespace Platform.Articulate var trainer = new BotTrainer(); var parsedAgent = agent.ToObject(); - var info = await trainer.Train(parsedAgent, new BotTrainOptions + var intents = new List>(); + + foreach (DomainModel domain in (agent as AgentModel).Domains) + { + foreach (IntentModel intent in domain.Intents) + { + foreach (IntentExampleModel example in intent.Examples) + { + var parsedIntent = new TrainingIntentExpression + { + Intent = intent.IntentName, + Text = example.UserSays, + Entities = example.Entities.Select(x => new TrainingIntentExpressionPart + { + Entity = x.Entity, + Start = x.Start, + Value = x.Value + }).ToList() + }; + + intents.Add(parsedIntent); + } + } + } + + parsedAgent.Corpus = new TrainingCorpus + { + Entities = (agent as AgentModel).Entities.Select(x => new TrainingEntity + { + Entity = x.EntityName, + Values = x.Examples.Select(y => new TrainingEntitySynonym + { + Value = y.Value, + Synonyms = y.Synonyms + }).ToList() + }).ToList(), + + UserSays = intents + }; + + var trainOptions = new BotTrainOptions { AgentDir = projectPath, Model = model - }); + }; + + var info = await trainer.Train(parsedAgent, trainOptions); return true; } diff --git a/Platform.Articulate/Controllers/ParseControllercs.cs b/Platform.Articulate/Controllers/ParseControllercs.cs index 1e93215d..3db1c560 100644 --- a/Platform.Articulate/Controllers/ParseControllercs.cs +++ b/Platform.Articulate/Controllers/ParseControllercs.cs @@ -13,9 +13,9 @@ namespace Platform.Articulate.Controllers private readonly IConfiguration configuration; [HttpGet("/agent/{agentId}/converse")] - public void ParseText([FromRoute] string agentId, [FromQuery] string text, [FromQuery] string sessionId) + public ActionResult ParseText([FromRoute] string agentId, [FromQuery] string text, [FromQuery] string sessionId) { - + return Ok(); } } #endif diff --git a/Platform.Articulate/Controllers/TrainController.cs b/Platform.Articulate/Controllers/TrainController.cs index 9db75630..5764068a 100644 --- a/Platform.Articulate/Controllers/TrainController.cs +++ b/Platform.Articulate/Controllers/TrainController.cs @@ -26,10 +26,15 @@ namespace Platform.Articulate.Controllers public async Task TrainAgent([FromRoute] string agentId) { var agent = builder.GetAgentById(agentId); + var corpus = builder.ExtractorCorpus(agent); + await builder.Train(agent, corpus); + agent.Status = "Ready"; + builder.SaveAgent(agent); + return agent; } }