diff --git a/BotSharp.Core/AgentStorage/AgentStorageInFile.cs b/BotSharp.Core/AgentStorage/AgentStorageInFile.cs index a930be30..f937bb37 100644 --- a/BotSharp.Core/AgentStorage/AgentStorageInFile.cs +++ b/BotSharp.Core/AgentStorage/AgentStorageInFile.cs @@ -3,6 +3,7 @@ using BotSharp.Platform.Models; using CSRedis; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; using System.IO; @@ -76,6 +77,11 @@ namespace BotSharp.Core.AgentStorage { NullValueHandling = NullValueHandling.Ignore, Formatting = Formatting.Indented, + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = new List + { + new Newtonsoft.Json.Converters.StringEnumConverter() + } }); string dataPath = Path.Combine(storageDir, agent.Id + ".json"); diff --git a/BotSharp.Core/Engines/BotPredictor.cs b/BotSharp.Core/Engines/BotPredictor.cs index ec9c6f7d..5c27007c 100644 --- a/BotSharp.Core/Engines/BotPredictor.cs +++ b/BotSharp.Core/Engines/BotPredictor.cs @@ -22,7 +22,7 @@ namespace BotSharp.Core.Engines { public async Task Predict(AgentBase agent, AiRequest request) { - // load model + // load model per context var dir = Path.Combine(request.AgentDir, request.Model); Console.WriteLine($"Load model from {dir}"); var metaJson = File.ReadAllText(Path.Combine(dir, "model-meta.json")); diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 35c00a95..3833ff20 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Text; @@ -94,6 +95,9 @@ namespace BotSharp.Core.Engines for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++) { + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain; // set configuration to current section pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]); @@ -105,8 +109,11 @@ namespace BotSharp.Core.Engines Time = DateTime.UtcNow }; meta.Pipeline.Add(pipeModel); - + await pipe.Train(agent, data, pipeModel); + + stopwatch.Stop(); + Console.WriteLine($"Executed pipe {pipeModel.Name} elapsed {stopwatch.Elapsed}"); } // save model meta data @@ -118,8 +125,6 @@ namespace BotSharp.Core.Engines }); File.WriteAllText(Path.Combine(settings.ModelDir, "model-meta.json"), metaJson); - Console.WriteLine(metaJson); - return meta; } } diff --git a/BotSharp.Core/PlatformBuilderBase.cs b/BotSharp.Core/PlatformBuilderBase.cs index 334f8c2b..2becc2b3 100644 --- a/BotSharp.Core/PlatformBuilderBase.cs +++ b/BotSharp.Core/PlatformBuilderBase.cs @@ -1,6 +1,9 @@ using BotSharp.Core.Engines; +using BotSharp.Models.NLP; using BotSharp.Platform.Abstraction; using BotSharp.Platform.Models; +using BotSharp.Platform.Models.AiRequest; +using BotSharp.Platform.Models.AiResponse; using BotSharp.Platform.Models.MachineLearning; using DotNetToolkit; using Microsoft.Extensions.Configuration; @@ -16,6 +19,8 @@ namespace BotSharp.Core { public abstract class PlatformBuilderBase where TAgent : AgentBase { + public TAgent Agent { get; set; } + public IAgentStorage Storage { get; set; } private readonly IAgentStorageFactory agentStorageFactory; @@ -58,6 +63,8 @@ namespace BotSharp.Core Console.WriteLine($"Loaded agent: {agent.Name} {agent.Id}"); + Agent = agent; + return agent; } @@ -95,12 +102,81 @@ namespace BotSharp.Core options.Model = "model_" + DateTime.UtcNow.ToString("yyyyMMdd"); } - var trainer = new BotTrainer(settings); - agent.Corpus = corpus; + ModelMetaData meta = null; - var info = await trainer.Train(agent, options); + // train by contexts + corpus.UserSays.GroupBy(x => x.ContextHash).Select(g => new + { + Context = g.Key, + Corpus = new TrainingCorpus + { + Entities = corpus.Entities, + UserSays = corpus.UserSays.Where(x => x.ContextHash == g.Key).ToList() + } + }) + .ToList() + .ForEach(async c => + { + var trainer = new BotTrainer(settings); + agent.Corpus = c.Corpus; - return info; + meta = await trainer.Train(agent, new BotTrainOptions + { + AgentDir = options.AgentDir, + Model = options.Model + $"{Path.DirectorySeparatorChar}{c.Context}" + }); + }); + + meta.Pipeline.Clear(); + meta.Model = options.Model; + + return meta; + } + + public virtual async Task TextRequest(AiRequest request) + { + string contexts = String.Join("_", request.Contexts); + string contextHash = contexts.GetMd5Hash(); + + Console.WriteLine($"TextRequest: {request.Text}, {contexts}, {request.SessionId}"); + + // Load agent + var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.AgentId); + var model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last(); + var modelPath = Path.Combine(projectPath, model); + request.AgentDir = projectPath; + request.Model = model + $"{Path.DirectorySeparatorChar}{contextHash}"; + + Agent = await GetAgentById(request.AgentId); + + var preditor = new BotPredictor(); + var doc = await preditor.Predict(Agent, request); + + var parameters = new Dictionary(); + if (doc.Sentences[0].Entities == null) + { + doc.Sentences[0].Entities = new List(); + } + doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value); + + var predictedIntent = doc.Sentences[0].Intent; + + var aiResponse = new AiResponse + { + ResolvedQuery = request.Text, + Score = predictedIntent.Confidence, + Source = predictedIntent.Classifier, + Intent = predictedIntent.Label + }; + + Console.WriteLine($"TextResponse: {aiResponse.Intent}, {request.SessionId}"); + + return await AssembleResult(aiResponse); + } + + public virtual async Task AssembleResult(AiResponse response) + { + throw new NotImplementedException(); } public virtual async Task SaveAgent(TAgent agent) diff --git a/BotSharp.Platform.Abstraction/IPlatformBuilder.cs b/BotSharp.Platform.Abstraction/IPlatformBuilder.cs index 0a58b276..499c2e63 100644 --- a/BotSharp.Platform.Abstraction/IPlatformBuilder.cs +++ b/BotSharp.Platform.Abstraction/IPlatformBuilder.cs @@ -12,6 +12,8 @@ namespace BotSharp.Platform.Abstraction /// public interface IPlatformBuilder { + TAgent Agent { get; set; } + /// /// Agent storage /// @@ -48,5 +50,7 @@ namespace BotSharp.Platform.Abstraction Task Train(TAgent agent, TrainingCorpus corpus, BotTrainOptions options); Task TextRequest(AiRequest request); + + Task AssembleResult(AiResponse response); } } diff --git a/BotSharp.Platform.Models/AiRequest/AIRequest.cs b/BotSharp.Platform.Models/AiRequest/AIRequest.cs index 9a924810..4fa2f37f 100644 --- a/BotSharp.Platform.Models/AiRequest/AIRequest.cs +++ b/BotSharp.Platform.Models/AiRequest/AIRequest.cs @@ -6,12 +6,19 @@ namespace BotSharp.Platform.Models.AiRequest { public class AiRequest { + public AiRequest() + { + Contexts = new List(); + } + public string AgentId { get; set; } public string Text { get; set; } public string SessionId { get; set; } + public List Contexts { get; set; } + public bool ResetContexts { get; set; } /// diff --git a/BotSharp.Platform.Models/AiResponse/AIResponse.cs b/BotSharp.Platform.Models/AiResponse/AIResponse.cs index 4842bd96..1b6c3184 100644 --- a/BotSharp.Platform.Models/AiResponse/AIResponse.cs +++ b/BotSharp.Platform.Models/AiResponse/AIResponse.cs @@ -6,8 +6,12 @@ namespace BotSharp.Platform.Models.AiResponse { public class AiResponse { - public string Speech { get; set; } + public String ResolvedQuery { get; set; } public string Intent { get; set; } + + public string Source { get; set; } + + public double Score { get; set; } } } diff --git a/BotSharp.Platform.Models/Intents/Intent.cs b/BotSharp.Platform.Models/Intents/Intent.cs index 2d0559b6..c033a902 100644 --- a/BotSharp.Platform.Models/Intents/Intent.cs +++ b/BotSharp.Platform.Models/Intents/Intent.cs @@ -32,16 +32,7 @@ namespace BotSharp.Platform.Models.Intents /// Get input contexts hash /// [NotMapped] - public String ContextHash - { - get - { - return string.Empty; - /*return Contexts == null || Contexts.Count == 0 - ? Guid.Empty.ToString("N") - : $"{String.Join(",", Contexts.OrderBy(x => x.Name).Select(x => x.Name))}".GetMd5Hash();*/ - } - } + public String ContextHash { get; set; } [ForeignKey("IntentId")] public List UserSays { get; set; }