From 21943002c0ae3cbcc45580a2ceab1f845fe98d53 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 28 Aug 2023 13:57:51 -0500 Subject: [PATCH] add basic mongo store --- .../Records/ConversationRecord.cs | 3 + .../Utilities/StringExtensions.cs | 5 ++ .../Agents/Services/AgentRouter.cs | 16 +++- .../Agents/Services/AgentService.GetAgents.cs | 21 ++--- .../Services/ConversationService.cs | 3 +- .../Services/ConversationStateService.cs | 77 ++++++++++++++++--- .../Services/ConversationStorage.cs | 74 ++++++++++++++---- .../Repository/MongoRepository.cs | 8 ++ 8 files changed, 172 insertions(+), 35 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs index 6a4dc338..6ad0b25d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs @@ -15,6 +15,9 @@ public class ConversationRecord : RecordBase [MaxLength(64)] public string Title { get; set; } = string.Empty; + public string Dialog { get; set; } = string.Empty; + public string State { get; set; } = string.Empty; + [Required] public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index 651adad7..107461cb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -33,4 +33,9 @@ public static class StringExtensions return phoneNumber; } + + public static string[] SplitByNewLine(this string input) + { + return input.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); + } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs index ac114b3c..a636adc4 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs @@ -48,7 +48,19 @@ public class AgentRouter : IAgentRouting { var agentSettings = _services.GetRequiredService(); var dbSettings = _services.GetRequiredService(); - var filePath = Path.Combine(dbSettings.FileRepository, agentSettings.DataDir, agentSettings.RouterId, "route.json"); - return JsonSerializer.Deserialize(File.ReadAllText(filePath)); + //var filePath = Path.Combine(dbSettings.FileRepository, agentSettings.DataDir, agentSettings.RouterId, "route.json"); + + var db = _services.GetRequiredService(); + var agent = db.Agent.FirstOrDefault(x => x.Id == agentSettings.RouterId); + var routes = agent?.Routes ?? new List(); + var routingRecords = new RoutingRecord[routes.Count]; + + for (int i = 0; i < routes.Count; i++) + { + if (routes[i] == null) continue; + routingRecords[i] = JsonSerializer.Deserialize(routes[i]); + } + + return routingRecords; } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index b3a21240..2313da0c 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -25,27 +25,28 @@ public partial class AgentService select agent.ToAgent(); var profile = query.FirstOrDefault(); - var dir = GetAgentDataDir(id); + //var dir = GetAgentDataDir(id); - var instructionFile = Path.Combine(dir, $"instruction.{_settings.TemplateFormat}"); - if (File.Exists(instructionFile)) + var instructionFile = profile?.Instruction; + if (instructionFile != null) { - profile.Instruction = File.ReadAllText(instructionFile); + profile.Instruction = instructionFile; } else { _logger.LogError($"Can't find instruction file from {instructionFile}"); } - var samplesFile = Path.Combine(dir, $"samples.{_settings.TemplateFormat}"); - if (File.Exists(samplesFile)) + var samplesFile = profile?.Samples; + if (samplesFile != null) { - profile.Samples = File.ReadAllText(samplesFile); + profile.Samples = samplesFile; } - var functionsFile = Path.Combine(dir, "functions.json"); - if (File.Exists(functionsFile)) + + var functionsFile = profile?.Functions; + if (functionsFile != null) { - //profile.Functions = File.ReadAllText(functionsFile); + profile.Functions = functionsFile; } return profile; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 8c78356c..9600bd09 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Records; +using MongoDB.Bson; namespace BotSharp.Core.Conversations.Services; @@ -55,7 +56,7 @@ public partial class ConversationService : IConversationService var db = _services.GetRequiredService(); var record = ConversationRecord.FromConversation(sess); - record.Id = sess.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + record.Id = sess.Id.IfNullOrEmptyAs(ObjectId.GenerateNewId().ToString()); record.UserId = sess.UserId.IfNullOrEmptyAs(_user.Id); record.Title = "New Conversation"; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 5f16c3c7..18b87d98 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -1,5 +1,7 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Repositories.Records; +using MongoDB.Bson; using System.IO; namespace BotSharp.Core.Conversations.Services; @@ -11,6 +13,9 @@ public class ConversationStateService : IConversationStateService, IDisposable { private readonly ILogger _logger; private readonly IServiceProvider _services; + private readonly AgentSettings _agentSettings; + private readonly IUserIdentity _user; + private readonly IBotSharpRepository _db; private ConversationState _state; private MyDatabaseSettings _dbSettings; private string _conversationId; @@ -18,11 +23,17 @@ public class ConversationStateService : IConversationStateService, IDisposable public ConversationStateService(ILogger logger, IServiceProvider services, - MyDatabaseSettings dbSettings) + MyDatabaseSettings dbSettings, + AgentSettings agentSettings, + IUserIdentity user, + IBotSharpRepository db) { _logger = logger; _services = services; _dbSettings = dbSettings; + _agentSettings = agentSettings; + _user = user; + _db = db; } public void SetState(string name, string value) @@ -55,11 +66,12 @@ public class ConversationStateService : IConversationStateService, IDisposable _state = new ConversationState(); - _file = GetStorageFile(_conversationId); + _file = GetConversationState(_conversationId); - if (File.Exists(_file)) + if (_file != null) { - var dict = File.ReadAllLines(_file); + //var dict = File.ReadAllLines(_file); + var dict = _file.SplitByNewLine(); foreach (var line in dict) { _state[line.Split('=')[0]] = line.Split('=')[1]; @@ -78,19 +90,39 @@ public class ConversationStateService : IConversationStateService, IDisposable public void Save() { - var states = new List(); - + var states = new StringBuilder(); + var conversation = _db.Conversation.FirstOrDefault(x => x.Id == _conversationId); + foreach (var dic in _state) { - states.Add($"{dic.Key}={dic.Value}"); + //states.Add($"{dic.Key}={dic.Value}"); + states.AppendLine($"{dic.Key}={dic.Value}"); } - File.WriteAllLines(_file, states); + //File.WriteAllLines(_file, states); _logger.LogInformation($"Saved state {_conversationId}"); + + if (conversation != null) + { + conversation.State = states.ToString(); + _db.Transaction(delegate + { + _db.Add(conversation); + }); + } } public void CleanState() { - File.Delete(_file); + //File.Delete(_file); + var conversation = _db.Conversation.FirstOrDefault(x => x.Id == _conversationId); + if (conversation != null) + { + conversation.State = string.Empty; + _db.Transaction(delegate + { + _db.Add(conversation); + }); + } } private string GetStorageFile(string conversationId) @@ -103,6 +135,33 @@ public class ConversationStateService : IConversationStateService, IDisposable return Path.Combine(dir, "state.dict"); } + private string GetConversationState(string conversationId) + { + var conversation = _db.Conversation.FirstOrDefault(x => x.Id == conversationId); + if (conversation == null) + { + var user = _db.User.FirstOrDefault(x => x.ExternalId == _user.Id); + var record = new ConversationRecord() + { + Id = ObjectId.GenerateNewId().ToString(), + AgentId = _agentSettings.RouterId, + UserId = user?.Id ?? ObjectId.GenerateNewId().ToString(), + Title = "New Conversation", + Dialog = string.Empty, + State = string.Empty + }; + + _db.Transaction(delegate + { + _db.Add(record); + }); + + conversation = _db.Conversation.FirstOrDefault(x => x.Id == record.Id); + } + + return conversation.State ?? string.Empty; + } + public string GetState(string name) { if (!_state.ContainsKey(name)) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 1f37ca85..4edc7940 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,24 +1,37 @@ using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Repositories.Records; +using MongoDB.Bson; using System.IO; +using Tensorflow; namespace BotSharp.Core.Conversations.Services; public class ConversationStorage : IConversationStorage { private readonly MyDatabaseSettings _dbSettings; + private readonly AgentSettings _agentSettings; private readonly IServiceProvider _services; - public ConversationStorage(MyDatabaseSettings dbSettings, IServiceProvider services) + private readonly IUserIdentity _user; + public ConversationStorage( + MyDatabaseSettings dbSettings, + AgentSettings agentSettings, + IServiceProvider services, + IUserIdentity user) { _dbSettings = dbSettings; + _agentSettings = agentSettings; _services = services; + _user = user; } public void Append(string conversationId, string agentId, RoleDialogModel dialog) { - var conversationFile = GetStorageFile(conversationId); - var sb = new StringBuilder(); + var dialogs = GetConversationDialogs(conversationId); + var sb = new StringBuilder(dialogs); + var db = _services.GetRequiredService(); if (dialog.Role == AgentRole.Function) { @@ -35,7 +48,6 @@ public class ConversationStorage : IConversationStorage } else { - var db = _services.GetRequiredService(); var agent = db.Agent.First(x => x.Id == agentId); sb.AppendLine($"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{agent.Name}|"); @@ -47,14 +59,22 @@ public class ConversationStorage : IConversationStorage sb.AppendLine($" - {content}"); } - var conversation = sb.ToString(); - File.AppendAllText(conversationFile, conversation); + var updatedDialogs = sb.ToString(); + //File.AppendAllText(conversationFile, conversation); + + var conversation = db.Conversation.FirstOrDefault(x => x.Id == conversationId); + conversation.AgentId = agentId; + conversation.Dialog = updatedDialogs; + db.Transaction(delegate + { + db.Add(conversation); + }); } public List GetDialogs(string conversationId) { - var conversationFile = GetStorageFile(conversationId); - var dialogs = File.ReadAllLines(conversationFile); + var conversationFile = GetConversationDialogs(conversationId); + var dialogs = conversationFile.SplitByNewLine(); var results = new List(); for (int i = 0; i < dialogs.Length; i += 2) @@ -81,11 +101,13 @@ public class ConversationStorage : IConversationStorage public void InitStorage(string conversationId) { - var file = GetStorageFile(conversationId); - if (!File.Exists(file)) - { - File.WriteAllLines(file, new string[0]); - } + //var file = GetStorageFile(conversationId); + //if (!File.Exists(file)) + //{ + // File.WriteAllLines(file, new string[0]); + //} + + GetConversationDialogs(conversationId); } private string GetStorageFile(string conversationId) @@ -97,4 +119,30 @@ public class ConversationStorage : IConversationStorage } return Path.Combine(dir, "dialogs.txt"); } + + private string GetConversationDialogs(string conversationId) + { + var db = _services.GetRequiredService(); + var conversation = db.Conversation.FirstOrDefault(x => x.Id == conversationId); + if (conversation == null) + { + var user = db.User.FirstOrDefault(x => x.ExternalId == _user.Id); + var record = new ConversationRecord() + { + Id = ObjectId.GenerateNewId().ToString(), + AgentId = _agentSettings.RouterId, + UserId = user?.Id ?? ObjectId.GenerateNewId().ToString(), + Title = "New Conversation" + }; + + db.Transaction(delegate + { + db.Add(record); + }); + + conversation = db.Conversation.FirstOrDefault(x => x.Id == record.Id); + } + + return conversation.Dialog ?? string.Empty; + } } diff --git a/src/Plugins/BotSharp.Plugin.MongoRepository/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoRepository/Repository/MongoRepository.cs index 4fe5ce96..2ce1433f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoRepository/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoRepository/Repository/MongoRepository.cs @@ -115,6 +115,8 @@ public class MongoRepository : IBotSharpRepository AgentId = x.AgentId, UserId = x.UserId, Title = x.Title, + Dialog = x.Dialog, + State = x.State, CreatedTime = x.CreatedTime, UpdatedTime = x.UpdatedTime }).ToList(); @@ -162,6 +164,9 @@ public class MongoRepository : IBotSharpRepository Id = x.Id.IfNullOrEmptyAs(ObjectId.GenerateNewId().ToString()), AgentId = x.AgentId, UserId = x.UserId, + Title = x.Title, + Dialog = x.Dialog, + State = x.State, CreatedTime = x.CreatedTime, UpdatedTime = x.UpdatedTime }).ToList(); @@ -172,6 +177,9 @@ public class MongoRepository : IBotSharpRepository var update = Builders.Update .Set(x => x.AgentId, conversation.AgentId) .Set(x => x.UserId, conversation.UserId) + .Set(x => x.Title, conversation.Title) + .Set(x => x.Dialog, conversation.Dialog) + .Set(x => x.State, conversation.State) .Set(x => x.CreatedTime, conversation.CreatedTime) .Set(x => x.UpdatedTime, conversation.UpdatedTime); _dc.Conversations.UpdateOne(filter, update, _options);