From 37f506687d756c3499f87eb8da8b8a207fb701b4 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 21 Nov 2023 18:16:45 -0600 Subject: [PATCH] parse conv dialog --- .../Conversations/Models/Conversation.cs | 19 +++- .../Repositories/IBotSharpRepository.cs | 4 +- .../Services/AgentService.RefreshAgents.cs | 1 + .../Services/ConversationStorage.cs | 50 +++++----- .../Repository/BotSharpDbContext.cs | 4 +- .../Repository/FileRepository.cs | 95 +++++++++++++------ .../ConversationDialogCollection.cs | 4 +- .../Models/DialogMongoElement.cs | 32 +++++++ .../Repository/MongoRepository.cs | 30 +++--- 9 files changed, 164 insertions(+), 75 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index e119aa36..0eccb301 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -10,7 +10,7 @@ public class Conversation public string Title { get; set; } = string.Empty; [JsonIgnore] - public string Dialog { get; set; } = string.Empty; + public List Dialogs { get; set; } = new List(); [JsonIgnore] public ConversationState States { get; set; } = new ConversationState(); @@ -22,3 +22,20 @@ public class Conversation public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; public DateTime CreatedTime { get; set; } = DateTime.UtcNow; } + +public class DialogElement +{ + public string MetaData { get; set; } + public string Content { get; set; } + + public DialogElement() + { + + } + + public DialogElement(string meta, string content) + { + MetaData = meta; + Content = content; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 95b85e15..1bf8b0e9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -28,8 +28,8 @@ public interface IBotSharpRepository #region Conversation void CreateNewConversation(Conversation conversation); - string GetConversationDialog(string conversationId); - void UpdateConversationDialog(string conversationId, string dialogs); + List GetConversationDialogs(string conversationId); + void AppendConversationDialogs(string conversationId, List dialogs); List GetConversationStates(string conversationId); void UpdateConversationStates(string conversationId, List states); void UpdateConversationStatus(string conversationId, string status); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index 5deb6ebd..e9b054be 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories; using System.IO; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 5ebe95ba..aa7372da 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Repositories; +using System; using System.IO; namespace BotSharp.Core.Conversations.Services; @@ -19,49 +20,44 @@ public class ConversationStorage : IConversationStorage { var agentId = dialog.CurrentAgentId; var db = _services.GetRequiredService(); - var dialogText = db.GetConversationDialog(conversationId); - var sb = new StringBuilder(dialogText); + var dialogElements = new List(); if (dialog.Role == AgentRole.Function) { // var args = dialog.FunctionArgs.RemoveNewLine(); - - sb.AppendLine($"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.FunctionName}"); - - var content = dialog.Content; - content = content.RemoveNewLine(); - if (string.IsNullOrEmpty(content)) - { - return; - } - sb.AppendLine($" - {content}"); - } - else - { - sb.AppendLine($"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.SenderId}"); + var meta = $"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.FunctionName}"; var content = dialog.Content.RemoveNewLine(); if (string.IsNullOrEmpty(content)) { return; } - sb.AppendLine($" - {content}"); + dialogElements.Add(new DialogElement(meta, content)); + } + else + { + var meta = $"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.SenderId}"; + var content = dialog.Content.RemoveNewLine(); + if (string.IsNullOrEmpty(content)) + { + return; + } + + dialogElements.Add(new DialogElement(meta, content)); } - var updatedDialogs = sb.ToString(); - db.UpdateConversationDialog(conversationId, updatedDialogs); + db.AppendConversationDialogs(conversationId, dialogElements); } public List GetDialogs(string conversationId) { var db = _services.GetRequiredService(); - var dialogText = db.GetConversationDialog(conversationId); - var dialogs = dialogText.SplitByNewLine(); + var dialogs = db.GetConversationDialogs(conversationId); var results = new List(); - for (int i = 0; i < dialogs.Length; i += 2) + foreach (var dialog in dialogs) { - var meta = dialogs[i]; - var dialog = dialogs[i + 1]; + var meta = dialog.MetaData; + var content = dialog.Content; var blocks = meta.Split('|'); var createdAt = DateTime.Parse(blocks[0]); var role = blocks[1]; @@ -69,13 +65,11 @@ public class ConversationStorage : IConversationStorage var messageId = blocks[3]; var senderId = role == AgentRole.Function ? currentAgentId : blocks[4]; var function = role == AgentRole.Function ? blocks[4] : null; - var text = dialog.Substring(4); - - results.Add(new RoleDialogModel(role, text) + + results.Add(new RoleDialogModel(role, content) { CurrentAgentId = currentAgentId, MessageId = messageId, - Content = text, CreatedAt = createdAt, SenderId = senderId, FunctionName = function diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index eb2540d4..16db5a18 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -136,7 +136,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public string GetConversationDialog(string conversationId) + public List GetConversationDialogs(string conversationId) { throw new NotImplementedException(); } @@ -146,7 +146,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public void UpdateConversationDialog(string conversationId, string dialogs) + public void AppendConversationDialogs(string conversationId, List dialogs) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 8df8a1ee..2c596804 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -625,22 +625,20 @@ public class FileRepository : IBotSharpRepository } } - public string GetConversationDialog(string conversationId) + public List GetConversationDialogs(string conversationId) { + var dialogs = new List(); var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { var dialogDir = Path.Combine(convDir, "dialogs.txt"); - if (File.Exists(dialogDir)) - { - return File.ReadAllText(dialogDir); - } + dialogs = CollectDialogElements(dialogDir); } - return string.Empty; + return dialogs; } - public void UpdateConversationDialog(string conversationId, string dialogs) + public void AppendConversationDialogs(string conversationId, List dialogs) { var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) @@ -648,7 +646,8 @@ public class FileRepository : IBotSharpRepository var dialogDir = Path.Combine(convDir, "dialogs.txt"); if (File.Exists(dialogDir)) { - File.WriteAllText(dialogDir, dialogs); + var texts = ParseDialogElements(dialogs); + File.AppendAllLines(dialogDir, texts); } } @@ -662,15 +661,7 @@ public class FileRepository : IBotSharpRepository if (!string.IsNullOrEmpty(convDir)) { var stateDir = Path.Combine(convDir, "state.dict"); - if (File.Exists(stateDir)) - { - var dict = File.ReadAllLines(stateDir); - foreach (var line in dict) - { - var data = line.Split('='); - curStates.Add(new StateKeyValue(data[0], data[1])); - } - } + curStates = CollectConversationStates(stateDir); } return curStates; @@ -721,16 +712,16 @@ public class FileRepository : IBotSharpRepository var record = JsonSerializer.Deserialize(content, _options); var dialogFile = Path.Combine(convDir, "dialogs.txt"); - if (record != null && File.Exists(dialogFile)) + if (record != null) { - record.Dialog = File.ReadAllText(dialogFile); + record.Dialogs = CollectDialogElements(dialogFile); } var stateFile = Path.Combine(convDir, "state.dict"); - if (record != null && File.Exists(stateFile)) + if (record != null) { - var states = File.ReadLines(stateFile); - record.States = new ConversationState(states.Select(x => new StateKeyValue(x.Split('=')[0], x.Split('=')[1])).ToList()); + var states = CollectConversationStates(stateFile); + record.States = new ConversationState(states); } return record; @@ -772,14 +763,13 @@ public class FileRepository : IBotSharpRepository var json = File.ReadAllText(path); var record = JsonSerializer.Deserialize(json, _options); - if (record != null) - { - records.Add(record); - } + if (record == null) continue; + + records.Add(record); } return records.GroupBy(r => r.UserId) - .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) - .ToList(); + .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) + .ToList(); } public void AddExectionLogs(string conversationId, List logs) @@ -944,5 +934,54 @@ public class FileRepository : IBotSharpRepository return null; } + + private List CollectDialogElements(string dialogDir) + { + var dialogs = new List(); + + if (!File.Exists(dialogDir)) return dialogs; + + var rawDialogs = File.ReadAllLines(dialogDir); + if (!rawDialogs.IsNullOrEmpty()) + { + for (int i = 0; i < rawDialogs.Count(); i += 2) + { + var meta = rawDialogs[i]; + var content = rawDialogs[i + 1]; + var trimmed = content.Substring(4); + dialogs.Add(new DialogElement(meta, trimmed)); + } + } + return dialogs; + } + + private List ParseDialogElements(List dialogs) + { + var dialogTexts = new List(); + if (dialogs.IsNullOrEmpty()) return dialogTexts; + + foreach (var element in dialogs) + { + dialogTexts.Add(element.MetaData); + var content = $" - {element.Content}"; + dialogTexts.Add(content); + } + + return dialogTexts; + } + + private List CollectConversationStates(string stateDir) + { + var states = new List(); + if (!File.Exists(stateDir)) return states; + + var dict = File.ReadAllLines(stateDir); + foreach (var line in dict) + { + var data = line.Split('='); + states.Add(new StateKeyValue(data[0], data[1])); + } + return states; + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogCollection.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogCollection.cs index 3c605e0e..e527453d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogCollection.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogCollection.cs @@ -1,7 +1,9 @@ +using BotSharp.Plugin.MongoStorage.Models; + namespace BotSharp.Plugin.MongoStorage.Collections; public class ConversationDialogCollection : MongoBase { public string ConversationId { get; set; } - public string Dialog { get; set; } + public List Dialogs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs new file mode 100644 index 00000000..86105728 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -0,0 +1,32 @@ +using BotSharp.Abstraction.Conversations.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class DialogMongoElement +{ + public string MetaData { get; set; } + public string Content { get; set; } + + public DialogMongoElement() + { + + } + + public static DialogMongoElement ToMongoElement(DialogElement dialog) + { + return new DialogMongoElement + { + MetaData = dialog.MetaData, + Content = dialog.Content + }; + } + + public static DialogElement ToDomainElement(DialogMongoElement dialog) + { + return new DialogElement + { + MetaData = dialog.MetaData, + Content = dialog.Content + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index abdf3479..d6ad0e7f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -620,25 +620,27 @@ public class MongoRepository : IBotSharpRepository { Id = Guid.NewGuid().ToString(), ConversationId = conv.Id, - Dialog = string.Empty + Dialogs = new List() }; _dc.Conversations.InsertOne(conv); _dc.ConversationDialogs.InsertOne(dialog); } - public string GetConversationDialog(string conversationId) + public List GetConversationDialogs(string conversationId) { - if (string.IsNullOrEmpty(conversationId)) return string.Empty; + var dialogs = new List(); + if (string.IsNullOrEmpty(conversationId)) return dialogs; var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var foundDialog = _dc.ConversationDialogs.Find(filter).FirstOrDefault(); - if (foundDialog == null) return string.Empty; + if (foundDialog == null) return dialogs; - return foundDialog.Dialog; + var formattedDialog = foundDialog.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList(); + return formattedDialog ?? new List(); } - public void UpdateConversationDialog(string conversationId, string dialogs) + public void AppendConversationDialogs(string conversationId, List dialogs) { if (string.IsNullOrEmpty(conversationId)) return; @@ -650,7 +652,8 @@ public class MongoRepository : IBotSharpRepository var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); if (foundDialog == null) return; - var updateDialog = Builders.Update.Set(x => x.Dialog, dialogs); + var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); + var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationDialogs.UpdateOne(filterDialog, updateDialog); @@ -710,6 +713,8 @@ public class MongoRepository : IBotSharpRepository if (conv == null) return null; + var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); + return new Conversation { Id = conv.Id.ToString(), @@ -717,7 +722,7 @@ public class MongoRepository : IBotSharpRepository UserId = conv.UserId.ToString(), Title = conv.Title, Status = conv.Status, - Dialog = dialog?.Dialog ?? string.Empty, + Dialogs = dialogElements, States = new ConversationState(conv.States ?? new List()), CreatedTime = conv.CreatedTime, UpdatedTime = conv.UpdatedTime @@ -754,9 +759,8 @@ public class MongoRepository : IBotSharpRepository { var records = new List(); var conversations = _dc.Conversations.Aggregate() - .Group(c => c.UserId, - g => g.OrderByDescending(x => x.CreatedTime).First()) - .ToList(); + .Group(c => c.UserId, g => g.OrderByDescending(x => x.CreatedTime).First()) + .ToList(); return conversations.Select(c => new Conversation() { Id = c.Id.ToString(), @@ -775,8 +779,8 @@ public class MongoRepository : IBotSharpRepository var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var update = Builders.Update - .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .PushEach(x => x.Logs, logs); + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) + .PushEach(x => x.Logs, logs); _dc.ExectionLogs.UpdateOne(filter, update, _options); }