From 94f81cb173338ea9984834a59072a409128af1f5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 19 Dec 2023 22:20:08 -0600 Subject: [PATCH 1/2] refine llm prompt log --- .../Utilities/StringExtensions.cs | 4 +++- .../BotSharp.Core/Repository/FileRepository.cs | 12 ++++++++++-- .../Repository/MongoRepository.cs | 10 ++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index a7ffe5af..625a1f4b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -37,8 +37,10 @@ public static class StringExtensions return input.Replace("\r", " ").Replace("\n", " ").Trim(); } - public static bool IsEqualTo(this string str1, string str2, StringComparison option = StringComparison.OrdinalIgnoreCase) + public static bool IsEqualTo(this string? str1, string? str2, StringComparison option = StringComparison.OrdinalIgnoreCase) { + if (str1 == null) return str2 == null; + return str1.Equals(str2, option); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 81126616..4c5833f2 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -6,6 +6,8 @@ using BotSharp.Abstraction.Agents.Models; using MongoDB.Driver; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Utilities; +using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.Core.Repository; @@ -860,10 +862,15 @@ public class FileRepository : IBotSharpRepository #region LLM Completion Log public void SaveLlmCompletionLog(LlmCompletionLog log) { - if (log == null || string.IsNullOrEmpty(log.ConversationId)) return; + if (log == null) return; + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.Empty.ToString()); var convDir = FindConversationDirectory(log.ConversationId); - if (string.IsNullOrEmpty(convDir)) return; + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } var logDir = Path.Combine(convDir, "llm_prompt_log"); if (!Directory.Exists(logDir)) @@ -872,6 +879,7 @@ public class FileRepository : IBotSharpRepository } log.Id = Guid.NewGuid().ToString(); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 89c583c5..a523e8c6 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -4,8 +4,10 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Users.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; +using System.Reflection.Emit; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -892,13 +894,13 @@ public class MongoRepository : IBotSharpRepository #region LLM Completion Log public void SaveLlmCompletionLog(LlmCompletionLog log) { - if (log == null || string.IsNullOrEmpty(log.ConversationId)) return; + if (log == null) return; var completiongLog = new LlmCompletionLogDocument { - Id = string.IsNullOrEmpty(log.Id) ? Guid.NewGuid().ToString() : log.Id, - ConversationId = log.ConversationId, - MessageId = log.MessageId, + Id = log.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString()), + ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.Empty.ToString()), + MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()), AgentId = log.AgentId, Prompt = log.Prompt, Response = log.Response, From d79fbaabd278483f5bee6d57140a9307d3753da9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 20 Dec 2023 14:12:40 -0600 Subject: [PATCH 2/2] refine completion log --- .../Conversations/Models/LlmCompletionLog.cs | 1 - .../Repositories/IBotSharpRepository.cs | 2 + .../Models/DialogContentUpdateModel.cs | 8 ++++ .../Repository/BotSharpDbContext.cs | 6 +++ .../Repository/FileRepository.cs | 34 +++++++++++++-- .../Collections/LlmCompletionLogDocument.cs | 8 ++-- .../Models/PromptLogElement.cs | 10 +++++ .../Repository/MongoRepository.cs | 43 +++++++++++++++---- 8 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/Models/DialogContentUpdateModel.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs index 26f55494..cca39ff3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs @@ -2,7 +2,6 @@ namespace BotSharp.Abstraction.Conversations.Models; public class LlmCompletionLog { - public string Id { get; set; } = string.Empty; public string ConversationId { get; set; } = string.Empty; public string MessageId { get; set; } = string.Empty; public string AgentId { get; set; } = string.Empty; diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index c9ef8fee..1834dfd5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Abstraction.Repositories; @@ -30,6 +31,7 @@ public interface IBotSharpRepository void CreateNewConversation(Conversation conversation); bool DeleteConversation(string conversationId); List GetConversationDialogs(string conversationId); + void UpdateConversationDialogElements(string conversationId, List updateElements); void AppendConversationDialogs(string conversationId, List dialogs); List GetConversationStates(string conversationId); void UpdateConversationStates(string conversationId, List states); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/DialogContentUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/DialogContentUpdateModel.cs new file mode 100644 index 00000000..9267f110 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/DialogContentUpdateModel.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Repositories.Models +{ + public class DialogContentUpdateModel + { + public int Index { get; set; } + public string UpdateContent { get; set; } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index e2062c0a..ec8ac5f6 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Users.Models; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -146,6 +147,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } + public void UpdateConversationDialogElements(string conversationId, List updateElements) + { + throw new NotImplementedException(); + } + public List GetConversationStates(string conversationId) { throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 4c5833f2..2e516177 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -8,6 +8,7 @@ using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Repositories.Models; namespace BotSharp.Core.Repository; @@ -652,6 +653,33 @@ public class FileRepository : IBotSharpRepository return dialogs; } + public void UpdateConversationDialogElements(string conversationId, List updateElements) + { + var dialogElements = GetConversationDialogs(conversationId); + if (dialogElements.IsNullOrEmpty() || updateElements.IsNullOrEmpty()) return; + + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var dialogDir = Path.Combine(convDir, "dialogs.txt"); + if (File.Exists(dialogDir)) + { + var updated = dialogElements.Select((x, idx) => + { + var found = updateElements.FirstOrDefault(e => e.Index == idx); + if (found != null) + { + x.Content = found.UpdateContent; + } + return x; + }).ToList(); + + var texts = ParseDialogElements(updated); + File.WriteAllLines(dialogDir, texts); + } + } + } + public void AppendConversationDialogs(string conversationId, List dialogs) { var convDir = FindConversationDirectory(conversationId); @@ -864,7 +892,9 @@ public class FileRepository : IBotSharpRepository { if (log == null) return; - log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.Empty.ToString()); + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var convDir = FindConversationDirectory(log.ConversationId); if (string.IsNullOrEmpty(convDir)) { @@ -878,8 +908,6 @@ public class FileRepository : IBotSharpRepository Directory.CreateDirectory(logDir); } - log.Id = Guid.NewGuid().ToString(); - log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs index f55047e8..e90714cb 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs @@ -1,11 +1,9 @@ +using BotSharp.Plugin.MongoStorage.Models; + namespace BotSharp.Plugin.MongoStorage.Collections; public class LlmCompletionLogDocument : MongoBase { public string ConversationId { get; set; } - public string MessageId { get; set; } - public string AgentId { get; set; } - public string Prompt { get; set; } - public string? Response { get; set; } - public DateTime CreateDateTime { get; set; } + public List Logs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs new file mode 100644 index 00000000..91c5e4d1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.MongoStorage.Models; + +public class PromptLogElement +{ + public string MessageId { get; set; } + public string AgentId { get; set; } + public string Prompt { get; set; } + public string? Response { get; set; } + public DateTime CreateDateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index a523e8c6..1d111062 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -2,12 +2,11 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Users.Models; -using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; -using System.Reflection.Emit; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -637,7 +636,8 @@ public class MongoRepository : IBotSharpRepository var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0; + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 + || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; } public List GetConversationDialogs(string conversationId) @@ -653,6 +653,27 @@ public class MongoRepository : IBotSharpRepository return formattedDialog ?? new List(); } + public void UpdateConversationDialogElements(string conversationId, List updateElements) + { + if (string.IsNullOrEmpty(conversationId) || updateElements.IsNullOrEmpty()) return; + + var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); + if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) return; + + foundDialog.Dialogs = foundDialog.Dialogs.Select((x, idx) => + { + var found = updateElements.FirstOrDefault(e => e.Index == idx); + if (found != null) + { + x.Content = found.UpdateContent; + } + return x; + }).ToList(); + + _dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog); + } + public void AppendConversationDialogs(string conversationId, List dialogs) { if (string.IsNullOrEmpty(conversationId)) return; @@ -896,18 +917,24 @@ public class MongoRepository : IBotSharpRepository { if (log == null) return; - var completiongLog = new LlmCompletionLogDocument + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logElement = new PromptLogElement { - Id = log.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString()), - ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.Empty.ToString()), - MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()), + MessageId = messageId, AgentId = log.AgentId, Prompt = log.Prompt, Response = log.Response, CreateDateTime = log.CreateDateTime }; - _dc.LlmCompletionLogs.InsertOne(completiongLog); + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var update = Builders.Update + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) + .Push(x => x.Logs, logElement); + + _dc.LlmCompletionLogs.UpdateOne(filter, update, _options); } #endregion }