diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 0e1d4787..41a6333f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; namespace BotSharp.Abstraction.Conversations; @@ -14,6 +15,8 @@ public interface IConversationService Task> GetLastConversations(); Task DeleteConversation(string id); Task TruncateConversation(string conversationId, string messageId); + Task> GetConversationContentLogs(string conversationId); + Task> GetConversationStateLogs(string conversationId); /// /// Send message to LLM diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs deleted file mode 100644 index de80b485..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationStateLogModel.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BotSharp.Abstraction.Conversations.Models; - -public class ConversationStateLogModel -{ - [JsonPropertyName("conversation_id")] - public string ConvsersationId { get; set; } - [JsonPropertyName("states")] - public string States { get; set; } - [JsonPropertyName("created_at")] - public DateTime CreateTime { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index e37492f1..a1cf8123 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -9,4 +9,6 @@ public class ConversationSetting public int MaxRecursiveDepth { get; set; } = 3; public bool EnableLlmCompletionLog { get; set; } public bool EnableExecutionLog { get; set; } + public bool EnableContentLog { get; set; } + public bool EnableStateLog { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs similarity index 68% rename from src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs rename to src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs index e0986272..ee01feb8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs @@ -1,9 +1,11 @@ namespace BotSharp.Abstraction.Loggers.Models; -public class StreamingLogModel +public class ConversationContentLogModel { [JsonPropertyName("conversation_id")] public string ConversationId { get; set; } + [JsonPropertyName("message_id")] + public string MessageId { get; set; } [JsonPropertyName("name")] public string? Name { get; set; } [JsonPropertyName("role")] @@ -13,5 +15,5 @@ public class StreamingLogModel public string Content { get; set; } [JsonPropertyName("created_at")] - public DateTime CreateTime { get; set; } + public DateTime CreateTime { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs new file mode 100644 index 00000000..98624759 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Abstraction.Loggers.Models; + +public class ConversationStateLogModel +{ + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + [JsonPropertyName("states")] + public Dictionary States { get; set; } + [JsonPropertyName("created_at")] + public DateTime CreateTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs similarity index 87% rename from src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs rename to src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs index cca39ff3..02c5b312 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Abstraction.Conversations.Models; +namespace BotSharp.Abstraction.Loggers.Models; public class LlmCompletionLog { diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 77d37d30..af407d25 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; @@ -70,6 +71,16 @@ public interface IBotSharpRepository void SaveLlmCompletionLog(LlmCompletionLog log); #endregion + #region Conversation Content Log + void SaveConversationContentLog(ConversationContentLogModel log); + List GetConversationContentLogs(string conversationId); + #endregion + + #region Conversation State Log + void SaveConversationStateLog(ConversationStateLogModel log); + List GetConversationStateLogs(string conversationId); + #endregion + #region Statistics void IncrementConversationCount(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs new file mode 100644 index 00000000..0d13bd66 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs @@ -0,0 +1,22 @@ +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task> GetConversationContentLogs(string conversationId) + { + var db = _services.GetRequiredService(); + var logs = db.GetConversationContentLogs(conversationId); + return await Task.FromResult(logs); + } + + + public async Task> GetConversationStateLogs(string conversationId) + { + var db = _services.GetRequiredService(); + var logs = db.GetConversationStateLogs(conversationId); + return await Task.FromResult(logs); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 4223aedc..48f0c159 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Filters; @@ -255,6 +256,30 @@ public class BotSharpDbContext : Database, IBotSharpRepository } #endregion + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + throw new NotImplementedException(); + } + + public List GetConversationContentLogs(string conversationId) + { + throw new NotImplementedException(); + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + throw new NotImplementedException(); + } + + public List GetConversationStateLogs(string conversationId) + { + throw new NotImplementedException(); + } + #endregion + #region Stats public void IncrementConversationCount() { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 78e8f9b1..72b5d882 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Loggers.Models; +using Serilog; using System.IO; namespace BotSharp.Core.Repository @@ -54,14 +56,112 @@ namespace BotSharp.Core.Repository Directory.CreateDirectory(logDir); } - var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); + var index = GetNextLogIndex(logDir, log.MessageId); var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } #endregion + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "content_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + + public List GetConversationContentLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return logs; + + var logDir = Path.Combine(convDir, "content_log"); + if (!Directory.Exists(logDir)) return logs; + + foreach (var file in Directory.GetFiles(logDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + logs.Add(log); + } + return logs.OrderBy(x => x.CreateTime).ToList(); + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "state_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + + public List GetConversationStateLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return logs; + + var logDir = Path.Combine(convDir, "state_log"); + if (!Directory.Exists(logDir)) return logs; + + foreach (var file in Directory.GetFiles(logDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + logs.Add(log); + } + return logs.OrderBy(x => x.CreateTime).ToList(); + } + #endregion + #region Private methods - private int GetNextLlmCompletionLogIndex(string logDir, string id) + private int GetNextLogIndex(string logDir, string id) { var files = Directory.GetFiles(logDir); if (files.IsNullOrEmpty()) diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs index 7a9a0d99..68d83c87 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Loggers.Models; + public class CommonContentGeneratingHook : IContentGeneratingHook { private readonly IServiceProvider _services; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index c0e52caa..135a3625 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using Microsoft.AspNetCore.Hosting; using SharpCompress.Compressors.Xz; using System; @@ -34,4 +35,18 @@ public class LoggerController : ControllerBase return NotFound(); } } + + [HttpGet("/logger/conversation/{conversationId}/content-log")] + public async Task> GetConversationContentLogs([FromRoute] string conversationId) + { + var conversationService = _services.GetRequiredService(); + return await conversationService.GetConversationContentLogs(conversationId); + } + + [HttpGet("/logger/conversation/{conversationId}/state-log")] + public async Task> GetConversationStateLogs([FromRoute] string conversationId) + { + var conversationService = _services.GetRequiredService(); + return await conversationService.GetConversationStateLogs(conversationId); + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 67645f5e..7840517b 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.JsonConverters; using BotSharp.Abstraction.Messaging.Models.RichContent; +using BotSharp.Abstraction.Repositories; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -117,20 +119,28 @@ public class ChatHubConversationHook : ConversationHookBase } }, _serializerOptions); await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageReceivedFromAssistant", json); - await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates())); + await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates(), message)); await base.OnResponseGenerated(message); } - private string BuildConversationStates(string conversationId, Dictionary states) + private string BuildConversationStates(string conversationId, Dictionary states, RoleDialogModel message) { - var model = new ConversationStateLogModel + var log = new ConversationStateLogModel { - ConvsersationId = conversationId, - States = JsonSerializer.Serialize(states, _serializerOptions), + ConversationId = conversationId, + MessageId = message.MessageId, + States = states, CreateTime = DateTime.UtcNow }; - return JsonSerializer.Serialize(model, _serializerOptions); + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableStateLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationStateLog(log); + } + + return JsonSerializer.Serialize(log, _serializerOptions); } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index c4e74ec4..c0a90ced 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Repositories; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -37,7 +38,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook { var conversationId = _state.GetConversationId(); var log = $"MessageId: {message.MessageId} ==>\r\n{message.Role}: {message.Content}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, message.Role, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, _user.UserName, log, message)); } public async Task BeforeGenerating(Agent agent, List conversations) @@ -64,25 +65,34 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook var conversationId = _state.GetConversationId(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, message.Role, tokenStats.Prompt)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, tokenStats.Prompt, message)); var log = message.Role == AgentRole.Function ? $"[{agent?.Name}]: {message.FunctionName}({message.FunctionArgs})" : $"[{agent?.Name}]: {message.Content}"; log += $"\r\n<== MessageId: {message.MessageId}"; - await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, message.Role, log)); + await _chatHub.Clients.User(_user.Id).SendAsync("OnContentLogGenerated", BuildLog(conversationId, agent?.Name, log, message)); } - private string BuildLog(string conversationId, string? name, string role, string content) + private string BuildLog(string conversationId, string? name, string content, RoleDialogModel message) { - var log = new StreamingLogModel + var log = new ConversationContentLogModel { ConversationId = conversationId, + MessageId = message.MessageId, Name = name, - Role = role, + Role = message.Role, Content = content, CreateTime = DateTime.UtcNow }; + + var convSettings = _services.GetRequiredService(); + if (convSettings.EnableContentLog) + { + var db = _services.GetRequiredService(); + db.SaveConversationContentLog(log); + } + return JsonSerializer.Serialize(log, _serializerOptions); } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs new file mode 100644 index 00000000..eec715dc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationContentLogDocument.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationContentLogDocument : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public string? Name { get; set; } + public string Role { get; set; } + public string Content { get; set; } + public DateTime CreateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs new file mode 100644 index 00000000..c17c86c8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateLogDocument.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationStateLogDocument : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public Dictionary States { get; set; } + public DateTime CreateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 5fa08da7..ca691e05 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -14,7 +14,6 @@ public class MongoDbContext _mongoClient = new MongoClient(mongoDbConnectionString); _mongoDbDatabaseName = GetDatabaseName(mongoDbConnectionString); _collectionPrefix = dbSettings.TablePrefix.IfNullOrEmptyAs("BotSharp"); - //CreateIndex(); } private string GetDatabaseName(string mongoDbConnectionString) @@ -29,6 +28,7 @@ public class MongoDbContext private IMongoDatabase Database { get { return _mongoClient.GetDatabase(_mongoDbDatabaseName); } } + #region Indexes private IMongoCollection CreateConversationIndex() { var collection = Database.GetCollection($"{_collectionPrefix}_Conversations"); @@ -39,7 +39,6 @@ public class MongoDbContext var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); } - return collection; } @@ -53,28 +52,44 @@ public class MongoDbContext var indexDef = Builders.IndexKeys.Descending(x => x.CreatedTime); collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); } - return collection; } + private IMongoCollection CreateContentLogIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationContentLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending(x => x.CreateTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + + private IMongoCollection CreateStateLogIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationStateLogs"); + var indexes = collection.Indexes.List().ToList(); + var createTimeIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("CreateTime")); + if (createTimeIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending(x => x.CreateTime); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + #endregion + public IMongoCollection Agents => Database.GetCollection($"{_collectionPrefix}_Agents"); public IMongoCollection AgentTasks - { - get - { - return CreateAgentTaskIndex(); - } - } + => CreateAgentTaskIndex(); public IMongoCollection Conversations - { - get - { - return CreateConversationIndex(); - } - } + => CreateConversationIndex(); public IMongoCollection ConversationDialogs => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); @@ -85,15 +100,21 @@ public class MongoDbContext public IMongoCollection ExectionLogs => Database.GetCollection($"{_collectionPrefix}_ExecutionLogs"); + public IMongoCollection LlmCompletionLogs + => Database.GetCollection($"{_collectionPrefix}_LlmCompletionLogs"); + + public IMongoCollection ContentLogs + => CreateContentLogIndex(); + + public IMongoCollection StateLogs + => CreateStateLogIndex(); + public IMongoCollection Users => Database.GetCollection($"{_collectionPrefix}_Users"); public IMongoCollection UserAgents => Database.GetCollection($"{_collectionPrefix}_UserAgents"); - public IMongoCollection LlmCompletionLogs - => Database.GetCollection($"{_collectionPrefix}_Llm_Completion_Logs"); - public IMongoCollection Plugins => Database.GetCollection($"{_collectionPrefix}_Plugins"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 1b113537..aa0deda3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -63,14 +63,20 @@ public partial class MongoRepository var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterContentLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterStateLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); + var contentLogDeleted = _dc.ContentLogs.DeleteMany(filterContentLog); + var stateLogDeleted = _dc.StateLogs.DeleteMany(filterStateLog); var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); var convDeleted = _dc.Conversations.DeleteMany(filterConv); + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 - || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; + || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0 + || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0; } public List GetConversationDialogs(string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 20266450..efe7537c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; @@ -58,4 +58,82 @@ public partial class MongoRepository } #endregion + + #region Conversation Content Log + public void SaveConversationContentLog(ConversationContentLogModel log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logDoc = new ConversationContentLogDocument + { + ConversationId = conversationId, + MessageId = messageId, + Name = log.Name, + Role = log.Role, + Content = log.Content, + CreateTime = log.CreateTime + }; + + _dc.ContentLogs.InsertOne(logDoc); + } + + public List GetConversationContentLogs(string conversationId) + { + var logs = _dc.ContentLogs + .AsQueryable() + .Where(x => x.ConversationId == conversationId) + .Select(x => new ConversationContentLogModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + Name = x.Name, + Role = x.Role, + Content = x.Content, + CreateTime = x.CreateTime + }) + .OrderBy(x => x.CreateTime) + .ToList(); + return logs; + } + #endregion + + #region Conversation State Log + public void SaveConversationStateLog(ConversationStateLogModel log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logDoc = new ConversationStateLogDocument + { + ConversationId = conversationId, + MessageId = messageId, + States = log.States, + CreateTime = log.CreateTime + }; + + _dc.StateLogs.InsertOne(logDoc); + } + + public List GetConversationStateLogs(string conversationId) + { + var logs = _dc.StateLogs + .AsQueryable() + .Where(x => x.ConversationId == conversationId) + .Select(x => new ConversationStateLogModel + { + ConversationId = x.ConversationId, + MessageId = x.MessageId, + States = x.States, + CreateTime = x.CreateTime + }) + .OrderBy(x => x.CreateTime) + .ToList(); + return logs; + } + #endregion } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c5be322c..7694990a 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -87,7 +87,9 @@ "DataDir": "conversations", "ShowVerboseLog": false, "EnableLlmCompletionLog": false, - "EnableExecutionLog": true + "EnableExecutionLog": true, + "EnableContentLog": true, + "EnableStateLog": true }, "Statistics": {