diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 18f8580b..446b8869 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -30,7 +30,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index baaeff2e..4a137070 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -6,6 +6,7 @@ public interface IBotSharpFileService Task> GetChatImages(string conversationId, string source, IEnumerable fileTypes, List conversations, int? offset = null); IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); + IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); bool SaveMessageFiles(string conversationId, string messageId, string source, List files); string GetUserAvatar(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs index c31c87cb..f752633b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs @@ -28,4 +28,9 @@ public class KeyValue { public string Key { get; set; } public string? Value { get; set; } + + public override string ToString() + { + return $"Key: {Key}, Value: {Value}"; + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs index a3908b2b..d8a2d7c6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs @@ -1,3 +1,4 @@ +using AspectInjector.Broker; using BotSharp.Abstraction.Files.Converters; using Microsoft.EntityFrameworkCore; using System.IO; @@ -126,7 +127,7 @@ public partial class BotSharpFileService string source, bool imageOnly = false) { var files = new List(); - if (messageIds.IsNullOrEmpty()) return files; + if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; foreach (var messageId in messageIds) { @@ -159,7 +160,8 @@ public partial class BotSharpFileService FileStorageUrl = file, FileName = fileName, FileType = fileType, - ContentType = contentType + ContentType = contentType, + FileSource = source }; files.Add(model); } @@ -181,6 +183,30 @@ public partial class BotSharpFileService return found; } + public IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds) + { + var foundMsgs = new List(); + if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return foundMsgs; + + foreach (var messageId in messageIds) + { + var prefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId); + var userDir = Path.Combine(prefix, FileSourceType.User); + if (ExistDirectory(userDir)) + { + foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User }); + } + + var botDir = Path.Combine(prefix, FileSourceType.Bot); + if (ExistDirectory(botDir)) + { + foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot }); + } + } + + return foundMsgs; + } + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index cac85ac5..f2bbc2eb 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -31,7 +31,7 @@ public class AgentController : ControllerBase var agents = await GetAgents(new AgentFilter { AgentIds = new List { id } - }); + }, useHook: true); var targetAgent = agents.Items.FirstOrDefault(); if (targetAgent == null) return null; @@ -63,26 +63,35 @@ public class AgentController : ControllerBase targetAgent.Editable = editable; return targetAgent; } - + [HttpGet("/agents")] - public async Task> GetAgents([FromQuery] AgentFilter filter) + public async Task> GetAgents([FromQuery] AgentFilter filter, [FromQuery] bool useHook = false) { var agentSetting = _services.GetRequiredService(); var pagedAgents = await _agentService.GetAgents(filter); - // prerender agent var items = new List(); - foreach (var agent in pagedAgents.Items) + var agents = new List(); + if (useHook) { - var renderedAgent = await _agentService.LoadAgent(agent.Id); - items.Add(renderedAgent); - } + // prerender agent + foreach (var agent in pagedAgents.Items) + { + var renderedAgent = await _agentService.LoadAgent(agent.Id); + items.Add(renderedAgent); + } - // Set IsHost - var agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); - foreach(var agent in agents) + // Set IsHost + agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); + foreach (var agent in agents) + { + agent.IsHost = agentSetting.HostAgentId == agent.Id; + } + } + else { - agent.IsHost = agentSetting.HostAgentId == agent.Id; + items = pagedAgents.Items.ToList(); + agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); } return new PagedItems diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index b3e29e31..c4850b1f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -80,6 +80,10 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); + + var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); + var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); var dialogs = new List(); foreach (var message in history) @@ -96,7 +100,8 @@ public class ConversationController : ControllerBase Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, Data = message.Data, Sender = UserViewModel.FromUser(user), - Payload = message.Payload + Payload = message.Payload, + HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.User) }); } else if (message.Role == AgentRole.Assistant) @@ -115,11 +120,11 @@ public class ConversationController : ControllerBase FirstName = agent?.Name ?? "Unkown", Role = message.Role, }, - RichContent = message.SecondaryRichContent ?? message.RichContent + RichContent = message.SecondaryRichContent ?? message.RichContent, + HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.Bot) }); } } - return dialogs; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs index a03f1211..d2d7041f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs @@ -32,6 +32,9 @@ public class ChatResponseModel : InstructResult [JsonPropertyName("payload")] public string? Payload { get; set; } + [JsonPropertyName("has_message_files")] + public bool HasMessageFiles { get; set; } + [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs index a9eb33bd..131a9baf 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -16,6 +16,9 @@ public class MessageFileViewModel [JsonPropertyName("content_type")] public string ContentType { get; set; } + [JsonPropertyName("file_source")] + public string FileSource { get; set; } + public MessageFileViewModel() { @@ -28,7 +31,8 @@ public class MessageFileViewModel FileUrl = model.FileUrl, FileName = model.FileName, FileType = model.FileType, - ContentType = model.ContentType + ContentType = model.ContentType, + FileSource = model.FileSource }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 075efc2b..4f1c5194 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.MongoStorage.Models; - namespace BotSharp.Plugin.MongoStorage.Collections; public class AgentDocument : MongoBase diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs index 21444c04..b71ae40c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.MongoStorage.Models; - namespace BotSharp.Plugin.MongoStorage.Collections; public class ConversationDialogDocument : MongoBase diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs index 1f8f0e90..c6068a1b 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.MongoStorage.Models; - namespace BotSharp.Plugin.MongoStorage.Collections; public class ConversationStateDocument : MongoBase diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs index f04cb46c..f5b96c0a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.MongoStorage.Models; - namespace BotSharp.Plugin.MongoStorage.Collections; public class LlmCompletionLogDocument : MongoBase diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index dfbbfbaa..50647776 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -40,6 +40,19 @@ public class MongoDbContext return collection; } + private IMongoCollection CreateConversationStateIndex() + { + var collection = Database.GetCollection($"{_collectionPrefix}_ConversationStates"); + var indexes = collection.Indexes.List().ToList(); + var stateIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("States.Key")); + if (stateIndex == null) + { + var indexDef = Builders.IndexKeys.Ascending("States.Key"); + collection.Indexes.CreateOne(new CreateIndexModel(indexDef)); + } + return collection; + } + private IMongoCollection CreateAgentTaskIndex() { var collection = Database.GetCollection($"{_collectionPrefix}_AgentTasks"); @@ -93,7 +106,7 @@ public class MongoDbContext => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); public IMongoCollection ConversationStates - => Database.GetCollection($"{_collectionPrefix}_ConversationStates"); + => CreateConversationStateIndex(); public IMongoCollection ExectionLogs => Database.GetCollection($"{_collectionPrefix}_ExecutionLogs"); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index bed3a255..88163b60 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,9 @@ +using Amazon.Util.Internal; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using System.Collections.Immutable; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -225,91 +229,77 @@ public partial class MongoRepository public PagedItems GetConversations(ConversationFilter filter) { - var conversations = new List(); - var builder = Builders.Filter; - var filters = new List>() { builder.Empty }; + var convBuilder = Builders.Filter; + var convFilters = new List>() { convBuilder.Empty }; + // Filter conversations if (!string.IsNullOrEmpty(filter?.Id)) { - filters.Add(builder.Eq(x => x.Id, filter.Id)); + convFilters.Add(convBuilder.Eq(x => x.Id, filter.Id)); } if (!string.IsNullOrEmpty(filter?.AgentId)) { - filters.Add(builder.Eq(x => x.AgentId, filter.AgentId)); + convFilters.Add(convBuilder.Eq(x => x.AgentId, filter.AgentId)); } if (!string.IsNullOrEmpty(filter?.Status)) { - filters.Add(builder.Eq(x => x.Status, filter.Status)); + convFilters.Add(convBuilder.Eq(x => x.Status, filter.Status)); } if (!string.IsNullOrEmpty(filter?.Channel)) { - filters.Add(builder.Eq(x => x.Channel, filter.Channel)); + convFilters.Add(convBuilder.Eq(x => x.Channel, filter.Channel)); } if (!string.IsNullOrEmpty(filter?.UserId)) { - filters.Add(builder.Eq(x => x.UserId, filter.UserId)); + convFilters.Add(convBuilder.Eq(x => x.UserId, filter.UserId)); } if (!string.IsNullOrEmpty(filter?.TaskId)) { - filters.Add(builder.Eq(x => x.TaskId, filter.TaskId)); + convFilters.Add(convBuilder.Eq(x => x.TaskId, filter.TaskId)); } if (filter?.StartTime != null) { - filters.Add(builder.Gte(x => x.CreatedTime, filter.StartTime.Value)); + convFilters.Add(convBuilder.Gte(x => x.CreatedTime, filter.StartTime.Value)); } - // Check states - if (filter != null && !filter.States.IsNullOrEmpty()) + // Filter states + var stateFilters = new List>(); + if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty()) { - var targetConvIds = new List(); - foreach (var pair in filter.States) { - if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - - var query = _dc.ConversationStates.AsQueryable(); - var convIds = query.AsEnumerable().Where(x => + var elementFilters = new List> { Builders.Filter.Eq(x => x.Key, pair.Key) }; + if (!string.IsNullOrEmpty(pair.Value)) { - var foundState = x.States.FirstOrDefault(s => s.Key.IsEqualTo(pair.Key)); - if (foundState == null) return false; - - if (!string.IsNullOrWhiteSpace(pair.Value)) - { - return pair.Value.IsEqualTo(foundState.Values.LastOrDefault()?.Data); - } - - return true; - }).Select(x => x.ConversationId).ToList(); - - targetConvIds = targetConvIds.Concat(convIds).Distinct().ToList(); + elementFilters.Add(Builders.Filter.Eq("Values.Data", pair.Value)); + } + stateFilters.Add(Builders.Filter.ElemMatch(x => x.States, Builders.Filter.And(elementFilters))); } - filters.Add(builder.In(x => x.Id, targetConvIds)); + var targetConvIds = _dc.ConversationStates.Find(Builders.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList(); + convFilters.Add(convBuilder.In(x => x.Id, targetConvIds)); } - var filterDef = builder.And(filters); + // Sort and paginate + var filterDef = convBuilder.And(convFilters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); var pager = filter?.Pager ?? new Pagination(); var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList(); var count = _dc.Conversations.CountDocuments(filterDef); - foreach (var conv in conversationDocs) + var conversations = conversationDocs.Select(x => new Conversation { - var convId = conv.Id.ToString(); - conversations.Add(new Conversation - { - Id = convId, - AgentId = conv.AgentId.ToString(), - UserId = conv.UserId.ToString(), - TaskId = conv.TaskId, - Title = conv.Title, - Channel = conv.Channel, - Status = conv.Status, - DialogCount = conv.DialogCount, - CreatedTime = conv.CreatedTime, - UpdatedTime = conv.UpdatedTime - }); - } + Id = x.Id.ToString(), + AgentId = x.AgentId.ToString(), + UserId = x.UserId.ToString(), + TaskId = x.TaskId, + Title = x.Title, + Channel = x.Channel, + Status = x.Status, + DialogCount = x.DialogCount, + CreatedTime = x.CreatedTime, + UpdatedTime = x.UpdatedTime + }).ToList(); return new PagedItems {