From ed1576f7e02ed866330a7fc4d85bddfe9e9f429b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 11 Aug 2025 15:46:35 -0500 Subject: [PATCH] refine list loading --- .../Filters/ConversationStateKeysFilter.cs | 2 + .../Filters/InstructLogKeysFilter.cs | 2 + .../Repositories/IBotSharpRepository.cs | 12 +- .../Users/IUserService.cs | 1 + .../Utilities/Pagination.cs | 2 +- .../Services/CrontabService.cs | 2 +- .../Services/ConversationService.cs | 2 +- .../Services/LoggerService.Instruction.cs | 12 +- .../Repository/BotSharpDbContext.cs | 196 ------------------ .../FileRepository.AgentTask.cs | 2 +- .../FileRepository.Conversation.cs | 8 +- .../FileRepository/FileRepository.Crontab.cs | 4 +- .../FileRepository.KnowledgeBase.cs | 2 +- .../FileRepository/FileRepository.Log.cs | 6 +- .../FileRepository/FileRepository.User.cs | 2 +- .../Tasks/Services/AgentTaskService.cs | 2 +- .../Users/Services/UserService.cs | 10 +- .../Controllers/ConversationController.cs | 4 +- .../Services/KnowledgeService.Document.cs | 6 +- .../Repository/MongoRepository.AgentTask.cs | 22 +- .../MongoRepository.Conversation.cs | 28 ++- .../Repository/MongoRepository.Crontab.cs | 19 +- .../MongoRepository.KnowledgeBase.cs | 18 +- .../Repository/MongoRepository.Log.cs | 27 ++- .../Repository/MongoRepository.User.cs | 19 +- 25 files changed, 149 insertions(+), 261 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs index bff4b43c..201a031c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs @@ -8,6 +8,8 @@ public class ConversationStateKeysFilter public bool PreLoad { get; set; } public List? AgentIds { get; set; } public List? UserIds { get; set; } + public DateTime? StartTime { get; set; } + public DateTime? EndTime { get; set; } public ConversationStateKeysFilter() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs index c3eade89..ef04ede6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs @@ -8,6 +8,8 @@ public class InstructLogKeysFilter public bool PreLoad { get; set; } public List? AgentIds { get; set; } public List? UserIds { get; set; } + public DateTime? StartTime { get; set; } + public DateTime? EndTime { get; set; } public InstructLogKeysFilter() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index d608d71f..8e0e2d55 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -56,7 +56,7 @@ public interface IBotSharpRepository : IHaveServiceProvider void UpdateUserPhone(string userId, string Iphone, string regionCode) => throw new NotImplementedException(); void UpdateUserIsDisable(string userId, bool isDisable) => throw new NotImplementedException(); void UpdateUsersIsDisable(List userIds, bool isDisable) => throw new NotImplementedException(); - PagedItems GetUsers(UserFilter filter) => throw new NotImplementedException(); + ValueTask> GetUsers(UserFilter filter) => throw new NotImplementedException(); List SearchLoginUsers(User filter, string source = UserSource.Internal) =>throw new NotImplementedException(); User? GetUserDetails(string userId, bool includeAgent = false) => throw new NotImplementedException(); bool UpdateUser(User user, bool updateUserAgents = false) => throw new NotImplementedException(); @@ -93,7 +93,7 @@ public interface IBotSharpRepository : IHaveServiceProvider #endregion #region Agent Task - PagedItems GetAgentTasks(AgentTaskFilter filter) + ValueTask> GetAgentTasks(AgentTaskFilter filter) => throw new NotImplementedException(); AgentTask? GetAgentTask(string agentId, string taskId) => throw new NotImplementedException(); @@ -126,7 +126,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); Conversation GetConversation(string conversationId, bool isLoadStates = false) => throw new NotImplementedException(); - PagedItems GetConversations(ConversationFilter filter) + ValueTask> GetConversations(ConversationFilter filter) => throw new NotImplementedException(); void UpdateConversationTitle(string conversationId, string title) => throw new NotImplementedException(); @@ -179,7 +179,7 @@ public interface IBotSharpRepository : IHaveServiceProvider bool SaveInstructionLogs(IEnumerable logs) => throw new NotImplementedException(); - PagedItems GetInstructionLogs(InstructLogFilter filter) + ValueTask> GetInstructionLogs(InstructLogFilter filter) => throw new NotImplementedException(); List GetInstructionLogSearchKeys(InstructLogKeysFilter filter) @@ -227,7 +227,7 @@ public interface IBotSharpRepository : IHaveServiceProvider /// bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) => throw new NotImplementedException(); - PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + ValueTask> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) => throw new NotImplementedException(); #endregion @@ -236,7 +236,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); bool DeleteCrontabItem(string conversationId) => throw new NotImplementedException(); - PagedItems GetCrontabItems(CrontabItemFilter filter) + ValueTask> GetCrontabItems(CrontabItemFilter filter) => throw new NotImplementedException(); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 1499ea17..d03687ff 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -7,6 +7,7 @@ namespace BotSharp.Abstraction.Users; public interface IUserService { Task GetUser(string id); + Task> GetUsers(List ids); Task> GetUsers(UserFilter filter); Task> SearchLoginUsers(User filter); Task GetUserDetails(string userId, bool includeAgent = false); diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs index 4d21013b..c63b0056 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs @@ -50,6 +50,6 @@ public class Pagination : ICacheKey public class PagedItems { - public int Count { get; set; } + public long Count { get; set; } public IEnumerable Items { get; set; } = new List(); } diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 255faa71..b2101ee5 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -44,7 +44,7 @@ public class CrontabService : ICrontabService, ITaskFeeder public async Task> GetCrontable() { var repo = _services.GetRequiredService(); - var crontable = repo.GetCrontabItems(CrontabItemFilter.Empty()); + var crontable = await repo.GetCrontabItems(CrontabItemFilter.Empty()); // Add fixed crontab items from cronsources var fixedCrantabItems = crontable.Items.ToList(); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 676ea362..9e410b49 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -87,7 +87,7 @@ public partial class ConversationService : IConversationService } var db = _services.GetRequiredService(); - var conversations = db.GetConversations(filter); + var conversations = await db.GetConversations(filter); return conversations; } diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs index 42ad45ab..965212ca 100644 --- a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs @@ -18,22 +18,18 @@ public partial class LoggerService filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null; - var agents = new List(); var users = new List(); var db = _services.GetRequiredService(); - var logs = db.GetInstructionLogs(filter); + var agentService = _services.GetRequiredService(); + var logs = await db.GetInstructionLogs(filter); var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList(); - agents = db.GetAgents(new AgentFilter - { - AgentIds = agentIds, - Pager = new Pagination { Size = filter.Size } - }); + var agents = await agentService.GetAgentOptions(agentIds); if (isAdmin) { - users = db.GetUserByIds(userIds); + users = await userService.GetUsers(userIds); } var items = logs.Items.Select(x => diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 492dcde3..c2dfc25f 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,202 +1,6 @@ -using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Tasks.Models; -using BotSharp.Abstraction.Translation.Models; -using BotSharp.Abstraction.VectorStorage.Models; - namespace BotSharp.Core.Repository; public class BotSharpDbContext : Database, IBotSharpRepository { public IServiceProvider ServiceProvider => throw new NotImplementedException(); - - #region Plugin - public PluginConfig GetPluginConfig() => throw new NotImplementedException(); - public void SavePluginConfig(PluginConfig config) => throw new NotImplementedException(); - #endregion - - #region Agent - public Agent GetAgent(string agentId, bool basicsOnly = false) - => throw new NotImplementedException(); - - public List GetAgents(AgentFilter filter) - => throw new NotImplementedException(); - - public List GetUserAgents(string userId) - => throw new NotImplementedException(); - - public void UpdateAgent(Agent agent, AgentField field) - => throw new NotImplementedException(); - - public string GetAgentTemplate(string agentId, string templateName) - => throw new NotImplementedException(); - - public bool PatchAgentTemplate(string agentId, AgentTemplate template) - => throw new NotImplementedException(); - - public List GetAgentResponses(string agentId, string prefix, string intent) - => throw new NotImplementedException(); - - public void BulkInsertAgents(List agents) - => throw new NotImplementedException(); - - public void BulkInsertUserAgents(List userAgents) - => throw new NotImplementedException(); - - public bool DeleteAgents() - => throw new NotImplementedException(); - - public bool DeleteAgent(string agentId) - => throw new NotImplementedException(); - #endregion - - #region Agent Task - public PagedItems GetAgentTasks(AgentTaskFilter filter) - => throw new NotImplementedException(); - - public AgentTask? GetAgentTask(string agentId, string taskId) - => throw new NotImplementedException(); - - public void InsertAgentTask(AgentTask task) - => throw new NotImplementedException(); - - public void BulkInsertAgentTasks(List tasks) - => throw new NotImplementedException(); - - public void UpdateAgentTask(AgentTask task, AgentTaskField field) - => throw new NotImplementedException(); - - public bool DeleteAgentTask(string agentId, List taskIds) - => throw new NotImplementedException(); - - public bool DeleteAgentTasks() - => throw new NotImplementedException(); - #endregion - - #region Conversation - public void CreateNewConversation(Conversation conversation) - => throw new NotImplementedException(); - - public bool DeleteConversations(IEnumerable conversationIds) - => throw new NotImplementedException(); - - public Conversation GetConversation(string conversationId, bool isLoadStates = false) - => throw new NotImplementedException(); - - public PagedItems GetConversations(ConversationFilter filter) - => throw new NotImplementedException(); - - public List GetLastConversations() - => throw new NotImplementedException(); - - public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) - => throw new NotImplementedException(); - - [SideCar] - public List GetConversationDialogs(string conversationId) - => throw new NotImplementedException(); - - public ConversationState GetConversationStates(string conversationId) - => throw new NotImplementedException(); - - [SideCar] - public void AppendConversationDialogs(string conversationId, List dialogs) - => throw new NotImplementedException(); - - public void UpdateConversationTitle(string conversationId, string title) - => throw new NotImplementedException(); - public void UpdateConversationTitleAlias(string conversationId, string titleAlias) - => throw new NotImplementedException(); - - public bool UpdateConversationTags(string conversationId, List tags) - => throw new NotImplementedException(); - - public bool AppendConversationTags(string conversationId, List tags) - => throw new NotImplementedException(); - - public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) - => throw new NotImplementedException(); - - [SideCar] - public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) - => throw new NotImplementedException(); - - [SideCar] - public ConversationBreakpoint? GetConversationBreakpoint(string conversationId) - => throw new NotImplementedException(); - - public void UpdateConversationStates(string conversationId, List states) - => throw new NotImplementedException(); - - public void UpdateConversationStatus(string conversationId, string status) - => throw new NotImplementedException(); - - public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) - => throw new NotImplementedException(); - #endregion - - #region LLM Completion Log - public void SaveLlmCompletionLog(LlmCompletionLog log) - { - throw new NotImplementedException(); - } - #endregion - - #region Conversation Content Log - public void SaveConversationContentLog(ContentLogOutputModel log) - { - throw new NotImplementedException(); - } - - public List GetConversationContentLogs(string conversationId) - { - throw new NotImplementedException(); - } - #endregion - - #region Conversation State Log - public void SaveConversationStateLog(ConversationStateLogModel log) - { - throw new NotImplementedException(); - } - - public List GetConversationStateLogs(string conversationId) - { - throw new NotImplementedException(); - } - #endregion - - #region Stats - public void IncrementConversationCount() - { - throw new NotImplementedException(); - } - #endregion - - #region Translation - public IEnumerable GetTranslationMemories(IEnumerable queries) - => throw new NotImplementedException(); - public bool SaveTranslationMemories(IEnumerable inputs) => - throw new NotImplementedException(); - #endregion - - #region KnowledgeBase - public bool AddKnowledgeCollectionConfigs(List configs, bool reset = false) => - throw new NotImplementedException(); - - public bool DeleteKnowledgeCollectionConfig(string collectionName) => - throw new NotImplementedException(); - - public IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) => - throw new NotImplementedException(); - - public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) => - throw new NotImplementedException(); - - public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) => - throw new NotImplementedException(); - - public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) => - throw new NotImplementedException(); - #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs index 016e0c36..2b84f212 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Repository; public partial class FileRepository { #region Task - public PagedItems GetAgentTasks(AgentTaskFilter filter) + public async ValueTask> GetAgentTasks(AgentTaskFilter filter) { if (filter == null) { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index a42bc988..68881ea3 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -379,7 +379,7 @@ public partial class FileRepository return record; } - public PagedItems GetConversations(ConversationFilter filter) + public async ValueTask> GetConversations(ConversationFilter filter) { if (filter == null) { @@ -546,7 +546,7 @@ public partial class FileRepository return new PagedItems { Items = records.OrderByDescending(x => x.CreatedTime).Skip(pager.Offset).Take(pager.Size), - Count = records.Count(), + Count = records.Count() }; } @@ -708,7 +708,9 @@ public partial class FileRepository if (conv == null || states.IsNullOrEmpty() || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(conv.AgentId)) - || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId))) + || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId)) + || (filter.StartTime.HasValue && conv.CreatedTime < filter.StartTime.Value) + || (filter.EndTime.HasValue && conv.CreatedTime > filter.EndTime.Value)) { continue; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs index 4a99027a..84f2794c 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs @@ -63,7 +63,7 @@ public partial class FileRepository } - public PagedItems GetCrontabItems(CrontabItemFilter filter) + public async ValueTask> GetCrontabItems(CrontabItemFilter filter) { if (filter == null) @@ -111,7 +111,7 @@ public partial class FileRepository return new PagedItems { Items = records.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size), - Count = records.Count(), + Count = records.Count() }; } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index b7fa26ab..ae9d0405 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -155,7 +155,7 @@ public partial class FileRepository return true; } - public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + public async ValueTask> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider)) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index 5b8d095f..c0851401 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -160,7 +160,7 @@ namespace BotSharp.Core.Repository return true; } - public PagedItems GetInstructionLogs(InstructLogFilter filter) + public async ValueTask> GetInstructionLogs(InstructLogFilter filter) { if (filter == null) { @@ -316,7 +316,9 @@ namespace BotSharp.Core.Repository if (log == null || log.InnerStates.IsNullOrEmpty() || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(log.UserId)) - || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId))) + || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId)) + || (filter.StartTime.HasValue && log.CreatedTime < filter.StartTime.Value) + || (filter.EndTime.HasValue && log.CreatedTime > filter.EndTime.Value)) { continue; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs index 7bd5a1a9..09212e20 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs @@ -105,7 +105,7 @@ public partial class FileRepository File.WriteAllText(path, JsonSerializer.Serialize(user, _options)); } - public PagedItems GetUsers(UserFilter filter) + public async ValueTask> GetUsers(UserFilter filter) { if (filter == null) { diff --git a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs index ce5d72b8..5ae52a73 100644 --- a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs +++ b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs @@ -39,7 +39,7 @@ public class AgentTaskService : IAgentTaskService else { var db = _services.GetRequiredService(); - var pagedTasks = db.GetAgentTasks(filter); + var pagedTasks = await db.GetAgentTasks(filter); return await Task.FromResult(pagedTasks); } } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index e4ebe311..f7dd61c6 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -411,10 +411,18 @@ public class UserService : IUserService return user; } + [SharpCache(10)] + public async Task> GetUsers(List ids) + { + var db = _services.GetRequiredService(); + var users = db.GetUserByIds(ids); + return users; + } + public async Task> GetUsers(UserFilter filter) { var db = _services.GetRequiredService(); - var users = db.GetUsers(filter); + var users = await db.GetUsers(filter); return users; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index ec7dead5..45f53c0c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -66,11 +66,11 @@ public class ConversationController : ControllerBase var agents = await agentService.GetAgentOptions(agentIds); var userIds = list.Select(x => x.User.Id).ToList(); - var users = await userService.GetUsers(new UserFilter { UserIds = userIds, Size = filter.Pager.Size }); + var users = await userService.GetUsers(userIds); foreach (var item in list) { - user = users.Items.FirstOrDefault(x => x.Id == item.User.Id); + user = users.FirstOrDefault(x => x.Id == item.User.Id); item.User = UserViewModel.FromUser(user); var agent = agents.FirstOrDefault(x => x.Id == item.AgentId); item.AgentName = agent?.Name ?? "Unkown"; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index d55d5a4a..3f4853e2 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -191,7 +191,7 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { Size = 1, FileIds = [ fileId ] @@ -281,7 +281,7 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter); + var pagedData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter); var files = pagedData.Items?.Select(x => new KnowledgeFileModel { @@ -308,7 +308,7 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc binary data - var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter + var pageData = await db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { Size = 1, FileIds = [ fileId ] diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs index db3cdfc6..85f14215 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.MongoStorage.Repository; public partial class MongoRepository { #region Task - public PagedItems GetAgentTasks(AgentTaskFilter filter) + public async ValueTask> GetAgentTasks(AgentTaskFilter filter) { if (filter == null) { @@ -34,13 +34,23 @@ public partial class MongoRepository var filterDef = builder.And(filters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); - var totalTasks = _dc.AgentTasks.CountDocuments(filterDef); - var taskDocs = _dc.AgentTasks.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList(); - var agentIds = taskDocs.Select(x => x.AgentId).Distinct().ToList(); + var docsTask = _dc.AgentTasks.FindAsync(filterDef, options: new() + { + Sort = sortDef, + Skip = pager.Offset, + Limit = pager.Size + }); + var countTask = _dc.AgentTasks.CountDocumentsAsync(filterDef); + await Task.WhenAll([docsTask, countTask]); + + var docs = docsTask.Result.ToList(); + var count = countTask.Result; + + var agentIds = docs.Select(x => x.AgentId).Distinct().ToList(); var agents = GetAgents(new AgentFilter { AgentIds = agentIds }); - var tasks = taskDocs.Select(x => + var tasks = docs.Select(x => { var task = AgentTaskDocument.ToDomainModel(x); task.Agent = agents.FirstOrDefault(a => a.Id == x.AgentId); @@ -50,7 +60,7 @@ public partial class MongoRepository return new PagedItems { Items = tasks, - Count = (int)totalTasks + Count = count }; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index fb424f02..6611c352 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -336,7 +336,7 @@ public partial class MongoRepository }; } - public PagedItems GetConversations(ConversationFilter filter) + public async ValueTask> GetConversations(ConversationFilter filter) { if (filter == null) { @@ -466,10 +466,19 @@ public partial class MongoRepository } } - var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList(); - var count = _dc.Conversations.CountDocuments(filterDef); + var docsTask = _dc.Conversations.FindAsync(filterDef, options: new() + { + Sort = sortDef, + Skip = pager.Offset, + Limit = pager.Size + }); + var countTask = _dc.Conversations.CountDocumentsAsync(filterDef); + await Task.WhenAll([docsTask, countTask]); - var conversations = conversationDocs.Select(x => + var docs = docsTask.Result.ToList(); + var count = countTask.Result; + + var conversations = docs.Select(x => { var states = new Dictionary(); if (filter.IsLoadLatestStates) @@ -502,7 +511,7 @@ public partial class MongoRepository return new PagedItems { Items = conversations, - Count = (int)count + Count = count }; } @@ -693,11 +702,18 @@ public partial class MongoRepository { filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); } - if (!filter.UserIds.IsNullOrEmpty()) { filters.Add(builder.In(x => x.UserId, filter.UserIds)); } + if (filter.StartTime.HasValue) + { + filters.Add(builder.Gte(x => x.CreatedTime, filter.StartTime.Value)); + } + if (filter.EndTime.HasValue) + { + filters.Add(builder.Lte(x => x.CreatedTime, filter.EndTime.Value)); + } var convDocs = _dc.Conversations.Find(builder.And(filters)) .Sort(sortDef) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs index af10c373..4c307222 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs @@ -45,7 +45,7 @@ public partial class MongoRepository } - public PagedItems GetCrontabItems(CrontabItemFilter filter) + public async ValueTask> GetCrontabItems(CrontabItemFilter filter) { if (filter == null) { @@ -73,15 +73,24 @@ public partial class MongoRepository var filterDef = cronBuilder.And(cronFilters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); - var cronDocs = _dc.CrontabItems.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); - var count = _dc.CrontabItems.CountDocuments(filterDef); + var docsTask = _dc.CrontabItems.FindAsync(filterDef, options: new() + { + Sort = sortDef, + Skip = filter.Offset, + Limit = filter.Size + }); + var countTask = _dc.CrontabItems.CountDocumentsAsync(filterDef); + await Task.WhenAll([docsTask, countTask]); - var crontabItems = cronDocs.Select(x => CrontabItemDocument.ToDomainModel(x)).ToList(); + var docs = docsTask.Result.ToList(); + var count = countTask.Result; + + var crontabItems = docs.Select(x => CrontabItemDocument.ToDomainModel(x)).ToList(); return new PagedItems { Items = crontabItems, - Count = (int)count + Count = count }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index e47ed1c3..f6bcba89 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -168,7 +168,7 @@ public partial class MongoRepository return res.DeletedCount > 0; } - public PagedItems GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) + public async ValueTask> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(vectorStoreProvider)) @@ -209,8 +209,18 @@ public partial class MongoRepository var filterDef = builder.And(docFilters); var sortDef = Builders.Sort.Descending(x => x.CreatedDate); - var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); - var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef); + + var docsTask = _dc.KnowledgeCollectionFileMeta.FindAsync(filterDef, options: new() + { + Sort = sortDef, + Skip = filter.Offset, + Limit = filter.Size + }); + var countTask = _dc.KnowledgeCollectionFileMeta.CountDocumentsAsync(filterDef); + await Task.WhenAll([docsTask, countTask]); + + var docs = docsTask.Result.ToList(); + var count = countTask.Result; var files = docs?.Select(x => new KnowledgeDocMetaData { @@ -229,7 +239,7 @@ public partial class MongoRepository return new PagedItems { Items = files, - Count = (int)count + Count = count }; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 196c6dfc..9a236277 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -169,7 +169,7 @@ public partial class MongoRepository return true; } - public PagedItems GetInstructionLogs(InstructLogFilter filter) + public async ValueTask> GetInstructionLogs(InstructLogFilter filter) { if (filter == null) { @@ -251,8 +251,18 @@ public partial class MongoRepository var filterDef = logBuilder.And(logFilters); var sortDef = Builders.Sort.Descending(x => x.CreatedTime); - var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); - var count = _dc.InstructionLogs.CountDocuments(filterDef); + + var docsTask = _dc.InstructionLogs.FindAsync(filterDef, options: new() + { + Sort = sortDef, + Skip = filter.Offset, + Limit = filter.Size + }); + var countTask = _dc.InstructionLogs.CountDocumentsAsync(filterDef); + await Task.WhenAll([docsTask, countTask]); + + var docs = docsTask.Result.ToList(); + var count = countTask.Result; var logs = docs.Select(x => { @@ -270,7 +280,7 @@ public partial class MongoRepository return new PagedItems { Items = logs, - Count = (int)count + Count = count }; } @@ -288,11 +298,18 @@ public partial class MongoRepository { filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); } - if (!filter.UserIds.IsNullOrEmpty()) { filters.Add(builder.In(x => x.UserId, filter.UserIds)); } + if (filter.StartTime.HasValue) + { + filters.Add(builder.Gte(x => x.CreatedTime, filter.StartTime.Value)); + } + if (filter.EndTime.HasValue) + { + filters.Add(builder.Lte(x => x.CreatedTime, filter.EndTime.Value)); + } var convDocs = _dc.InstructionLogs.Find(builder.And(filters)) .Sort(sortDef) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index a6f316f2..f30e52fc 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -205,7 +205,7 @@ public partial class MongoRepository } } - public PagedItems GetUsers(UserFilter filter) + public async ValueTask> GetUsers(UserFilter filter) { if (filter == null) { @@ -246,14 +246,23 @@ public partial class MongoRepository var sortDef = Builders.Sort.Descending(x => x.CreatedTime); // Search - var userDocs = _dc.Users.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); - var count = _dc.Users.CountDocuments(filterDef); + var docsTask = _dc.Users.FindAsync(filterDef, options: new() + { + Sort = sortDef, + Skip = filter.Offset, + Limit = filter.Size + }); + var countTask = _dc.Users.CountDocumentsAsync(filterDef); + await Task.WhenAll([docsTask, countTask]); - var users = userDocs.Select(x => x.ToUser()).ToList(); + var docs = docsTask.Result.ToList(); + var count = countTask.Result; + + var users = docs.Select(x => x.ToUser()).ToList(); return new PagedItems { Items = users, - Count = (int)count + Count = count }; }