From f423a40f5377554d201ef6f647198778ed6409c3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 11 Feb 2025 14:27:53 -0600 Subject: [PATCH 1/2] refine state search --- .../Conversations/IConversationService.cs | 9 ++++ .../Repositories/IBotSharpRepository.cs | 4 +- .../Services/ConversationService.cs | 14 ++++++ .../Repository/BotSharpDbContext.cs | 2 +- .../FileRepository.Conversation.cs | 42 ++++++++++++++++- .../Controllers/ConversationController.cs | 10 +++++ .../Collections/ConversationDialogDocument.cs | 1 + .../Collections/ConversationStateDocument.cs | 1 + .../MongoRepository.Conversation.cs | 45 +++++++++++++++---- .../Repository/MongoRepository.User.cs | 5 +++ 10 files changed, 122 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index ddf985b4..a606045b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -63,4 +63,13 @@ public interface IConversationService bool IsConversationMode(); void SaveStates(); + + /// + /// Get conversation keys for searching + /// + /// search query + /// conversation limit + /// if pre-loading, then keys are not filter by the search query + /// + Task> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 67a1f901..caf58c43 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -146,7 +146,9 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) => throw new NotImplementedException(); - IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + => throw new NotImplementedException(); + List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 02bcab6e..315fe72b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -221,4 +221,18 @@ public partial class ConversationService : IConversationService { _state.Save(); } + + public async Task> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false) + { + var keys = new List(); + if (!preLoad && string.IsNullOrWhiteSpace(query)) + { + return keys; + } + + var db = _services.GetRequiredService(); + keys = db.GetConversationSearchKeys(convlimit: convlimit); + keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.Take(keyLimit).ToList(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 7d96357d..100bbf16 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -131,7 +131,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository public void UpdateConversationStatus(string conversationId, string status) => throw new NotImplementedException(); - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index c1cdd69f..4591b694 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -547,7 +547,7 @@ namespace BotSharp.Core.Repository } - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { var deletedMessageIds = new List(); if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) @@ -603,6 +603,46 @@ namespace BotSharp.Core.Repository } + public List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) + { + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(dir)) return []; + + var count = 0; + var keys = new List(); + + foreach (var d in Directory.GetDirectories(dir)) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + var stateFile = Path.Combine(d, STATE_FILE); + if (!File.Exists(convFile) || !File.Exists(stateFile)) + { + continue; + } + + var convJson = File.ReadAllText(convFile); + var stateJson = File.ReadAllText(stateFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + var states = JsonSerializer.Deserialize>(stateJson, _options); + if (conv == null || conv.DialogCount < messageLimit) + { + continue; + } + + var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; + keys.AddRange(stateKeys); + count++; + + if (count > convlimit) + { + break; + } + } + + return keys.Distinct().ToList(); + } + + #region Private methods private string? FindConversationDirectory(string conversationId) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index b2defb2e..4dd2e8f0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -553,6 +553,16 @@ public class ConversationController : ControllerBase } #endregion + #region Search state keys + [HttpGet("/conversation/state/keys")] + public async Task> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] bool preLoad = false) + { + var convService = _services.GetRequiredService(); + var keys = await convService.GetConversationSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad); + return keys; + } + #endregion + #region Private methods private void SetStates(IConversationService conv, NewMessageModel input) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs index 12442cdc..a0e66e98 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs @@ -4,5 +4,6 @@ public class ConversationDialogDocument : MongoBase { public string ConversationId { get; set; } public string AgentId { get; set; } + public DateTime UpdatedTime { get; set; } public List Dialogs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs index 7b515616..d83945a8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs @@ -4,6 +4,7 @@ public class ConversationStateDocument : MongoBase { public string ConversationId { get; set; } public string AgentId { get; set; } + public DateTime UpdatedTime { get; set; } public List States { get; set; } = new List(); public List Breakpoints { get; set; } = new List(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index c83ccf17..ae5082ce 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -30,7 +30,8 @@ public partial class MongoRepository Id = Guid.NewGuid().ToString(), ConversationId = convDoc.Id, AgentId = conversation.AgentId, - Dialogs = new List() + Dialogs = [], + UpdatedTime = utcNow }; var stateDoc = new ConversationStateDocument @@ -38,8 +39,9 @@ public partial class MongoRepository Id = Guid.NewGuid().ToString(), ConversationId = convDoc.Id, AgentId = conversation.AgentId, - States = new List(), - Breakpoints = new List() + States = [], + Breakpoints = [], + UpdatedTime = utcNow }; _dc.Conversations.InsertOne(convDoc); @@ -97,7 +99,8 @@ public partial class MongoRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); - var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); + var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements) + .Set(x => x.UpdatedTime, DateTime.UtcNow); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) .Inc(x => x.DialogCount, dialogs.Count); @@ -190,7 +193,8 @@ public partial class MongoRepository found.SecondaryRichContent = request.Message.RichContent; } - var update = Builders.Update.Set(x => x.Dialogs, dialogs); + var update = Builders.Update.Set(x => x.Dialogs, dialogs) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationDialogs.UpdateOne(filter, update); return true; } @@ -208,7 +212,8 @@ public partial class MongoRepository Reason = breakpoint.Reason }; var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var updateState = Builders.Update.Push(x => x.Breakpoints, newBreakpoint); + var updateState = Builders.Update.Push(x => x.Breakpoints, newBreakpoint) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterState, updateState); } @@ -258,7 +263,8 @@ public partial class MongoRepository var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); - var updateStates = Builders.Update.Set(x => x.States, saveStates); + var updateStates = Builders.Update.Set(x => x.States, saveStates) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); } @@ -500,7 +506,7 @@ public partial class MongoRepository return conversationIds.Take(batchSize).ToList(); } - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { var deletedMessageIds = new List(); if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) @@ -566,11 +572,13 @@ public partial class MongoRepository } // Update + foundStates.UpdatedTime = DateTime.UtcNow; _dc.ConversationStates.ReplaceOne(stateFilter, foundStates); } // Save dialogs foundDialog.Dialogs = truncatedDialogs; + foundDialog.UpdatedTime = DateTime.UtcNow; _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog); // Update conversation @@ -603,6 +611,27 @@ public partial class MongoRepository return deletedMessageIds; } + + public List GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100) + { + var convFilter = Builders.Filter.Gte(x => x.DialogCount, messageLimit); + var conversations = _dc.Conversations.Find(convFilter) + .SortByDescending(x => x.UpdatedTime) + .Limit(convlimit) + .ToList(); + + if (conversations.IsNullOrEmpty()) return []; + + var convIds = conversations.Select(x => x.Id).ToList(); + var stateFilter = Builders.Filter.In(x => x.ConversationId, convIds); + + var states = _dc.ConversationStates.Find(stateFilter).ToList(); + var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + return keys; + } + + + private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 3b6dd05f..9c88e8bd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -429,6 +429,11 @@ public partial class MongoRepository return true; } + public Dashboard? GetDashboard(string userId = null) + { + return null; + } + public void AddDashboardConversation(string userId, string conversationId) { var user = _dc.Users.AsQueryable() From 96b23e4244f26a347b074580f3b6e6bfc9c50833 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 11 Feb 2025 15:16:54 -0600 Subject: [PATCH 2/2] sort --- .../BotSharp.Core/Conversations/Services/ConversationService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 315fe72b..beb4e33a 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -233,6 +233,6 @@ public partial class ConversationService : IConversationService var db = _services.GetRequiredService(); keys = db.GetConversationSearchKeys(convlimit: convlimit); keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); - return keys.Take(keyLimit).ToList(); + return keys.OrderBy(x => x).Take(keyLimit).ToList(); } }