diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs index 8eaee99b..c9fa1cd7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs @@ -7,4 +7,5 @@ public interface IInstructHook string SelfId { get; } Task BeforeCompletion(Agent agent, RoleDialogModel message); Task AfterCompletion(Agent agent, InstructResult result); + Task OnResponseGenerated(InstructResponseModel response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs index b59e4640..f19cca5c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs @@ -15,4 +15,9 @@ public class InstructHookBase : IInstructHook { return; } + + public virtual async Task OnResponseGenerated(InstructResponseModel response) + { + return; + } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs new file mode 100644 index 00000000..d8cafcee --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructLogFilter : Pagination +{ + public List? AgentIds { get; set; } + public List? Providers { get; set; } + public List? Models { get; set; } + public List? States { get; set; } + + public static InstructLogFilter Empty() + { + return new InstructLogFilter(); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs new file mode 100644 index 00000000..e2ee794d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructResponseModel +{ + public string? AgentId { get; set; } + public string Provider { get; set; } + public string Model { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs new file mode 100644 index 00000000..f77b08fe --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -0,0 +1,24 @@ +namespace BotSharp.Abstraction.Loggers.Models; + +public class InstructionLogModel +{ + [JsonPropertyName("agent_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentId { get; set; } + + [JsonPropertyName("provider")] + public string Provider { get; set; } = default!; + + [JsonPropertyName("model")] + public string Model { get; set; } = default!; + + [JsonPropertyName("user_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserId { get; set; } + + [JsonPropertyName("states")] + public Dictionary States { get; set; } = []; + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs index 81286489..b1dfa734 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -13,3 +13,17 @@ public class KeyValue return $"Key: {Key}, Value: {Value}"; } } + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public T? Value { get; set; } + + public override string ToString() + { + return $"Key: {Key}, Value: {Value}"; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 235c449b..6f8de07d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; @@ -171,6 +172,14 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); #endregion + #region Instruction Log + bool SaveInstructionLogs(IEnumerable logs) + => throw new NotImplementedException(); + + PagedItems GetInstructionLogs(InstructLogFilter filter) + => throw new NotImplementedException(); + #endregion + #region Statistics BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index 65f7b78e..af222d0d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -92,4 +92,20 @@ public static class StringExtensions return JsonSerializer.Deserialize(text, options); } + + public static bool IsPrimitiveValue(this string value) + { + return int.TryParse(value, out _) || + long.TryParse(value, out _) || + double.TryParse(value, out _) || + float.TryParse(value, out _) || + bool.TryParse(value, out _) || + char.TryParse(value, out _) || + byte.TryParse(value, out _) || + sbyte.TryParse(value, out _) || + short.TryParse(value, out _) || + ushort.TryParse(value, out _) || + uint.TryParse(value, out _) || + ulong.TryParse(value, out _); + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 7bd1dd15..5769320b 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -33,13 +33,19 @@ public partial class FileRepository var stateFile = Path.Combine(dir, STATE_FILE); if (!File.Exists(stateFile)) { - File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); + File.WriteAllText(stateFile, "[]"); + } + + var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE); + if (!File.Exists(latestStateFile)) + { + File.WriteAllText(latestStateFile, "{}"); } var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); if (!File.Exists(breakpointFile)) { - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); + File.WriteAllText(breakpointFile, "[]"); } } @@ -300,14 +306,21 @@ public partial class FileRepository if (states.IsNullOrEmpty()) return; var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (string.IsNullOrEmpty(convDir)) return; + + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) { - var stateFile = Path.Combine(convDir, STATE_FILE); - if (File.Exists(stateFile)) - { - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateFile, stateStr); - } + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); + } + + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) + { + var latestStates = BuildLatestStates(states); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateFile, stateStr); } } @@ -427,25 +440,57 @@ public partial class FileRepository } // Check states - if (filter != null && !filter.States.IsNullOrEmpty()) + if (matched && filter != null && !filter.States.IsNullOrEmpty()) { - var stateFile = Path.Combine(d, STATE_FILE); - var convStates = CollectConversationStates(stateFile); - foreach (var pair in filter.States) + var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE); + var convStates = CollectConversationLatestStates(latestStateFile); + + if (convStates.IsNullOrEmpty()) { - if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - - var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); - if (foundState == null) + matched = false; + } + else + { + foreach (var pair in filter.States) { - matched = false; - break; - } + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - if (!string.IsNullOrWhiteSpace(pair.Value)) - { - var curValue = foundState.Values.LastOrDefault()?.Data; - matched = matched && pair.Value.IsEqualTo(curValue); + var components = pair.Key.Split(".").ToList(); + var primaryKey = components[0]; + if (convStates.TryGetValue(primaryKey, out var doc)) + { + var elem = doc.RootElement.GetProperty("data"); + if (components.Count < 2) + { + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + if (elem.ValueKind == JsonValueKind.Array) + { + matched = elem.EnumerateArray().Select(x => x.ToString()).Contains(pair.Value); + } + else if (elem.ValueKind == JsonValueKind.String) + { + matched = elem.GetString() == pair.Value; + } + else + { + matched = elem.GetRawText() == pair.Value; + } + } + } + else + { + var paths = components.Where((_, idx) => idx > 0); + var found = FindState(elem, paths, pair.Value); + matched = found != null; + } + } + else + { + matched = false; + } + + if (!matched) break; } } } @@ -575,8 +620,9 @@ public partial class FileRepository // Handle truncated states var refTime = dialogs.ElementAt(foundIdx).MetaData.CreatedTime; var stateDir = Path.Combine(convDir, STATE_FILE); + var latestStateDir = Path.Combine(convDir, CONV_LATEST_STATE_FILE); var states = CollectConversationStates(stateDir); - isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); + isSaved = HandleTruncatedStates(stateDir, latestStateDir, states, messageId, refTime); // Handle truncated breakpoints var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); @@ -703,7 +749,7 @@ public partial class FileRepository return isSaved; } - private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) + private bool HandleTruncatedStates(string stateDir, string latestStateDir, List states, string refMsgId, DateTime refTime) { var truncatedStates = new List(); foreach (var state in states) @@ -724,6 +770,10 @@ public partial class FileRepository } var isSaved = SaveTruncatedStates(stateDir, truncatedStates); + if (isSaved) + { + SaveTruncatedLatestStates(latestStateDir, truncatedStates); + } return isSaved; } @@ -794,6 +844,17 @@ public partial class FileRepository return true; } + private bool SaveTruncatedLatestStates(string latestStateDir, List states) + { + if (string.IsNullOrEmpty(latestStateDir) || states == null) return false; + if (!File.Exists(latestStateDir)) File.Create(latestStateDir); + + var latestStates = BuildLatestStates(states); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateDir, stateStr); + return true; + } + private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) { if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; @@ -804,22 +865,98 @@ public partial class FileRepository return true; } - private string? EncodeText(string? text) + private Dictionary CollectConversationLatestStates(string latestStateDir) { - if (string.IsNullOrEmpty(text)) return text; + if (string.IsNullOrEmpty(latestStateDir) || !File.Exists(latestStateDir)) return []; - var bytes = Encoding.UTF8.GetBytes(text); - var encoded = Convert.ToBase64String(bytes); - return encoded; + var str = File.ReadAllText(latestStateDir); + var states = JsonSerializer.Deserialize>(str, _options); + return states ?? []; } - private string? DecodeText(string? text) + private Dictionary BuildLatestStates(List states) { - if (string.IsNullOrEmpty(text)) return text; + var endNodes = new Dictionary(); + foreach (var pair in states) + { + var value = pair.Values?.LastOrDefault(); + if (value == null || !value.Active) continue; - var decoded = Convert.FromBase64String(text); - var origin = Encoding.UTF8.GetString(decoded); - return origin; + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _options); + var json = JsonDocument.Parse(jsonStr); + endNodes[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = value.Data }, _options); + var json = JsonDocument.Parse(str); + endNodes[pair.Key] = json; + } + } + + return endNodes; + } + + private JsonElement? FindState(JsonElement? root, IEnumerable paths, string? targetValue) + { + JsonElement? elem = null; + + if (root == null || paths.IsNullOrEmpty()) + { + return elem; + } + + elem = root; + for (int i = 0; i < paths.Count(); i++) + { + var field = paths.ElementAt(i); + if (elem.Value.ValueKind == JsonValueKind.Array) + { + if (elem.Value.EnumerateArray().IsNullOrEmpty()) + { + elem = null; + break; + } + else + { + foreach (var item in elem.Value.EnumerateArray()) + { + var subPaths = paths.Where((_, idx) => idx >= i); + elem = FindState(item, subPaths, targetValue); + if (elem != null) + { + return elem; + } + } + } + } + else if (elem.Value.TryGetProperty(field, out var prop)) + { + elem = prop; + } + } + + if (elem != null && !string.IsNullOrWhiteSpace(targetValue)) + { + if (elem.Value.ValueKind == JsonValueKind.Array) + { + var isInArray = elem.Value.EnumerateArray().Select(x => x.ToString()).Contains(targetValue); + return isInArray ? elem : null; + } + else if ((elem.Value.ValueKind == JsonValueKind.String && elem.Value.GetString() == targetValue) + || (elem.Value.ValueKind != JsonValueKind.String && elem.Value.GetRawText() == targetValue)) + { + return elem; + } + else + { + return null; + } + } + + return elem; } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index f82cb6fa..9f7ca305 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; using System.IO; @@ -122,6 +123,40 @@ namespace BotSharp.Core.Repository } #endregion + #region Instruction Log + public bool SaveInstructionLogs(IEnumerable logs) + { + if (logs.IsNullOrEmpty()) return false; + + var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER); + if (!Directory.Exists(baseDir)) + { + Directory.CreateDirectory(baseDir); + } + + foreach (var log in logs) + { + var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.log"); + var text = JsonSerializer.Serialize(log, _options); + File.WriteAllText(file, text); + } + return true; + } + + public PagedItems GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + return new PagedItems + { + + }; + } + #endregion + #region Private methods private int GetNextLogIndex(string logDir, string id) { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 78e90e4b..f26e8bc0 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -6,7 +6,6 @@ using BotSharp.Abstraction.Users.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; - namespace BotSharp.Core.Repository; public partial class FileRepository : IBotSharpRepository @@ -34,6 +33,7 @@ public partial class FileRepository : IBotSharpRepository private const string DIALOG_FILE = "dialogs.json"; private const string STATE_FILE = "state.json"; private const string BREAKPOINT_FILE = "breakpoint.json"; + private const string CONV_LATEST_STATE_FILE = "latest-state.json"; private const string TRANSLATION_MEMORY_FILE = "memory.json"; private const string USERS_FOLDER = "users"; @@ -54,6 +54,7 @@ public partial class FileRepository : IBotSharpRepository private const string STATS_FILE = "stats.json"; private const string CRON_FILE = "cron.json"; + private const string INSTRUCTION_LOG_FOLDER = "instruction-logs"; public FileRepository( IServiceProvider services, diff --git a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs index 6ccd70ac..0404ceb4 100644 --- a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs +++ b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs @@ -15,6 +15,7 @@ public static class BotSharpLoggerExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); return services; } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs new file mode 100644 index 00000000..03e1f8ff --- /dev/null +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -0,0 +1,96 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Users; +using System.Text.Json; + +namespace BotSharp.Logger.Hooks; + +public class InstructionLogHook : InstructHookBase +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly IUserIdentity _user; + + public InstructionLogHook( + IServiceProvider services, + ILogger logger, + IUserIdentity user, + BotSharpOptions options) + { + _services = services; + _logger = logger; + _user = user; + _options = options; + } + + public override async Task OnResponseGenerated(InstructResponseModel response) + { + if (response == null) return; + + var db = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + var user = db.GetUserById(_user.Id); + db.SaveInstructionLogs(new List + { + new InstructionLogModel + { + AgentId = response.AgentId, + Provider = response.Provider, + Model = response.Model, + States = state.GetStates(), + UserId = user?.Id + } + }); + return; + } + + private Dictionary CollectStates() + { + var res = new Dictionary(); + var state = _services.GetRequiredService(); + var curStates = state.GetStates(); + + curStates["test"] = JsonSerializer.Serialize(new + { + Number = "789", + Dummy = new + { + Id = 123, + Name = "name", + Score = 12.123 + }, + Items = new List + { + new + { + Name = "image", + Label = "before-service", + Attribute = new + { + Location = "Chicago", + Time = "afternoon" + } + }, + new + { + Name = "pdf", + Label = "after-service", + Attribute = new + { + Location = "New York", + Time = "morning" + } + }, + }, + Lists = new List + { + "abc", + "bcd" + } + }, _options.JsonSerializerOptions); + return curStates; + } +} diff --git a/src/Infrastructure/BotSharp.Logger/Using.cs b/src/Infrastructure/BotSharp.Logger/Using.cs index 5115e776..9c975007 100644 --- a/src/Infrastructure/BotSharp.Logger/Using.cs +++ b/src/Infrastructure/BotSharp.Logger/Using.cs @@ -12,4 +12,5 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Instructs; global using BotSharp.Logger.Hooks; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index dbce9598..45054223 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -75,6 +75,18 @@ public class InstructModeController : ControllerBase { new RoleDialogModel(AgentRole.User, input.Text) }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = input.AgentId, + Provider = input.Provider, + Model = input.Model + }); + } + return message.Content; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index ace9472d..8fb85239 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -14,4 +14,5 @@ public class ConversationDocument : MongoBase public List Tags { get; set; } = []; public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } + public Dictionary LatestStates { get; set; } = new(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs new file mode 100644 index 00000000..615e93af --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs @@ -0,0 +1,38 @@ +using BotSharp.Abstraction.Loggers.Models; +using System.Text.Json; + +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class InstructionLogBetaDocument : MongoBase +{ + public string? AgentId { get; set; } + public string Provider { get; set; } = default!; + public string Model { get; set; } = default!; + public string? UserId { get; set; } + public Dictionary States { get; set; } = new(); + public DateTime CreatedTime { get; set; } + + public static InstructionLogBetaDocument ToMongoModel(InstructionLogModel log) + { + return new InstructionLogBetaDocument + { + AgentId = log.AgentId, + Provider = log.Provider, + Model = log.Model, + UserId = log.UserId, + CreatedTime = log.CreatedTime + }; + } + + public static InstructionLogModel ToDomainModel(InstructionLogBetaDocument log) + { + return new InstructionLogModel + { + AgentId = log.AgentId, + Provider = log.Provider, + Model = log.Model, + UserId = log.UserId, + CreatedTime = log.CreatedTime + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 4aa82b2e..9adc04b9 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -193,4 +193,6 @@ public class MongoDbContext public IMongoCollection GlobalStatistics => GetCollectionOrCreate("GlobalStatistics"); + public IMongoCollection InstructionLogs + => GetCollectionOrCreate("InstructionLogsBeta"); } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index a78db60c..1c0eeb2f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -23,7 +24,8 @@ public partial class MongoRepository Status = conversation.Status, Tags = conversation.Tags ?? new(), CreatedTime = utcNow, - UpdatedTime = utcNow + UpdatedTime = utcNow, + LatestStates = [] }; var dialogDoc = new ConversationDialogDocument @@ -72,8 +74,9 @@ public partial class MongoRepository var cronDeleted = _dc.CrontabItems.DeleteMany(conbTabItems); var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0 - || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0; + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 + || promptLogDeleted.DeletedCount > 0 || contentLogDeleted.DeletedCount > 0 + || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0; } [SideCar] @@ -266,6 +269,14 @@ public partial class MongoRepository .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); + + // Update latest states + var endNodes = BuildLatestStates(saveStates); + var filter = Builders.Filter.Eq(x => x.Id, conversationId); + var update = Builders.Update.Set(x => x.LatestStates, endNodes) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Conversations.UpdateOne(filter, update); } public void UpdateConversationStatus(string conversationId, string status) @@ -371,21 +382,47 @@ public partial class MongoRepository } // Filter states - var stateFilters = new List>(); if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty()) { foreach (var pair in filter.States) { - var elementFilters = new List> { Builders.Filter.Eq(x => x.Key, pair.Key) }; - if (!string.IsNullOrEmpty(pair.Value)) - { - elementFilters.Add(Builders.Filter.Eq("Values.Data", pair.Value)); - } - stateFilters.Add(Builders.Filter.ElemMatch(x => x.States, Builders.Filter.And(elementFilters))); - } + if (string.IsNullOrWhiteSpace(pair.Key)) continue; - var targetConvIds = _dc.ConversationStates.Find(Builders.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList(); - convFilters.Add(convBuilder.In(x => x.Id, targetConvIds)); + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "LatestStates"); + var formattedKey = string.Join(".", keys); + + if (string.IsNullOrWhiteSpace(pair.Value)) + { + convFilters.Add(convBuilder.Exists(formattedKey)); + } + else if (bool.TryParse(pair.Value, out var boolValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, boolValue)); + } + else if (int.TryParse(pair.Value, out var intValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, intValue)); + } + else if (decimal.TryParse(pair.Value, out var decimalValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, decimalValue)); + } + else if (float.TryParse(pair.Value, out var floatValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, floatValue)); + } + else if (double.TryParse(pair.Value, out var doubleValue)) + { + convFilters.Add(convBuilder.Eq(formattedKey, doubleValue)); + } + else + { + convFilters.Add(convBuilder.Eq(formattedKey, pair.Value)); + } + } } // Sort and paginate @@ -527,6 +564,7 @@ public partial class MongoRepository var stateFilter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault(); + var endNodes = new Dictionary(); if (foundStates != null) { // Truncate states @@ -550,6 +588,7 @@ public partial class MongoRepository truncatedStates.Add(state); } foundStates.States = truncatedStates; + endNodes = BuildLatestStates(truncatedStates); } // Truncate breakpoints @@ -573,6 +612,7 @@ public partial class MongoRepository // Update conversation var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.LatestStates, endNodes) .Set(x => x.DialogCount, truncatedDialogs.Count); _dc.Conversations.UpdateOne(convFilter, updateConv); @@ -621,8 +661,6 @@ public partial class MongoRepository return keys; } - - private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); @@ -640,4 +678,29 @@ public partial class MongoRepository return pascalCase.ToString(); } + + private Dictionary BuildLatestStates(List states) + { + var endNodes = new Dictionary(); + foreach (var pair in states) + { + var value = pair.Values?.LastOrDefault(); + if (value == null || !value.Active) continue; + + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + endNodes[pair.Key] = json; + } + catch + { + var str = JsonSerializer.Serialize(new { Data = value.Data }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(str); + endNodes[pair.Key] = json; + } + } + + return endNodes; + } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index e4c64527..c90960dd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; +using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -110,4 +112,102 @@ public partial class MongoRepository return logs; } #endregion + + #region Instruction Log + public bool SaveInstructionLogs(IEnumerable logs) + { + if (logs.IsNullOrEmpty()) return false; + + var docs = new List(); + foreach (var log in logs) + { + var doc = InstructionLogBetaDocument.ToMongoModel(log); + foreach (var pair in log.States) + { + try + { + var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + doc.States[pair.Key] = json; + } + catch + { + var jsonStr = JsonSerializer.Serialize(new { Data = pair.Value }, _botSharpOptions.JsonSerializerOptions); + var json = BsonDocument.Parse(jsonStr); + doc.States[pair.Key] = json; + } + } + docs.Add(doc); + } + + _dc.InstructionLogs.InsertMany(docs); + return true; + } + + public PagedItems GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var builder = Builders.Filter; + var filters = new List>() { builder.Empty }; + + // Filter logs + if (!filter.AgentIds.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.AgentId, filter.AgentIds)); + } + if (!filter.Providers.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.Provider, filter.Providers)); + } + if (!filter.Models.IsNullOrEmpty()) + { + filters.Add(builder.In(x => x.Model, filter.Models)); + } + + if (!filter.States.IsNullOrEmpty()) + { + foreach (var pair in filter.States) + { + if (string.IsNullOrWhiteSpace(pair.Key)) continue; + + // Format key + var keys = pair.Key.Split(".").ToList(); + keys.Insert(1, "data"); + keys.Insert(0, "States"); + var formattedKey = string.Join(".", keys); + + if (pair.Value == null) + { + filters.Add(builder.Exists(formattedKey)); + } + else + { + filters.Add(builder.Eq(formattedKey, pair.Value)); + } + } + } + + var filterDef = builder.And(filters); + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); + var count = _dc.InstructionLogs.CountDocuments(filterDef); + + var logs = docs.Select(x => + { + var log = InstructionLogBetaDocument.ToDomainModel(x); + log.States = x.States.ToDictionary(x => x.Key, x => x.Value.GetElement("data").Value.ToString() ?? string.Empty); + return log; + }).ToList(); + + return new PagedItems + { + Items = logs, + Count = (int)count + }; + } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 258c1883..c91a291a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Options; using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -7,16 +8,19 @@ public partial class MongoRepository : IBotSharpRepository private readonly MongoDbContext _dc; private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly BotSharpOptions _botSharpOptions; private UpdateOptions _options; public MongoRepository( MongoDbContext dc, IServiceProvider services, - ILogger logger) + ILogger logger, + BotSharpOptions botSharpOptions) { _dc = dc; _services = services; _logger = logger; + _botSharpOptions = botSharpOptions; _options = new UpdateOptions { IsUpsert = true,