add latest state

src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs

@@ -7,4 +7,5 @@ public interface IInstructHook
     string SelfId { get; }
     Task BeforeCompletion(Agent agent, RoleDialogModel message);
     Task AfterCompletion(Agent agent, InstructResult result);
+    Task OnResponseGenerated(InstructResponseModel response);
 }

src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs

@@ -15,4 +15,9 @@ public class InstructHookBase : IInstructHook
     {
         return;
     }
+
+    public virtual async Task OnResponseGenerated(InstructResponseModel response)
+    {
+        return;
+    }
 }

src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+
+namespace BotSharp.Abstraction.Instructs.Models;
+
+public class InstructLogFilter : Pagination
+{
+    public List<string>? AgentIds { get; set; }
+    public List<string>? Providers { get; set; }
+    public List<string>? Models { get; set; }
+    public List<KeyValue>? States { get; set; }
+
+    public static InstructLogFilter Empty()
+    {
+        return new InstructLogFilter();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Instructs.Models;
+
+public class InstructResponseModel
+{
+    public string? AgentId { get; set; }
+    public string Provider { get; set; }
+    public string Model { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs

@@ -0,0 +1,24 @@
+namespace BotSharp.Abstraction.Loggers.Models;
+
+public class InstructionLogModel
+{
+    [JsonPropertyName("agent_id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? AgentId { get; set; }
+
+    [JsonPropertyName("provider")]
+    public string Provider { get; set; } = default!;
+
+    [JsonPropertyName("model")]
+    public string Model { get; set; } = default!;
+
+    [JsonPropertyName("user_id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? UserId { get; set; }
+
+    [JsonPropertyName("states")]
+    public Dictionary<string, string> States { get; set; } = [];
+
+    [JsonPropertyName("created_time")]
+    public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
+}

src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs

@@ -13,3 +13,17 @@ public class KeyValue
         return $"Key: {Key}, Value: {Value}";
     }
 }
+
+public class KeyValue<T>
+{
+    [JsonPropertyName("key")]
+    public string Key { get; set; }
+
+    [JsonPropertyName("value")]
+    public T? Value { get; set; }
+
+    public override string ToString()
+    {
+        return $"Key: {Key}, Value: {Value}";
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
@@ -171,6 +172,14 @@ public interface IBotSharpRepository : IHaveServiceProvider
         => throw new NotImplementedException();
     #endregion
 
+    #region Instruction Log
+    bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
+        => throw new NotImplementedException();
+
+    PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+        => throw new NotImplementedException();
+    #endregion
+
     #region Statistics
     BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
         => throw new NotImplementedException();

src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs

@@ -92,4 +92,20 @@ public static class StringExtensions
 
         return JsonSerializer.Deserialize<T[]>(text, options);
     }
+
+    public static bool IsPrimitiveValue(this string value)
+    {
+        return int.TryParse(value, out _) ||
+               long.TryParse(value, out _) ||
+               double.TryParse(value, out _) ||
+               float.TryParse(value, out _) ||
+               bool.TryParse(value, out _) ||
+               char.TryParse(value, out _) ||
+               byte.TryParse(value, out _) ||
+               sbyte.TryParse(value, out _) ||
+               short.TryParse(value, out _) ||
+               ushort.TryParse(value, out _) ||
+               uint.TryParse(value, out _) ||
+               ulong.TryParse(value, out _);
+    }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -33,13 +33,19 @@ public partial class FileRepository
         var stateFile = Path.Combine(dir, STATE_FILE);
         if (!File.Exists(stateFile))
         {
-            File.WriteAllText(stateFile, JsonSerializer.Serialize(new List<StateKeyValue>(), _options));
+            File.WriteAllText(stateFile, "[]");
+        }
+
+        var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE);
+        if (!File.Exists(latestStateFile))
+        {
+            File.WriteAllText(latestStateFile, "{}");
         }
 
         var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE);
         if (!File.Exists(breakpointFile))
         {
-            File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List<ConversationBreakpoint>(), _options));
+            File.WriteAllText(breakpointFile, "[]");
         }
     }
 
@@ -300,14 +306,21 @@ public partial class FileRepository
         if (states.IsNullOrEmpty()) return;
 
         var convDir = FindConversationDirectory(conversationId);
-        if (!string.IsNullOrEmpty(convDir))
+        if (string.IsNullOrEmpty(convDir)) return;
+
+        var stateFile = Path.Combine(convDir, STATE_FILE);
+        if (File.Exists(stateFile))
         {
-            var stateFile = Path.Combine(convDir, STATE_FILE);
+            var stateStr = JsonSerializer.Serialize(states, _options);
-            if (File.Exists(stateFile))
+            File.WriteAllText(stateFile, stateStr);
-            {
+        }
-                var stateStr = JsonSerializer.Serialize(states, _options);
+
-                File.WriteAllText(stateFile, stateStr);
+        var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
-            }
+        if (File.Exists(latestStateFile))
+        {
+            var latestStates = BuildLatestStates(states);
+            var stateStr = JsonSerializer.Serialize(latestStates, _options);
+            File.WriteAllText(latestStateFile, stateStr);
         }
     }
 
@@ -427,25 +440,57 @@ public partial class FileRepository
             }
 
             // Check states
-            if (filter != null && !filter.States.IsNullOrEmpty())
+            if (matched && filter != null && !filter.States.IsNullOrEmpty())
             {
-                var stateFile = Path.Combine(d, STATE_FILE);
+                var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE);
-                var convStates = CollectConversationStates(stateFile);
+                var convStates = CollectConversationLatestStates(latestStateFile);
-                foreach (var pair in filter.States)
+
+                if (convStates.IsNullOrEmpty())
                 {
-                    if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue;
+                    matched = false;
-
+                }
-                    var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key));
+                else
-                    if (foundState == null)
+                {
+                    foreach (var pair in filter.States)
                     {
-                        matched = false;
+                        if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue;
-                        break;
-                    }
 
-                    if (!string.IsNullOrWhiteSpace(pair.Value))
+                        var components = pair.Key.Split(".").ToList();
-                    {
+                        var primaryKey = components[0];
-                        var curValue = foundState.Values.LastOrDefault()?.Data;
+                        if (convStates.TryGetValue(primaryKey, out var doc))
-                        matched = matched && pair.Value.IsEqualTo(curValue);
+                        {
+                            var elem = doc.RootElement.GetProperty("data");
+                            if (components.Count < 2)
+                            {
+                                if (!string.IsNullOrWhiteSpace(pair.Value))
+                                {
+                                    if (elem.ValueKind == JsonValueKind.Array)
+                                    {
+                                        matched = elem.EnumerateArray().Select(x => x.ToString()).Contains(pair.Value);
+                                    }
+                                    else if (elem.ValueKind == JsonValueKind.String)
+                                    {
+                                        matched = elem.GetString() == pair.Value;
+                                    }
+                                    else
+                                    {
+                                        matched = elem.GetRawText() == pair.Value;
+                                    }
+                                }
+                            }
+                            else
+                            {
+                                var paths = components.Where((_, idx) => idx > 0);
+                                var found = FindState(elem, paths, pair.Value);
+                                matched = found != null;
+                            }
+                        }
+                        else
+                        {
+                            matched = false;
+                        }
+
+                        if (!matched) break;
                     }
                 }
             }
@@ -575,8 +620,9 @@ public partial class FileRepository
         // Handle truncated states
         var refTime = dialogs.ElementAt(foundIdx).MetaData.CreatedTime;
         var stateDir = Path.Combine(convDir, STATE_FILE);
+        var latestStateDir = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
         var states = CollectConversationStates(stateDir);
-        isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime);
+        isSaved = HandleTruncatedStates(stateDir, latestStateDir, states, messageId, refTime);
 
         // Handle truncated breakpoints
         var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE);
@@ -703,7 +749,7 @@ public partial class FileRepository
         return isSaved;
     }
 
-    private bool HandleTruncatedStates(string stateDir, List<StateKeyValue> states, string refMsgId, DateTime refTime)
+    private bool HandleTruncatedStates(string stateDir, string latestStateDir, List<StateKeyValue> states, string refMsgId, DateTime refTime)
     {
         var truncatedStates = new List<StateKeyValue>();
         foreach (var state in states)
@@ -724,6 +770,10 @@ public partial class FileRepository
         }
 
         var isSaved = SaveTruncatedStates(stateDir, truncatedStates);
+        if (isSaved)
+        {
+            SaveTruncatedLatestStates(latestStateDir, truncatedStates);
+        }
         return isSaved;
     }
 
@@ -794,6 +844,17 @@ public partial class FileRepository
         return true;
     }
 
+    private bool SaveTruncatedLatestStates(string latestStateDir, List<StateKeyValue> states)
+    {
+        if (string.IsNullOrEmpty(latestStateDir) || states == null) return false;
+        if (!File.Exists(latestStateDir)) File.Create(latestStateDir);
+
+        var latestStates = BuildLatestStates(states);
+        var stateStr = JsonSerializer.Serialize(latestStates, _options);
+        File.WriteAllText(latestStateDir, stateStr);
+        return true;
+    }
+
     private bool SaveTruncatedBreakpoints(string breakpointDir, List<ConversationBreakpoint> breakpoints)
     {
         if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false;
@@ -804,22 +865,98 @@ public partial class FileRepository
         return true;
     }
 
-    private string? EncodeText(string? text)
+    private Dictionary<string, JsonDocument> CollectConversationLatestStates(string latestStateDir)
     {
-        if (string.IsNullOrEmpty(text)) return text;
+        if (string.IsNullOrEmpty(latestStateDir) || !File.Exists(latestStateDir)) return [];
 
-        var bytes = Encoding.UTF8.GetBytes(text);
+        var str = File.ReadAllText(latestStateDir);
-        var encoded = Convert.ToBase64String(bytes);
+        var states = JsonSerializer.Deserialize<Dictionary<string, JsonDocument>>(str, _options);
-        return encoded;
+        return states ?? [];
     }
 
-    private string? DecodeText(string? text)
+    private Dictionary<string, JsonDocument> BuildLatestStates(List<StateKeyValue> states)
     {
-        if (string.IsNullOrEmpty(text)) return text;
+        var endNodes = new Dictionary<string, JsonDocument>();
+        foreach (var pair in states)
+        {
+            var value = pair.Values?.LastOrDefault();
+            if (value == null || !value.Active) continue;
 
-        var decoded = Convert.FromBase64String(text);
+            try
-        var origin = Encoding.UTF8.GetString(decoded);
+            {
-        return origin;
+                var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _options);
+                var json = JsonDocument.Parse(jsonStr);
+                endNodes[pair.Key] = json;
+            }
+            catch
+            {
+                var str = JsonSerializer.Serialize(new { Data = value.Data }, _options);
+                var json = JsonDocument.Parse(str);
+                endNodes[pair.Key] = json;
+            }
+        }
+
+        return endNodes;
+    }
+
+    private JsonElement? FindState(JsonElement? root, IEnumerable<string> paths, string? targetValue)
+    {
+        JsonElement? elem = null;
+
+        if (root == null || paths.IsNullOrEmpty())
+        {
+            return elem;
+        }
+
+        elem = root;
+        for (int i = 0; i < paths.Count(); i++)
+        {
+            var field = paths.ElementAt(i);
+            if (elem.Value.ValueKind == JsonValueKind.Array)
+            {
+                if (elem.Value.EnumerateArray().IsNullOrEmpty())
+                {
+                    elem = null;
+                    break;
+                }
+                else
+                {
+                    foreach (var item in elem.Value.EnumerateArray())
+                    {
+                        var subPaths = paths.Where((_, idx) => idx >= i);
+                        elem = FindState(item, subPaths, targetValue);
+                        if (elem != null)
+                        {
+                            return elem;
+                        }
+                    }
+                }
+            }
+            else if (elem.Value.TryGetProperty(field, out var prop))
+            {
+                elem = prop;
+            }
+        }
+
+        if (elem != null && !string.IsNullOrWhiteSpace(targetValue))
+        {
+            if (elem.Value.ValueKind == JsonValueKind.Array)
+            {
+                var isInArray = elem.Value.EnumerateArray().Select(x => x.ToString()).Contains(targetValue);
+                return isInArray ? elem : null;
+            }
+            else if ((elem.Value.ValueKind == JsonValueKind.String && elem.Value.GetString() == targetValue)
+                || (elem.Value.ValueKind != JsonValueKind.String && elem.Value.GetRawText() == targetValue))
+            {
+                return elem;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        return elem;
     }
     #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
 using System.IO;
 
@@ -122,6 +123,40 @@ namespace BotSharp.Core.Repository
         }
         #endregion
 
+        #region Instruction Log
+        public bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
+        {
+            if (logs.IsNullOrEmpty()) return false;
+
+            var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER);
+            if (!Directory.Exists(baseDir))
+            {
+                Directory.CreateDirectory(baseDir);
+            }
+
+            foreach (var log in logs)
+            {
+                var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.log");
+                var text = JsonSerializer.Serialize(log, _options);
+                File.WriteAllText(file, text);
+            }
+            return true;
+        }
+
+        public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+        {
+            if (filter == null)
+            {
+                filter = InstructLogFilter.Empty();
+            }
+
+            return new PagedItems<InstructionLogModel>
+            {
+
+            };
+        }
+        #endregion
+
         #region Private methods
         private int GetNextLogIndex(string logDir, string id)
         {

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -6,7 +6,6 @@ using BotSharp.Abstraction.Users.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Tasks.Models;
 
-
 namespace BotSharp.Core.Repository;
 
 public partial class FileRepository : IBotSharpRepository
@@ -34,6 +33,7 @@ public partial class FileRepository : IBotSharpRepository
     private const string DIALOG_FILE = "dialogs.json";
     private const string STATE_FILE = "state.json";
     private const string BREAKPOINT_FILE = "breakpoint.json";
+    private const string CONV_LATEST_STATE_FILE = "latest-state.json";
     private const string TRANSLATION_MEMORY_FILE = "memory.json";
 
     private const string USERS_FOLDER = "users";
@@ -54,6 +54,7 @@ public partial class FileRepository : IBotSharpRepository
     private const string STATS_FILE = "stats.json";
 
     private const string CRON_FILE = "cron.json";
+    private const string INSTRUCTION_LOG_FOLDER = "instruction-logs";
 
     public FileRepository(
         IServiceProvider services,

src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs

@@ -15,6 +15,7 @@ public static class BotSharpLoggerExtensions
         services.AddScoped<IContentGeneratingHook, VerboseLogHook>();
         services.AddScoped<IContentGeneratingHook, GlobalStatsConversationHook>();
         services.AddScoped<IConversationHook, RateLimitConversationHook>();
+        services.AddScoped<IInstructHook, InstructionLogHook>();
         return services;
     }
 }

src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs

@@ -0,0 +1,96 @@
+using BotSharp.Abstraction.Instructs.Models;
+using BotSharp.Abstraction.Loggers.Models;
+using BotSharp.Abstraction.Options;
+using BotSharp.Abstraction.Users;
+using System.Text.Json;
+
+namespace BotSharp.Logger.Hooks;
+
+public class InstructionLogHook : InstructHookBase
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger<InstructionLogHook> _logger;
+    private readonly BotSharpOptions _options;
+    private readonly IUserIdentity _user;
+
+    public InstructionLogHook(
+        IServiceProvider services,
+        ILogger<InstructionLogHook> logger,
+        IUserIdentity user,
+        BotSharpOptions options)
+    {
+        _services = services;
+        _logger = logger;
+        _user = user;
+        _options = options;
+    }
+
+    public override async Task OnResponseGenerated(InstructResponseModel response)
+    {
+        if (response == null) return;
+
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var state = _services.GetRequiredService<IConversationStateService>();
+
+        var user = db.GetUserById(_user.Id);
+        db.SaveInstructionLogs(new List<InstructionLogModel>
+        {
+            new InstructionLogModel
+            {
+                AgentId = response.AgentId,
+                Provider = response.Provider,
+                Model = response.Model,
+                States = state.GetStates(),
+                UserId = user?.Id
+            }
+        });
+        return;
+    }
+
+    private Dictionary<string, string> CollectStates()
+    {
+        var res = new Dictionary<string, object>();
+        var state = _services.GetRequiredService<IConversationStateService>();
+        var curStates = state.GetStates();
+
+        curStates["test"] = JsonSerializer.Serialize(new
+        {
+            Number = "789",
+            Dummy = new
+            {
+                Id = 123,
+                Name = "name",
+                Score = 12.123
+            },
+            Items = new List<object>
+            {
+                new
+                {
+                    Name = "image",
+                    Label = "before-service",
+                    Attribute = new
+                    {
+                        Location = "Chicago",
+                        Time = "afternoon"
+                    }
+                },
+                new
+                {
+                    Name = "pdf",
+                    Label = "after-service",
+                    Attribute = new
+                    {
+                        Location = "New York",
+                        Time = "morning"
+                    }
+                },
+            },
+            Lists = new List<object>
+            {
+                "abc",
+                "bcd"
+            }
+        }, _options.JsonSerializerOptions);
+        return curStates;
+    }
+}

src/Infrastructure/BotSharp.Logger/Using.cs

@@ -12,4 +12,5 @@ global using BotSharp.Abstraction.Conversations.Models;
 global using BotSharp.Abstraction.Conversations;
 global using BotSharp.Abstraction.Repositories;
 global using BotSharp.Abstraction.Conversations.Settings;
+global using BotSharp.Abstraction.Instructs;
 global using BotSharp.Logger.Hooks;

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -75,6 +75,18 @@ public class InstructModeController : ControllerBase
         {
             new RoleDialogModel(AgentRole.User, input.Text)
         });
+
+        var hooks = _services.GetServices<IInstructHook>();
+        foreach (var hook in hooks)
+        {
+            await hook.OnResponseGenerated(new InstructResponseModel
+            {
+                AgentId = input.AgentId,
+                Provider = input.Provider,
+                Model = input.Model
+            });
+        }
+
         return message.Content;
     }
     #endregion

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs

@@ -14,4 +14,5 @@ public class ConversationDocument : MongoBase
     public List<string> Tags { get; set; } = [];
     public DateTime CreatedTime { get; set; }
     public DateTime UpdatedTime { get; set; }
+    public Dictionary<string, BsonDocument> LatestStates { get; set; } = new();
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs

@@ -0,0 +1,38 @@
+using BotSharp.Abstraction.Loggers.Models;
+using System.Text.Json;
+
+namespace BotSharp.Plugin.MongoStorage.Collections;
+
+public class InstructionLogBetaDocument : MongoBase
+{
+    public string? AgentId { get; set; }
+    public string Provider { get; set; } = default!;
+    public string Model { get; set; } = default!;
+    public string? UserId { get; set; }
+    public Dictionary<string, BsonDocument> States { get; set; } = new();
+    public DateTime CreatedTime { get; set; }
+
+    public static InstructionLogBetaDocument ToMongoModel(InstructionLogModel log)
+    {
+        return new InstructionLogBetaDocument
+        {
+            AgentId = log.AgentId,
+            Provider = log.Provider,
+            Model = log.Model,
+            UserId = log.UserId,
+            CreatedTime = log.CreatedTime
+        };
+    }
+
+    public static InstructionLogModel ToDomainModel(InstructionLogBetaDocument log)
+    {
+        return new InstructionLogModel
+        {
+            AgentId = log.AgentId,
+            Provider = log.Provider,
+            Model = log.Model,
+            UserId = log.UserId,
+            CreatedTime = log.CreatedTime
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

@@ -193,4 +193,6 @@ public class MongoDbContext
     public IMongoCollection<GlobalStatisticsDocument> GlobalStatistics
         => GetCollectionOrCreate<GlobalStatisticsDocument>("GlobalStatistics");
 
+    public IMongoCollection<InstructionLogBetaDocument> InstructionLogs
+        => GetCollectionOrCreate<InstructionLogBetaDocument>("InstructionLogsBeta");
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Repositories.Filters;
+using System.Text.Json;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -23,7 +24,8 @@ public partial class MongoRepository
             Status = conversation.Status,
             Tags = conversation.Tags ?? new(),
             CreatedTime = utcNow,
-            UpdatedTime = utcNow
+            UpdatedTime = utcNow,
+            LatestStates = []
         };
 
         var dialogDoc = new ConversationDialogDocument
@@ -72,8 +74,9 @@ public partial class MongoRepository
         var cronDeleted = _dc.CrontabItems.DeleteMany(conbTabItems);
         var convDeleted = _dc.Conversations.DeleteMany(filterConv);
 
-        return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0
+        return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 
-            || contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0;
+            || promptLogDeleted.DeletedCount > 0 || contentLogDeleted.DeletedCount > 0
+            || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0;
     }
 
     [SideCar]
@@ -266,6 +269,14 @@ public partial class MongoRepository
                                                                      .Set(x => x.UpdatedTime, DateTime.UtcNow);
 
         _dc.ConversationStates.UpdateOne(filterStates, updateStates);
+
+        // Update latest states
+        var endNodes = BuildLatestStates(saveStates);
+        var filter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
+        var update = Builders<ConversationDocument>.Update.Set(x => x.LatestStates, endNodes)
+                                                          .Set(x => x.UpdatedTime, DateTime.UtcNow);
+
+        _dc.Conversations.UpdateOne(filter, update);
     }
 
     public void UpdateConversationStatus(string conversationId, string status)
@@ -371,21 +382,47 @@ public partial class MongoRepository
         }
 
         // Filter states
-        var stateFilters = new List<FilterDefinition<ConversationStateDocument>>();
         if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty())
         {
             foreach (var pair in filter.States)
             {
-                var elementFilters = new List<FilterDefinition<StateMongoElement>> { Builders<StateMongoElement>.Filter.Eq(x => x.Key, pair.Key) };
+                if (string.IsNullOrWhiteSpace(pair.Key)) continue;
-                if (!string.IsNullOrEmpty(pair.Value))
-                {
-                    elementFilters.Add(Builders<StateMongoElement>.Filter.Eq("Values.Data", pair.Value));
-                }
-                stateFilters.Add(Builders<ConversationStateDocument>.Filter.ElemMatch(x => x.States, Builders<StateMongoElement>.Filter.And(elementFilters)));
-            }
 
-            var targetConvIds = _dc.ConversationStates.Find(Builders<ConversationStateDocument>.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList();
+                // Format key
-            convFilters.Add(convBuilder.In(x => x.Id, targetConvIds));
+                var keys = pair.Key.Split(".").ToList();
+                keys.Insert(1, "data");
+                keys.Insert(0, "LatestStates");
+                var formattedKey = string.Join(".", keys);
+
+                if (string.IsNullOrWhiteSpace(pair.Value))
+                {
+                    convFilters.Add(convBuilder.Exists(formattedKey));
+                }
+                else if (bool.TryParse(pair.Value, out var boolValue))
+                {
+                    convFilters.Add(convBuilder.Eq(formattedKey, boolValue));
+                }
+                else if (int.TryParse(pair.Value, out var intValue))
+                {
+                    convFilters.Add(convBuilder.Eq(formattedKey, intValue));
+                }
+                else if (decimal.TryParse(pair.Value, out var decimalValue))
+                {
+                    convFilters.Add(convBuilder.Eq(formattedKey, decimalValue));
+                }
+                else if (float.TryParse(pair.Value, out var floatValue))
+                {
+                    convFilters.Add(convBuilder.Eq(formattedKey, floatValue));
+                }
+                else if (double.TryParse(pair.Value, out var doubleValue))
+                {
+                    convFilters.Add(convBuilder.Eq(formattedKey, doubleValue));
+                }
+                else
+                {
+                    convFilters.Add(convBuilder.Eq(formattedKey, pair.Value));
+                }
+            }
         }
 
         // Sort and paginate
@@ -527,6 +564,7 @@ public partial class MongoRepository
         var stateFilter = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault();
 
+        var endNodes = new Dictionary<string, BsonDocument>();
         if (foundStates != null)
         {
             // Truncate states
@@ -550,6 +588,7 @@ public partial class MongoRepository
                     truncatedStates.Add(state);
                 }
                 foundStates.States = truncatedStates;
+                endNodes = BuildLatestStates(truncatedStates);
             }
 
             // Truncate breakpoints
@@ -573,6 +612,7 @@ public partial class MongoRepository
         // Update conversation
         var convFilter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
         var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow)
+                                                              .Set(x => x.LatestStates, endNodes)
                                                               .Set(x => x.DialogCount, truncatedDialogs.Count);
         _dc.Conversations.UpdateOne(convFilter, updateConv);
 
@@ -621,8 +661,6 @@ public partial class MongoRepository
         return keys;
     }
 
-
-
     private string ConvertSnakeCaseToPascalCase(string snakeCase)
     {
         string[] words = snakeCase.Split('_');
@@ -640,4 +678,29 @@ public partial class MongoRepository
 
         return pascalCase.ToString();
     }
+
+    private Dictionary<string, BsonDocument> BuildLatestStates(List<StateMongoElement> states)
+    {
+        var endNodes = new Dictionary<string, BsonDocument>();
+        foreach (var pair in states)
+        {
+            var value = pair.Values?.LastOrDefault();
+            if (value == null || !value.Active) continue;
+
+            try
+            {
+                var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _botSharpOptions.JsonSerializerOptions);
+                var json = BsonDocument.Parse(jsonStr);
+                endNodes[pair.Key] = json;
+            }
+            catch
+            {
+                var str = JsonSerializer.Serialize(new { Data = value.Data }, _botSharpOptions.JsonSerializerOptions);
+                var json = BsonDocument.Parse(str);
+                endNodes[pair.Key] = json;
+            }
+        }
+
+        return endNodes;
+    }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs

@@ -1,4 +1,6 @@
+using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
+using System.Text.Json;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -110,4 +112,102 @@ public partial class MongoRepository
         return logs;
     }
     #endregion
+
+    #region Instruction Log
+    public bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
+    {
+        if (logs.IsNullOrEmpty()) return false;
+
+        var docs = new List<InstructionLogBetaDocument>();
+        foreach (var log in logs)
+        {
+            var doc = InstructionLogBetaDocument.ToMongoModel(log);
+            foreach (var pair in log.States)
+            {
+                try
+                {
+                    var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _botSharpOptions.JsonSerializerOptions);
+                    var json = BsonDocument.Parse(jsonStr);
+                    doc.States[pair.Key] = json;
+                }
+                catch
+                {
+                    var jsonStr = JsonSerializer.Serialize(new { Data = pair.Value }, _botSharpOptions.JsonSerializerOptions);
+                    var json = BsonDocument.Parse(jsonStr);
+                    doc.States[pair.Key] = json;
+                }
+            }
+            docs.Add(doc);
+        }
+
+        _dc.InstructionLogs.InsertMany(docs);
+        return true;
+    }
+
+    public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+    {
+        if (filter == null)
+        {
+            filter = InstructLogFilter.Empty();
+        }
+
+        var builder = Builders<InstructionLogBetaDocument>.Filter;
+        var filters = new List<FilterDefinition<InstructionLogBetaDocument>>() { builder.Empty };
+
+        // Filter logs
+        if (!filter.AgentIds.IsNullOrEmpty())
+        {
+            filters.Add(builder.In(x => x.AgentId, filter.AgentIds));
+        }
+        if (!filter.Providers.IsNullOrEmpty())
+        {
+            filters.Add(builder.In(x => x.Provider, filter.Providers));
+        }
+        if (!filter.Models.IsNullOrEmpty())
+        {
+            filters.Add(builder.In(x => x.Model, filter.Models));
+        }
+
+        if (!filter.States.IsNullOrEmpty())
+        {
+            foreach (var pair in filter.States)
+            {
+                if (string.IsNullOrWhiteSpace(pair.Key)) continue;
+
+                // Format key
+                var keys = pair.Key.Split(".").ToList();
+                keys.Insert(1, "data");
+                keys.Insert(0, "States");
+                var formattedKey = string.Join(".", keys);
+
+                if (pair.Value == null)
+                {
+                    filters.Add(builder.Exists(formattedKey));
+                }
+                else
+                {
+                    filters.Add(builder.Eq(formattedKey, pair.Value));
+                }
+            }
+        }
+
+        var filterDef = builder.And(filters);
+        var sortDef = Builders<InstructionLogBetaDocument>.Sort.Descending(x => x.CreatedTime);
+        var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
+        var count = _dc.InstructionLogs.CountDocuments(filterDef);
+
+        var logs = docs.Select(x =>
+        {
+            var log = InstructionLogBetaDocument.ToDomainModel(x);
+            log.States = x.States.ToDictionary(x => x.Key, x => x.Value.GetElement("data").Value.ToString() ?? string.Empty);
+            return log;
+        }).ToList();
+
+        return new PagedItems<InstructionLogModel>
+        {
+            Items = logs,
+            Count = (int)count
+        };
+    }
+    #endregion
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Options;
 using Microsoft.Extensions.Logging;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
@@ -7,16 +8,19 @@ public partial class MongoRepository : IBotSharpRepository
     private readonly MongoDbContext _dc;
     private readonly IServiceProvider _services;
     private readonly ILogger<MongoRepository> _logger;
+    private readonly BotSharpOptions _botSharpOptions;
     private UpdateOptions _options;
 
     public MongoRepository(
         MongoDbContext dc,
         IServiceProvider services,
-        ILogger<MongoRepository> logger)
+        ILogger<MongoRepository> logger,
+        BotSharpOptions botSharpOptions)
     {
         _dc = dc;
         _services = services;
         _logger = logger;
+        _botSharpOptions = botSharpOptions;
         _options = new UpdateOptions
         {
             IsUpsert = true,