SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add latest state

Browse source
This commit is contained in:
Jicheng Lu 2025-03-04 17:25:35 -06:00
parent ded711d164
commit e0fffe191c
21 changed files with 637 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs
View file

@ -7,4 +7,5 @@ public interface IInstructHook
string SelfId { get; }
Task BeforeCompletion(Agent agent, RoleDialogModel message);
Task AfterCompletion(Agent agent, InstructResult result);
Task OnResponseGenerated(InstructResponseModel response);
}

5
src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs
View file

@ -15,4 +15,9 @@ public class InstructHookBase : IInstructHook
{
return;
}
public virtual async Task OnResponseGenerated(InstructResponseModel response)
{
return;
}
}

16
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs Normal file
View file

@ -0,0 +1,16 @@
using System.Collections.Generic;
namespace BotSharp.Abstraction.Instructs.Models;
public class InstructLogFilter : Pagination
{
public List<string>? AgentIds { get; set; }
public List<string>? Providers { get; set; }
public List<string>? Models { get; set; }
public List<KeyValue>? States { get; set; }
public static InstructLogFilter Empty()
{
return new InstructLogFilter();
}
}

8
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResponseModel.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Instructs.Models;
public class InstructResponseModel
{
public string? AgentId { get; set; }
public string Provider { get; set; }
public string Model { get; set; }
}

24
src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs Normal file
View file

@ -0,0 +1,24 @@
namespace BotSharp.Abstraction.Loggers.Models;
public class InstructionLogModel
{
[JsonPropertyName("agent_id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? AgentId { get; set; }
[JsonPropertyName("provider")]
public string Provider { get; set; } = default!;
[JsonPropertyName("model")]
public string Model { get; set; } = default!;
[JsonPropertyName("user_id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? UserId { get; set; }
[JsonPropertyName("states")]
public Dictionary<string, string> States { get; set; } = [];
[JsonPropertyName("created_time")]
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
}

14
src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs
View file

@ -13,3 +13,17 @@ public class KeyValue
return $"Key: {Key}, Value: {Value}";
}
}
public class KeyValue<T>
{
[JsonPropertyName("key")]
public string Key { get; set; }
[JsonPropertyName("value")]
public T? Value { get; set; }
public override string ToString()
{
return $"Key: {Key}, Value: {Value}";
}
}

9
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Repositories.Filters;
@ -171,6 +172,14 @@ public interface IBotSharpRepository : IHaveServiceProvider
=> throw new NotImplementedException();
#endregion
#region Instruction Log
bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
=> throw new NotImplementedException();
PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
=> throw new NotImplementedException();
#endregion
#region Statistics
BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
=> throw new NotImplementedException();

16
src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs
View file

@ -92,4 +92,20 @@ public static class StringExtensions
return JsonSerializer.Deserialize<T[]>(text, options);
}
public static bool IsPrimitiveValue(this string value)
{
return int.TryParse(value, out _) ||
long.TryParse(value, out _) ||
double.TryParse(value, out _) ||
float.TryParse(value, out _) ||
bool.TryParse(value, out _) ||
char.TryParse(value, out _) ||
byte.TryParse(value, out _) ||
sbyte.TryParse(value, out _) ||
short.TryParse(value, out _) ||
ushort.TryParse(value, out _) ||
uint.TryParse(value, out _) ||
ulong.TryParse(value, out _);
}
}

209
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
View file

@ -33,13 +33,19 @@ public partial class FileRepository
var stateFile = Path.Combine(dir, STATE_FILE);
if (!File.Exists(stateFile))
{
File.WriteAllText(stateFile, JsonSerializer.Serialize(new List<StateKeyValue>(), _options));
File.WriteAllText(stateFile, "[]");
}
var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE);
if (!File.Exists(latestStateFile))
{
File.WriteAllText(latestStateFile, "{}");
}
var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE);
if (!File.Exists(breakpointFile))
{
File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List<ConversationBreakpoint>(), _options));
File.WriteAllText(breakpointFile, "[]");
}
}
@ -300,14 +306,21 @@ public partial class FileRepository
if (states.IsNullOrEmpty()) return;
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
if (string.IsNullOrEmpty(convDir)) return;
var stateFile = Path.Combine(convDir, STATE_FILE);
if (File.Exists(stateFile))
{
var stateFile = Path.Combine(convDir, STATE_FILE);
if (File.Exists(stateFile))
{
var stateStr = JsonSerializer.Serialize(states, _options);
File.WriteAllText(stateFile, stateStr);
}
var stateStr = JsonSerializer.Serialize(states, _options);
File.WriteAllText(stateFile, stateStr);
}
var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
if (File.Exists(latestStateFile))
{
var latestStates = BuildLatestStates(states);
var stateStr = JsonSerializer.Serialize(latestStates, _options);
File.WriteAllText(latestStateFile, stateStr);
}
}
@ -427,25 +440,57 @@ public partial class FileRepository
}
// Check states
if (filter != null && !filter.States.IsNullOrEmpty())
if (matched && filter != null && !filter.States.IsNullOrEmpty())
{
var stateFile = Path.Combine(d, STATE_FILE);
var convStates = CollectConversationStates(stateFile);
foreach (var pair in filter.States)
var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE);
var convStates = CollectConversationLatestStates(latestStateFile);
if (convStates.IsNullOrEmpty())
{
if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue;
var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key));
if (foundState == null)
matched = false;
}
else
{
foreach (var pair in filter.States)
{
matched = false;
break;
}
if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue;
if (!string.IsNullOrWhiteSpace(pair.Value))
{
var curValue = foundState.Values.LastOrDefault()?.Data;
matched = matched && pair.Value.IsEqualTo(curValue);
var components = pair.Key.Split(".").ToList();
var primaryKey = components[0];
if (convStates.TryGetValue(primaryKey, out var doc))
{
var elem = doc.RootElement.GetProperty("data");
if (components.Count < 2)
{
if (!string.IsNullOrWhiteSpace(pair.Value))
{
if (elem.ValueKind == JsonValueKind.Array)
{
matched = elem.EnumerateArray().Select(x => x.ToString()).Contains(pair.Value);
}
else if (elem.ValueKind == JsonValueKind.String)
{
matched = elem.GetString() == pair.Value;
}
else
{
matched = elem.GetRawText() == pair.Value;
}
}
}
else
{
var paths = components.Where((_, idx) => idx > 0);
var found = FindState(elem, paths, pair.Value);
matched = found != null;
}
}
else
{
matched = false;
}
if (!matched) break;
}
}
}
@ -575,8 +620,9 @@ public partial class FileRepository
// Handle truncated states
var refTime = dialogs.ElementAt(foundIdx).MetaData.CreatedTime;
var stateDir = Path.Combine(convDir, STATE_FILE);
var latestStateDir = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
var states = CollectConversationStates(stateDir);
isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime);
isSaved = HandleTruncatedStates(stateDir, latestStateDir, states, messageId, refTime);
// Handle truncated breakpoints
var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE);
@ -703,7 +749,7 @@ public partial class FileRepository
return isSaved;
}
private bool HandleTruncatedStates(string stateDir, List<StateKeyValue> states, string refMsgId, DateTime refTime)
private bool HandleTruncatedStates(string stateDir, string latestStateDir, List<StateKeyValue> states, string refMsgId, DateTime refTime)
{
var truncatedStates = new List<StateKeyValue>();
foreach (var state in states)
@ -724,6 +770,10 @@ public partial class FileRepository
}
var isSaved = SaveTruncatedStates(stateDir, truncatedStates);
if (isSaved)
{
SaveTruncatedLatestStates(latestStateDir, truncatedStates);
}
return isSaved;
}
@ -794,6 +844,17 @@ public partial class FileRepository
return true;
}
private bool SaveTruncatedLatestStates(string latestStateDir, List<StateKeyValue> states)
{
if (string.IsNullOrEmpty(latestStateDir) || states == null) return false;
if (!File.Exists(latestStateDir)) File.Create(latestStateDir);
var latestStates = BuildLatestStates(states);
var stateStr = JsonSerializer.Serialize(latestStates, _options);
File.WriteAllText(latestStateDir, stateStr);
return true;
}
private bool SaveTruncatedBreakpoints(string breakpointDir, List<ConversationBreakpoint> breakpoints)
{
if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false;
@ -804,22 +865,98 @@ public partial class FileRepository
return true;
}
private string? EncodeText(string? text)
private Dictionary<string, JsonDocument> CollectConversationLatestStates(string latestStateDir)
{
if (string.IsNullOrEmpty(text)) return text;
if (string.IsNullOrEmpty(latestStateDir) || !File.Exists(latestStateDir)) return [];
var bytes = Encoding.UTF8.GetBytes(text);
var encoded = Convert.ToBase64String(bytes);
return encoded;
var str = File.ReadAllText(latestStateDir);
var states = JsonSerializer.Deserialize<Dictionary<string, JsonDocument>>(str, _options);
return states ?? [];
}
private string? DecodeText(string? text)
private Dictionary<string, JsonDocument> BuildLatestStates(List<StateKeyValue> states)
{
if (string.IsNullOrEmpty(text)) return text;
var endNodes = new Dictionary<string, JsonDocument>();
foreach (var pair in states)
{
var value = pair.Values?.LastOrDefault();
if (value == null || !value.Active) continue;
var decoded = Convert.FromBase64String(text);
var origin = Encoding.UTF8.GetString(decoded);
return origin;
try
{
var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _options);
var json = JsonDocument.Parse(jsonStr);
endNodes[pair.Key] = json;
}
catch
{
var str = JsonSerializer.Serialize(new { Data = value.Data }, _options);
var json = JsonDocument.Parse(str);
endNodes[pair.Key] = json;
}
}
return endNodes;
}
private JsonElement? FindState(JsonElement? root, IEnumerable<string> paths, string? targetValue)
{
JsonElement? elem = null;
if (root == null || paths.IsNullOrEmpty())
{
return elem;
}
elem = root;
for (int i = 0; i < paths.Count(); i++)
{
var field = paths.ElementAt(i);
if (elem.Value.ValueKind == JsonValueKind.Array)
{
if (elem.Value.EnumerateArray().IsNullOrEmpty())
{
elem = null;
break;
}
else
{
foreach (var item in elem.Value.EnumerateArray())
{
var subPaths = paths.Where((_, idx) => idx >= i);
elem = FindState(item, subPaths, targetValue);
if (elem != null)
{
return elem;
}
}
}
}
else if (elem.Value.TryGetProperty(field, out var prop))
{
elem = prop;
}
}
if (elem != null && !string.IsNullOrWhiteSpace(targetValue))
{
if (elem.Value.ValueKind == JsonValueKind.Array)
{
var isInArray = elem.Value.EnumerateArray().Select(x => x.ToString()).Contains(targetValue);
return isInArray ? elem : null;
}
else if ((elem.Value.ValueKind == JsonValueKind.String && elem.Value.GetString() == targetValue)
|| (elem.Value.ValueKind != JsonValueKind.String && elem.Value.GetRawText() == targetValue))
{
return elem;
}
else
{
return null;
}
}
return elem;
}
#endregion
}

35
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
using System.IO;
@ -122,6 +123,40 @@ namespace BotSharp.Core.Repository
}
#endregion
#region Instruction Log
public bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
{
if (logs.IsNullOrEmpty()) return false;
var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER);
if (!Directory.Exists(baseDir))
{
Directory.CreateDirectory(baseDir);
}
foreach (var log in logs)
{
var file = Path.Combine(baseDir, $"{Guid.NewGuid()}.log");
var text = JsonSerializer.Serialize(log, _options);
File.WriteAllText(file, text);
}
return true;
}
public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
{
if (filter == null)
{
filter = InstructLogFilter.Empty();
}
return new PagedItems<InstructionLogModel>
{
};
}
#endregion
#region Private methods
private int GetNextLogIndex(string logDir, string id)
{

3
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
View file

@ -6,7 +6,6 @@ using BotSharp.Abstraction.Users.Models;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Tasks.Models;
namespace BotSharp.Core.Repository;
public partial class FileRepository : IBotSharpRepository
@ -34,6 +33,7 @@ public partial class FileRepository : IBotSharpRepository
private const string DIALOG_FILE = "dialogs.json";
private const string STATE_FILE = "state.json";
private const string BREAKPOINT_FILE = "breakpoint.json";
private const string CONV_LATEST_STATE_FILE = "latest-state.json";
private const string TRANSLATION_MEMORY_FILE = "memory.json";
private const string USERS_FOLDER = "users";
@ -54,6 +54,7 @@ public partial class FileRepository : IBotSharpRepository
private const string STATS_FILE = "stats.json";
private const string CRON_FILE = "cron.json";
private const string INSTRUCTION_LOG_FOLDER = "instruction-logs";
public FileRepository(
IServiceProvider services,

1
src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs
View file

@ -15,6 +15,7 @@ public static class BotSharpLoggerExtensions
services.AddScoped<IContentGeneratingHook, VerboseLogHook>();
services.AddScoped<IContentGeneratingHook, GlobalStatsConversationHook>();
services.AddScoped<IConversationHook, RateLimitConversationHook>();
services.AddScoped<IInstructHook, InstructionLogHook>();
return services;
}
}

96
src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs Normal file
View file

@ -0,0 +1,96 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Users;
using System.Text.Json;
namespace BotSharp.Logger.Hooks;
public class InstructionLogHook : InstructHookBase
{
private readonly IServiceProvider _services;
private readonly ILogger<InstructionLogHook> _logger;
private readonly BotSharpOptions _options;
private readonly IUserIdentity _user;
public InstructionLogHook(
IServiceProvider services,
ILogger<InstructionLogHook> logger,
IUserIdentity user,
BotSharpOptions options)
{
_services = services;
_logger = logger;
_user = user;
_options = options;
}
public override async Task OnResponseGenerated(InstructResponseModel response)
{
if (response == null) return;
var db = _services.GetRequiredService<IBotSharpRepository>();
var state = _services.GetRequiredService<IConversationStateService>();
var user = db.GetUserById(_user.Id);
db.SaveInstructionLogs(new List<InstructionLogModel>
{
new InstructionLogModel
{
AgentId = response.AgentId,
Provider = response.Provider,
Model = response.Model,
States = state.GetStates(),
UserId = user?.Id
}
});
return;
}
private Dictionary<string, string> CollectStates()
{
var res = new Dictionary<string, object>();
var state = _services.GetRequiredService<IConversationStateService>();
var curStates = state.GetStates();
curStates["test"] = JsonSerializer.Serialize(new
{
Number = "789",
Dummy = new
{
Id = 123,
Name = "name",
Score = 12.123
},
Items = new List<object>
{
new
{
Name = "image",
Label = "before-service",
Attribute = new
{
Location = "Chicago",
Time = "afternoon"
}
},
new
{
Name = "pdf",
Label = "after-service",
Attribute = new
{
Location = "New York",
Time = "morning"
}
},
},
Lists = new List<object>
{
"abc",
"bcd"
}
}, _options.JsonSerializerOptions);
return curStates;
}
}

1
src/Infrastructure/BotSharp.Logger/Using.cs
View file

@ -12,4 +12,5 @@ global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Conversations.Settings;
global using BotSharp.Abstraction.Instructs;
global using BotSharp.Logger.Hooks;

12
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -75,6 +75,18 @@ public class InstructModeController : ControllerBase
{
new RoleDialogModel(AgentRole.User, input.Text)
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = input.AgentId,
Provider = input.Provider,
Model = input.Model
});
}
return message.Content;
}
#endregion

1
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs
View file

@ -14,4 +14,5 @@ public class ConversationDocument : MongoBase
public List<string> Tags { get; set; } = [];
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
public Dictionary<string, BsonDocument> LatestStates { get; set; } = new();
}

38
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/InstructionLogBetaDocument.cs Normal file
View file

@ -0,0 +1,38 @@
using BotSharp.Abstraction.Loggers.Models;
using System.Text.Json;
namespace BotSharp.Plugin.MongoStorage.Collections;
public class InstructionLogBetaDocument : MongoBase
{
public string? AgentId { get; set; }
public string Provider { get; set; } = default!;
public string Model { get; set; } = default!;
public string? UserId { get; set; }
public Dictionary<string, BsonDocument> States { get; set; } = new();
public DateTime CreatedTime { get; set; }
public static InstructionLogBetaDocument ToMongoModel(InstructionLogModel log)
{
return new InstructionLogBetaDocument
{
AgentId = log.AgentId,
Provider = log.Provider,
Model = log.Model,
UserId = log.UserId,
CreatedTime = log.CreatedTime
};
}
public static InstructionLogModel ToDomainModel(InstructionLogBetaDocument log)
{
return new InstructionLogModel
{
AgentId = log.AgentId,
Provider = log.Provider,
Model = log.Model,
UserId = log.UserId,
CreatedTime = log.CreatedTime
};
}
}

2
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -193,4 +193,6 @@ public class MongoDbContext
public IMongoCollection<GlobalStatisticsDocument> GlobalStatistics
=> GetCollectionOrCreate<GlobalStatisticsDocument>("GlobalStatistics");
public IMongoCollection<InstructionLogBetaDocument> InstructionLogs
=> GetCollectionOrCreate<InstructionLogBetaDocument>("InstructionLogsBeta");
}

93
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories.Filters;
using System.Text.Json;
namespace BotSharp.Plugin.MongoStorage.Repository;
@ -23,7 +24,8 @@ public partial class MongoRepository
Status = conversation.Status,
Tags = conversation.Tags ?? new(),
CreatedTime = utcNow,
UpdatedTime = utcNow
UpdatedTime = utcNow,
LatestStates = []
};
var dialogDoc = new ConversationDialogDocument
@ -72,8 +74,9 @@ public partial class MongoRepository
var cronDeleted = _dc.CrontabItems.DeleteMany(conbTabItems);
var convDeleted = _dc.Conversations.DeleteMany(filterConv);
return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0
|| contentLogDeleted.DeletedCount > 0 || stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0;
return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0
|| promptLogDeleted.DeletedCount > 0 || contentLogDeleted.DeletedCount > 0
|| stateLogDeleted.DeletedCount > 0 || convDeleted.DeletedCount > 0;
}
[SideCar]
@ -266,6 +269,14 @@ public partial class MongoRepository
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.ConversationStates.UpdateOne(filterStates, updateStates);
// Update latest states
var endNodes = BuildLatestStates(saveStates);
var filter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var update = Builders<ConversationDocument>.Update.Set(x => x.LatestStates, endNodes)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.Conversations.UpdateOne(filter, update);
}
public void UpdateConversationStatus(string conversationId, string status)
@ -371,21 +382,47 @@ public partial class MongoRepository
}
// Filter states
var stateFilters = new List<FilterDefinition<ConversationStateDocument>>();
if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty())
{
foreach (var pair in filter.States)
{
var elementFilters = new List<FilterDefinition<StateMongoElement>> { Builders<StateMongoElement>.Filter.Eq(x => x.Key, pair.Key) };
if (!string.IsNullOrEmpty(pair.Value))
{
elementFilters.Add(Builders<StateMongoElement>.Filter.Eq("Values.Data", pair.Value));
}
stateFilters.Add(Builders<ConversationStateDocument>.Filter.ElemMatch(x => x.States, Builders<StateMongoElement>.Filter.And(elementFilters)));
}
if (string.IsNullOrWhiteSpace(pair.Key)) continue;
var targetConvIds = _dc.ConversationStates.Find(Builders<ConversationStateDocument>.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList();
convFilters.Add(convBuilder.In(x => x.Id, targetConvIds));
// Format key
var keys = pair.Key.Split(".").ToList();
keys.Insert(1, "data");
keys.Insert(0, "LatestStates");
var formattedKey = string.Join(".", keys);
if (string.IsNullOrWhiteSpace(pair.Value))
{
convFilters.Add(convBuilder.Exists(formattedKey));
}
else if (bool.TryParse(pair.Value, out var boolValue))
{
convFilters.Add(convBuilder.Eq(formattedKey, boolValue));
}
else if (int.TryParse(pair.Value, out var intValue))
{
convFilters.Add(convBuilder.Eq(formattedKey, intValue));
}
else if (decimal.TryParse(pair.Value, out var decimalValue))
{
convFilters.Add(convBuilder.Eq(formattedKey, decimalValue));
}
else if (float.TryParse(pair.Value, out var floatValue))
{
convFilters.Add(convBuilder.Eq(formattedKey, floatValue));
}
else if (double.TryParse(pair.Value, out var doubleValue))
{
convFilters.Add(convBuilder.Eq(formattedKey, doubleValue));
}
else
{
convFilters.Add(convBuilder.Eq(formattedKey, pair.Value));
}
}
}
// Sort and paginate
@ -527,6 +564,7 @@ public partial class MongoRepository
var stateFilter = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault();
var endNodes = new Dictionary<string, BsonDocument>();
if (foundStates != null)
{
// Truncate states
@ -550,6 +588,7 @@ public partial class MongoRepository
truncatedStates.Add(state);
}
foundStates.States = truncatedStates;
endNodes = BuildLatestStates(truncatedStates);
}
// Truncate breakpoints
@ -573,6 +612,7 @@ public partial class MongoRepository
// Update conversation
var convFilter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow)
.Set(x => x.LatestStates, endNodes)
.Set(x => x.DialogCount, truncatedDialogs.Count);
_dc.Conversations.UpdateOne(convFilter, updateConv);
@ -621,8 +661,6 @@ public partial class MongoRepository
return keys;
}
private string ConvertSnakeCaseToPascalCase(string snakeCase)
{
string[] words = snakeCase.Split('_');
@ -640,4 +678,29 @@ public partial class MongoRepository
return pascalCase.ToString();
}
private Dictionary<string, BsonDocument> BuildLatestStates(List<StateMongoElement> states)
{
var endNodes = new Dictionary<string, BsonDocument>();
foreach (var pair in states)
{
var value = pair.Values?.LastOrDefault();
if (value == null || !value.Active) continue;
try
{
var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(value.Data) }, _botSharpOptions.JsonSerializerOptions);
var json = BsonDocument.Parse(jsonStr);
endNodes[pair.Key] = json;
}
catch
{
var str = JsonSerializer.Serialize(new { Data = value.Data }, _botSharpOptions.JsonSerializerOptions);
var json = BsonDocument.Parse(str);
endNodes[pair.Key] = json;
}
}
return endNodes;
}
}

100
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Loggers.Models;
using System.Text.Json;
namespace BotSharp.Plugin.MongoStorage.Repository;
@ -110,4 +112,102 @@ public partial class MongoRepository
return logs;
}
#endregion
#region Instruction Log
public bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
{
if (logs.IsNullOrEmpty()) return false;
var docs = new List<InstructionLogBetaDocument>();
foreach (var log in logs)
{
var doc = InstructionLogBetaDocument.ToMongoModel(log);
foreach (var pair in log.States)
{
try
{
var jsonStr = JsonSerializer.Serialize(new { Data = JsonDocument.Parse(pair.Value) }, _botSharpOptions.JsonSerializerOptions);
var json = BsonDocument.Parse(jsonStr);
doc.States[pair.Key] = json;
}
catch
{
var jsonStr = JsonSerializer.Serialize(new { Data = pair.Value }, _botSharpOptions.JsonSerializerOptions);
var json = BsonDocument.Parse(jsonStr);
doc.States[pair.Key] = json;
}
}
docs.Add(doc);
}
_dc.InstructionLogs.InsertMany(docs);
return true;
}
public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
{
if (filter == null)
{
filter = InstructLogFilter.Empty();
}
var builder = Builders<InstructionLogBetaDocument>.Filter;
var filters = new List<FilterDefinition<InstructionLogBetaDocument>>() { builder.Empty };
// Filter logs
if (!filter.AgentIds.IsNullOrEmpty())
{
filters.Add(builder.In(x => x.AgentId, filter.AgentIds));
}
if (!filter.Providers.IsNullOrEmpty())
{
filters.Add(builder.In(x => x.Provider, filter.Providers));
}
if (!filter.Models.IsNullOrEmpty())
{
filters.Add(builder.In(x => x.Model, filter.Models));
}
if (!filter.States.IsNullOrEmpty())
{
foreach (var pair in filter.States)
{
if (string.IsNullOrWhiteSpace(pair.Key)) continue;
// Format key
var keys = pair.Key.Split(".").ToList();
keys.Insert(1, "data");
keys.Insert(0, "States");
var formattedKey = string.Join(".", keys);
if (pair.Value == null)
{
filters.Add(builder.Exists(formattedKey));
}
else
{
filters.Add(builder.Eq(formattedKey, pair.Value));
}
}
}
var filterDef = builder.And(filters);
var sortDef = Builders<InstructionLogBetaDocument>.Sort.Descending(x => x.CreatedTime);
var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.InstructionLogs.CountDocuments(filterDef);
var logs = docs.Select(x =>
{
var log = InstructionLogBetaDocument.ToDomainModel(x);
log.States = x.States.ToDictionary(x => x.Key, x => x.Value.GetElement("data").Value.ToString() ?? string.Empty);
return log;
}).ToList();
return new PagedItems<InstructionLogModel>
{
Items = logs,
Count = (int)count
};
}
#endregion
}

6
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Options;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.MongoStorage.Repository;
@ -7,16 +8,19 @@ public partial class MongoRepository : IBotSharpRepository
private readonly MongoDbContext _dc;
private readonly IServiceProvider _services;
private readonly ILogger<MongoRepository> _logger;
private readonly BotSharpOptions _botSharpOptions;
private UpdateOptions _options;
public MongoRepository(
MongoDbContext dc,
IServiceProvider services,
ILogger<MongoRepository> logger)
ILogger<MongoRepository> logger,
BotSharpOptions botSharpOptions)
{
_dc = dc;
_services = services;
_logger = logger;
_botSharpOptions = botSharpOptions;
_options = new UpdateOptions
{
IsUpsert = true,