diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs index eeb39f73..d4e88991 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs @@ -8,11 +8,11 @@ namespace BotSharp.Abstraction.Conversations; public interface IConversationStateService { string GetConversationId(); - ConversationState Load(string conversationId); + Dictionary Load(string conversationId); string GetState(string name, string defaultValue = ""); bool ContainsState(string name); - ConversationState GetStates(); - IConversationStateService SetState(string name, T value); + Dictionary GetStates(); + IConversationStateService SetState(string name, T value, bool isNeedVersion = true); void SaveStateByArgs(JsonDocument args); void CleanState(); void Save(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index d98032fe..1903e913 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -13,7 +13,7 @@ public class Conversation public List Dialogs { get; set; } = new List(); [JsonIgnore] - public ConversationState States { get; set; } = new ConversationState(); + public Dictionary States { get; set; } = new Dictionary(); public string Status { get; set; } = ConversationStatus.Open; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs index f957f184..82cbcc02 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs @@ -1,22 +1,17 @@ namespace BotSharp.Abstraction.Conversations.Models; -public class ConversationState : Dictionary +public class ConversationState : Dictionary> { public ConversationState() { - + } public ConversationState(List pairs) { foreach (var pair in pairs) { - this[pair.Key] = pair.Value; + this[pair.Key] = pair.Values; } } - - public List ToKeyValueList() - { - return this.Select(x => new StateKeyValue(x.Key, x.Value)).ToList(); - } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs index ad02628f..3ad34ac0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs @@ -3,16 +3,27 @@ namespace BotSharp.Abstraction.Conversations.Models; public class StateKeyValue { public string Key { get; set; } - public string Value { get; set; } + public List Values { get; set; } = new List(); public StateKeyValue() { } - public StateKeyValue(string key, string value) + public StateKeyValue(string key, List values) { Key = key; - Value = value; + Values = values; } } + +public class StateValue +{ + public string Data { get; set; } + public DateTime UpdateTime { get; set; } + + public StateValue() + { + + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs index 6790d866..34b9f76d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs @@ -6,5 +6,5 @@ public class InstructResult : ITrackableMessage public string MessageId { get; set; } public string Text { get; set; } public object Data { get; set; } - public ConversationState States { get; set; } + public Dictionary States { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 1834dfd5..b8ca82a7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -33,7 +33,7 @@ public interface IBotSharpRepository List GetConversationDialogs(string conversationId); void UpdateConversationDialogElements(string conversationId, List updateElements); void AppendConversationDialogs(string conversationId, List dialogs); - List GetConversationStates(string conversationId); + ConversationState GetConversationStates(string conversationId); void UpdateConversationStates(string conversationId, List states); void UpdateConversationStatus(string conversationId, string status); Conversation GetConversation(string conversationId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs index 7a2f7dde..5375859d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.Abstraction.Repositories.Records; public class ConversationRecord : RecordBase @@ -18,9 +16,6 @@ public class ConversationRecord : RecordBase [JsonIgnore] public string Dialog { get; set; } - [JsonIgnore] - public List States { get; set; } - [Required] public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 23e589b1..0c106612 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -1,5 +1,5 @@ using BotSharp.Abstraction.Repositories; -using System.IO; +using System.Linq; namespace BotSharp.Core.Conversations.Services; @@ -11,60 +11,87 @@ public class ConversationStateService : IConversationStateService, IDisposable private readonly ILogger _logger; private readonly IServiceProvider _services; private ConversationState _states; - private BotSharpDatabaseSettings _dbSettings; private string _conversationId; private readonly IBotSharpRepository _db; - private List _savedStates; public ConversationStateService(ILogger logger, - IServiceProvider services, - BotSharpDatabaseSettings dbSettings, + IServiceProvider services, IBotSharpRepository db) { _logger = logger; _services = services; - _dbSettings = dbSettings; _db = db; _states = new ConversationState(); } public string GetConversationId() => _conversationId; - public IConversationStateService SetState(string name, T value) + + /// + /// Set conversation state + /// + /// + /// + /// + /// whether the state is related to message or not + /// + public IConversationStateService SetState(string name, T value, bool isNeedVersion = true) { if (value == null) { return this; } + var preValue = string.Empty; var currentValue = value.ToString(); var hooks = _services.GetServices(); - string preValue = _states.ContainsKey(name) ? _states[name] : ""; - if (!_states.ContainsKey(name) || _states[name] != currentValue) + + if (_states.TryGetValue(name, out var values)) + { + preValue = values?.LastOrDefault()?.Data ?? string.Empty; + } + + if (!_states.ContainsKey(name) || preValue != currentValue) { - _states[name] = currentValue; _logger.LogInformation($"[STATE] {name} = {value}"); foreach (var hook in hooks) { hook.OnStateChanged(name, preValue, currentValue).Wait(); } + + var stateValue = new StateValue + { + Data = currentValue, + UpdateTime = DateTime.UtcNow + }; + + if (!_states.ContainsKey(name) || !isNeedVersion) + { + _states[name] = new List { stateValue }; + } + else + { + _states[name].Add(stateValue); + } } return this; } - public ConversationState Load(string conversationId) + public Dictionary Load(string conversationId) { _conversationId = conversationId; - _savedStates = _db.GetConversationStates(_conversationId).ToList(); + _states = _db.GetConversationStates(_conversationId); + var curStates = new Dictionary(); - if (!_savedStates.IsNullOrEmpty()) + if (!_states.IsNullOrEmpty()) { - foreach (var data in _savedStates) + foreach (var state in _states) { - _states[data.Key] = data.Value; - _logger.LogInformation($"[STATE] {data.Key} : {data.Value}"); + var value = state.Value?.LastOrDefault()?.Data ?? string.Empty; + curStates[state.Key] = value; + _logger.LogInformation($"[STATE] {state.Key} : {value}"); } } @@ -75,7 +102,7 @@ public class ConversationStateService : IConversationStateService, IDisposable hook.OnStateLoaded(_states).Wait(); } - return _states; + return curStates; } public void Save() @@ -93,25 +120,32 @@ public class ConversationStateService : IConversationStateService, IDisposable } _db.UpdateConversationStates(_conversationId, states); - _logger.LogInformation($"Saved state {_conversationId}"); + _logger.LogInformation($"Saved states of conversation {_conversationId}"); } public void CleanState() { - //File.Delete(_file); + _states.Clear(); } - public ConversationState GetStates() - => _states; + public Dictionary GetStates() + { + var curStates = new Dictionary(); + foreach (var state in _states) + { + curStates[state.Key] = state.Value?.LastOrDefault()?.Data ?? string.Empty; + } + return curStates; + } public string GetState(string name, string defaultValue = "") { - if (!_states.ContainsKey(name)) + if (!_states.ContainsKey(name) || _states[name].IsNullOrEmpty()) { return defaultValue; } - return _states[name]; + return _states[name].Last().Data; } public void Dispose() @@ -121,7 +155,9 @@ public class ConversationStateService : IConversationStateService, IDisposable public bool ContainsState(string name) { - return _states.ContainsKey(name) && !string.IsNullOrEmpty(_states[name]); + return _states.ContainsKey(name) + && !_states[name].IsNullOrEmpty() + && !string.IsNullOrEmpty(_states[name].Last().Data); } public void SaveStateByArgs(JsonDocument args) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 62b52f53..1f824dd8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -47,14 +47,14 @@ public class TokenStatistics : ITokenStatistics // Accumulated Token var stat = _services.GetRequiredService(); var inputCount = int.Parse(stat.GetState("prompt_total", "0")); - stat.SetState("prompt_total", stats.PromptCount + inputCount); + stat.SetState("prompt_total", stats.PromptCount + inputCount, false); var outputCount = int.Parse(stat.GetState("completion_total", "0")); - stat.SetState("completion_total", stats.CompletionCount + outputCount); + stat.SetState("completion_total", stats.CompletionCount + outputCount, false); // Total cost var total_cost = float.Parse(stat.GetState("llm_total_cost", "0")); total_cost += Cost; - stat.SetState("llm_total_cost", total_cost); + stat.SetState("llm_total_cost", total_cost, false); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index ec8ac5f6..c3e5ad5e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -152,7 +152,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public List GetConversationStates(string conversationId) + public ConversationState GetConversationStates(string conversationId) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 0df9ed98..fb20db1f 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -9,6 +9,7 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Routing.Settings; using BotSharp.Abstraction.Evaluations.Settings; +using System.Text.Encodings.Web; namespace BotSharp.Core.Repository; @@ -20,6 +21,17 @@ public class FileRepository : IBotSharpRepository private readonly ConversationSetting _conversationSettings; private JsonSerializerOptions _options; + private const string AGENT_FILE = "agent.json"; + private const string AGENT_INSTRUCTION_FILE = "instruction"; + private const string AGENT_FUNCTIONS_FILE = "functions.json"; + private const string AGENT_SAMPLES_FILE = "samples.txt"; + private const string USER_FILE = "user.json"; + private const string USER_AGENT_FILE = "agents.json"; + private const string CONVERSATION_FILE = "conversation.json"; + private const string DIALOG_FILE = "dialogs.txt"; + private const string STATE_FILE = "state.json"; + private const string EXECUTION_LOG_FILE = "execution.log"; + public FileRepository( IServiceProvider services, BotSharpDatabaseSettings dbSettings, @@ -35,7 +47,9 @@ public class FileRepository : IBotSharpRepository { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true + WriteIndented = true, + AllowTrailingCommas = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; } @@ -59,7 +73,7 @@ public class FileRepository : IBotSharpRepository { foreach (var d in Directory.GetDirectories(dir)) { - var userFile = Path.Combine(d, "user.json"); + var userFile = Path.Combine(d, USER_FILE); if (!Directory.Exists(d) || !File.Exists(userFile)) continue; @@ -86,7 +100,7 @@ public class FileRepository : IBotSharpRepository { foreach (var d in Directory.GetDirectories(dir)) { - var file = Path.Combine(d, "agent.json"); + var file = Path.Combine(d, AGENT_FILE); if (!Directory.Exists(d) || !File.Exists(file)) continue; @@ -122,7 +136,7 @@ public class FileRepository : IBotSharpRepository { foreach (var d in Directory.GetDirectories(dir)) { - var file = Path.Combine(d, "agents.json"); + var file = Path.Combine(d, USER_AGENT_FILE); if (!Directory.Exists(d) || !File.Exists(file)) continue; @@ -171,7 +185,7 @@ public class FileRepository : IBotSharpRepository { Directory.CreateDirectory(dir); } - var path = Path.Combine(dir, "agent.json"); + var path = Path.Combine(dir, AGENT_FILE); File.WriteAllText(path, JsonSerializer.Serialize(agent, _options)); } } @@ -184,7 +198,7 @@ public class FileRepository : IBotSharpRepository { Directory.CreateDirectory(dir); } - var path = Path.Combine(dir, "user.json"); + var path = Path.Combine(dir, USER_FILE); File.WriteAllText(path, JsonSerializer.Serialize(user, _options)); } } @@ -198,7 +212,7 @@ public class FileRepository : IBotSharpRepository if (agents.Any()) { var dir = Path.Combine(_dbSettings.FileRepository, "users", uid); - var path = Path.Combine(dir, "agents.json"); + var path = Path.Combine(dir, USER_AGENT_FILE); File.WriteAllText(path, JsonSerializer.Serialize(agents, _options)); } }); @@ -357,7 +371,7 @@ public class FileRepository : IBotSharpRepository if (agent == null) return; var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, $"instruction.{_agentSettings.TemplateFormat}"); + agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); File.WriteAllText(instructionFile, instruction); } @@ -370,7 +384,7 @@ public class FileRepository : IBotSharpRepository if (agent == null) return; var functionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, "functions.json"); + agentId, AGENT_FUNCTIONS_FILE); var functionText = JsonSerializer.Serialize(inputFunctions, _options); File.WriteAllText(functionFile, functionText); @@ -436,7 +450,7 @@ public class FileRepository : IBotSharpRepository var (agent, agentFile) = GetAgentFromFile(agentId); if (agent == null) return; - var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "samples.txt"); + var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_SAMPLES_FILE); File.WriteAllLines(file, samples); } @@ -501,7 +515,7 @@ public class FileRepository : IBotSharpRepository if (!string.IsNullOrEmpty(dir)) { - var json = File.ReadAllText(Path.Combine(dir, "agent.json")); + var json = File.ReadAllText(Path.Combine(dir, AGENT_FILE)); if (string.IsNullOrEmpty(json)) return null; var record = JsonSerializer.Deserialize(json, _options); @@ -629,22 +643,31 @@ public class FileRepository : IBotSharpRepository Directory.CreateDirectory(dir); } - var convDir = Path.Combine(dir, "conversation.json"); - if (!File.Exists(convDir)) + var convFile = Path.Combine(dir, CONVERSATION_FILE); + if (!File.Exists(convFile)) { - File.WriteAllText(convDir, JsonSerializer.Serialize(conversation, _options)); + File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); } - var dialogDir = Path.Combine(dir, "dialogs.txt"); - if (!File.Exists(dialogDir)) + var dialogFile = Path.Combine(dir, DIALOG_FILE); + if (!File.Exists(dialogFile)) { - File.WriteAllText(dialogDir, string.Empty); + File.WriteAllText(dialogFile, string.Empty); } - var stateDir = Path.Combine(dir, "state.dict"); - if (!File.Exists(stateDir)) + var stateFile = Path.Combine(dir, STATE_FILE); + if (!File.Exists(stateFile)) { - File.WriteAllText(stateDir, string.Empty); + var states = conversation.States ?? new Dictionary(); + var initialStates = states.Select(x => new StateKeyValue + { + Key = x.Key, + Values = new List + { + new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow } + } + }).ToList(); + File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options)); } } @@ -665,7 +688,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var dialogDir = Path.Combine(convDir, "dialogs.txt"); + var dialogDir = Path.Combine(convDir, DIALOG_FILE); dialogs = CollectDialogElements(dialogDir); } @@ -680,7 +703,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var dialogDir = Path.Combine(convDir, "dialogs.txt"); + var dialogDir = Path.Combine(convDir, DIALOG_FILE); if (File.Exists(dialogDir)) { var updated = dialogElements.Select((x, idx) => @@ -704,22 +727,21 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var dialogDir = Path.Combine(convDir, "dialogs.txt"); + var dialogDir = Path.Combine(convDir, DIALOG_FILE); if (File.Exists(dialogDir)) { var texts = ParseDialogElements(dialogs); File.AppendAllLines(dialogDir, texts); } } - - return; } + public void UpdateConversationTitle(string conversationId, string title) { var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var convFile = Path.Combine(convDir, "conversation.json"); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); var content = File.ReadAllText(convFile); var record = JsonSerializer.Deserialize(content, _options); if (record != null) @@ -730,33 +752,32 @@ public class FileRepository : IBotSharpRepository } } } - public List GetConversationStates(string conversationId) + + public ConversationState GetConversationStates(string conversationId) { - var curStates = new List(); + var states = new List(); var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var stateDir = Path.Combine(convDir, "state.dict"); - curStates = CollectConversationStates(stateDir); + var stateFile = Path.Combine(convDir, STATE_FILE); + states = CollectConversationStates(stateFile); } - return curStates; + return new ConversationState(states); } public void UpdateConversationStates(string conversationId, List states) { - var localStates = new List(); + if (states.IsNullOrEmpty()) return; + var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var stateDir = Path.Combine(convDir, "state.dict"); - if (File.Exists(stateDir)) + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) { - foreach (var data in states) - { - localStates.Add($"{data.Key}={data.Value}"); - } - File.WriteAllLines(stateDir, localStates); + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); } } } @@ -766,7 +787,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var convFile = Path.Combine(convDir, "conversation.json"); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); if (File.Exists(convFile)) { var json = File.ReadAllText(convFile); @@ -783,21 +804,26 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (string.IsNullOrEmpty(convDir)) return null; - var convFile = Path.Combine(convDir, "conversation.json"); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); var content = File.ReadAllText(convFile); var record = JsonSerializer.Deserialize(content, _options); - var dialogFile = Path.Combine(convDir, "dialogs.txt"); + var dialogFile = Path.Combine(convDir, DIALOG_FILE); if (record != null) { record.Dialogs = CollectDialogElements(dialogFile); } - var stateFile = Path.Combine(convDir, "state.dict"); + var stateFile = Path.Combine(convDir, STATE_FILE); if (record != null) { var states = CollectConversationStates(stateFile); - record.States = new ConversationState(states); + var curStates = new Dictionary(); + states.ForEach(x => + { + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; + }); + record.States = curStates; } return record; @@ -810,7 +836,7 @@ public class FileRepository : IBotSharpRepository foreach (var d in Directory.GetDirectories(dir)) { - var path = Path.Combine(d, "conversation.json"); + var path = Path.Combine(d, CONVERSATION_FILE); if (!File.Exists(path)) continue; var json = File.ReadAllText(path); @@ -838,7 +864,7 @@ public class FileRepository : IBotSharpRepository foreach (var d in Directory.GetDirectories(dir)) { - var path = Path.Combine(d, "conversation.json"); + var path = Path.Combine(d, CONVERSATION_FILE); if (!File.Exists(path)) continue; var json = File.ReadAllText(path); @@ -889,7 +915,7 @@ public class FileRepository : IBotSharpRepository Directory.CreateDirectory(dir); } - var file = Path.Combine(dir, "execution.log"); + var file = Path.Combine(dir, EXECUTION_LOG_FILE); File.AppendAllLines(file, logs); } @@ -901,7 +927,7 @@ public class FileRepository : IBotSharpRepository var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); if (!Directory.Exists(dir)) return logs; - var file = Path.Combine(dir, "execution.log"); + var file = Path.Combine(dir, EXECUTION_LOG_FILE); logs = File.ReadAllLines(file)?.ToList() ?? new List(); return logs; } @@ -949,7 +975,7 @@ public class FileRepository : IBotSharpRepository private (Agent?, string) GetAgentFromFile(string agentId) { var dir = GetAgentDataDir(agentId); - var agentFile = Path.Combine(dir, "agent.json"); + var agentFile = Path.Combine(dir, AGENT_FILE); if (!File.Exists(agentFile)) return (null, string.Empty); var json = File.ReadAllText(agentFile); @@ -959,7 +985,7 @@ public class FileRepository : IBotSharpRepository private string FetchInstruction(string fileDir) { - var file = Path.Combine(fileDir, $"instruction.{_agentSettings.TemplateFormat}"); + var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); if (!File.Exists(file)) return string.Empty; var instruction = File.ReadAllText(file); @@ -968,7 +994,7 @@ public class FileRepository : IBotSharpRepository private List FetchFunctions(string fileDir) { - var file = Path.Combine(fileDir, "functions.json"); + var file = Path.Combine(fileDir, AGENT_FUNCTIONS_FILE); if (!File.Exists(file)) return new List(); var functionsJson = File.ReadAllText(file); @@ -978,7 +1004,7 @@ public class FileRepository : IBotSharpRepository private List FetchSamples(string fileDir) { - var file = Path.Combine(fileDir, "samples.txt"); + var file = Path.Combine(fileDir, AGENT_SAMPLES_FILE); if (!File.Exists(file)) return new List(); return File.ReadAllLines(file)?.ToList() ?? new List(); @@ -1082,18 +1108,16 @@ public class FileRepository : IBotSharpRepository return dialogTexts; } - private List CollectConversationStates(string stateDir) + private List CollectConversationStates(string stateFile) { var states = new List(); - if (!File.Exists(stateDir)) return states; + if (!File.Exists(stateFile)) return states; - var dict = File.ReadAllLines(stateDir); - foreach (var line in dict) - { - var data = line.Split('='); - states.Add(new StateKeyValue(data[0], data[1])); - } - return states; + var stateStr = File.ReadAllText(stateFile); + if (string.IsNullOrEmpty(stateStr)) return states; + + states = JsonSerializer.Deserialize>(stateStr, _options); + return states ?? new List(); } private int GetNextLlmCompletionLogIndex(string logDir, string id) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs index 556991ab..a863763e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs @@ -20,7 +20,7 @@ public class ConversationViewModel public string Channel { get; set; } = ConversationChannel.OpenAPI; public string Status { get; set; } - public ConversationState States { get; set; } + public Dictionary States { get; set; } [JsonPropertyName("updated_time")] public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index 4606974d..877d1c96 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Models; - namespace BotSharp.Plugin.MongoStorage.Collections; public class ConversationDocument : MongoBase @@ -9,7 +7,6 @@ public class ConversationDocument : MongoBase public string Title { get; set; } public string Channel { get; set; } public string Status { get; set; } - public List States { get; set; } public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs new file mode 100644 index 00000000..8f5edfa5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs @@ -0,0 +1,9 @@ +using BotSharp.Plugin.MongoStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationStateDocument : MongoBase +{ + public string ConversationId { get; set; } + public List States { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs index e90714cb..f04cb46c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs @@ -5,5 +5,5 @@ namespace BotSharp.Plugin.MongoStorage.Collections; public class LlmCompletionLogDocument : MongoBase { public string ConversationId { get; set; } - public List Logs { get; set; } + public List Logs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs rename to src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs index 91c5e4d1..ced9030d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; -public class PromptLogElement +public class PromptLogMongoElement { public string MessageId { get; set; } public string AgentId { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs new file mode 100644 index 00000000..03f0bda1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs @@ -0,0 +1,51 @@ +using BotSharp.Abstraction.Conversations.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class StateMongoElement +{ + public string Key { get; set; } + public List Values { get; set; } + + public static StateMongoElement ToMongoElement(StateKeyValue state) + { + return new StateMongoElement + { + Key = state.Key, + Values = state.Values?.Select(x => StateValueMongoElement.ToMongoElement(x))?.ToList() ?? new List() + }; + } + + public static StateKeyValue ToDomainElement(StateMongoElement state) + { + return new StateKeyValue + { + Key = state.Key, + Values = state.Values?.Select(x => StateValueMongoElement.ToDomainElement(x))?.ToList() ?? new List() + }; + } +} + +public class StateValueMongoElement +{ + public string Data { get; set; } + public DateTime UpdateTime { get; set; } + + public static StateValueMongoElement ToMongoElement(StateValue element) + { + return new StateValueMongoElement + { + Data = element.Data, + UpdateTime = element.UpdateTime + }; + } + + public static StateValue ToDomainElement(StateValueMongoElement element) + { + return new StateValue + { + Data = element.Data, + UpdateTime = element.UpdateTime + }; + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 11a1af33..1f5cbb92 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -37,6 +37,9 @@ public class MongoDbContext public IMongoCollection ConversationDialogs => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); + public IMongoCollection ConversationStates + => Database.GetCollection($"{_collectionPrefix}_ConversationStates"); + public IMongoCollection ExectionLogs => Database.GetCollection($"{_collectionPrefix}_ExecutionLogs"); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 4b94911d..3efc6cb3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -447,7 +447,7 @@ public class MongoRepository : IBotSharpRepository { query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower()); } - + if (filter.Disabled.HasValue) { query = query.Where(x => x.Disabled == filter.Disabled); @@ -514,9 +514,9 @@ public class MongoRepository : IBotSharpRepository public List GetAgentsByUser(string userId) { var agentIds = (from ua in _dc.UserAgents.AsQueryable() - join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id - where ua.UserId == userId || u.ExternalId == userId - select ua.AgentId).ToList(); + join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id + where ua.UserId == userId || u.ExternalId == userId + select ua.AgentId).ToList(); var filter = new AgentFilter { @@ -608,7 +608,7 @@ public class MongoRepository : IBotSharpRepository { return false; } - + } #endregion @@ -617,7 +617,7 @@ public class MongoRepository : IBotSharpRepository { if (conversation == null) return; - var conv = new ConversationDocument + var convDoc = new ConversationDocument { Id = !string.IsNullOrEmpty(conversation.Id) ? conversation.Id : Guid.NewGuid().ToString(), AgentId = conversation.AgentId, @@ -625,20 +625,37 @@ public class MongoRepository : IBotSharpRepository Title = conversation.Title, Channel = conversation.Channel, Status = conversation.Status, - States = conversation.States?.ToKeyValueList() ?? new List(), CreatedTime = DateTime.UtcNow, UpdatedTime = DateTime.UtcNow, }; - var dialog = new ConversationDialogDocument + var dialogDoc = new ConversationDialogDocument { Id = Guid.NewGuid().ToString(), - ConversationId = conv.Id, + ConversationId = convDoc.Id, Dialogs = new List() }; - _dc.Conversations.InsertOne(conv); - _dc.ConversationDialogs.InsertOne(dialog); + var states = conversation.States ?? new Dictionary(); + var initialStates = states.Select(x => new StateMongoElement + { + Key = x.Key, + Values = new List + { + new StateValueMongoElement { Data = x.Value, UpdateTime = DateTime.UtcNow } + } + }).ToList(); + + var stateDoc = new ConversationStateDocument + { + Id = Guid.NewGuid().ToString(), + ConversationId = convDoc.Id, + States = initialStates + }; + + _dc.Conversations.InsertOne(convDoc); + _dc.ConversationDialogs.InsertOne(dialogDoc); + _dc.ConversationStates.InsertOne(stateDoc); } public bool DeleteConversation(string conversationId) @@ -647,14 +664,16 @@ public class MongoRepository : IBotSharpRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); + var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; } @@ -688,7 +707,7 @@ public class MongoRepository : IBotSharpRepository } return x; }).ToList(); - + _dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog); } @@ -697,13 +716,7 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId)) return; var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault(); - if (foundConv == null) return; - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); - if (foundDialog == null) return; - var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -717,9 +730,6 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId)) return; var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault(); - if (foundConv == null) return; - var updateConv = Builders.Update .Set(x => x.UpdatedTime, DateTime.UtcNow) .Set(x => x.Title, title); @@ -727,30 +737,28 @@ public class MongoRepository : IBotSharpRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } - public List GetConversationStates(string conversationId) + public ConversationState GetConversationStates(string conversationId) { - var states = new List(); + var states = new ConversationState(); if (string.IsNullOrEmpty(conversationId)) return states; - var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConversation = _dc.Conversations.Find(filter).FirstOrDefault(); - var savedStates = foundConversation?.States ?? new List(); - return savedStates; + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(filter).FirstOrDefault(); + if (foundStates == null || foundStates.States.IsNullOrEmpty()) return states; + + var savedStates = foundStates.States.Select(x => StateMongoElement.ToDomainElement(x)).ToList(); + return new ConversationState(savedStates); } public void UpdateConversationStates(string conversationId, List states) { - if (string.IsNullOrEmpty(conversationId)) return; + if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; - var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filter).FirstOrDefault(); - if (foundConv == null) return; + var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); + var updateStates = Builders.Update.Set(x => x.States, saveStates); - var update = Builders.Update - .Set(x => x.States, states) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Conversations.UpdateOne(filter, update); + _dc.ConversationStates.UpdateOne(filterStates, updateStates); } public void UpdateConversationStatus(string conversationId, string status) @@ -758,9 +766,6 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(status)) return; var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filter).FirstOrDefault(); - if (foundConv == null) return; - var update = Builders.Update .Set(x => x.Status, status) .Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -774,14 +779,21 @@ public class MongoRepository : IBotSharpRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); var conv = _dc.Conversations.Find(filterConv).FirstOrDefault(); var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); + var states = _dc.ConversationStates.Find(filterState).FirstOrDefault(); if (conv == null) return null; var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); - + var curStates = new Dictionary(); + states.States.ForEach(x => + { + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; + }); + return new Conversation { Id = conv.Id.ToString(), @@ -791,7 +803,7 @@ public class MongoRepository : IBotSharpRepository Channel = conv.Channel, Status = conv.Status, Dialogs = dialogElements, - States = new ConversationState(conv.States ?? new List()), + States = curStates, CreatedTime = conv.CreatedTime, UpdatedTime = conv.UpdatedTime }; @@ -911,8 +923,8 @@ public class MongoRepository : IBotSharpRepository var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var update = Builders.Update - .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .PushEach(x => x.Logs, logs); + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) + .PushEach(x => x.Logs, logs); _dc.ExectionLogs.UpdateOne(filter, update, _options); } @@ -938,7 +950,7 @@ public class MongoRepository : IBotSharpRepository var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); - var logElement = new PromptLogElement + var logElement = new PromptLogMongoElement { MessageId = messageId, AgentId = log.AgentId,