From 010c92db1a1335945896ea1ae37732debf6556e9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Sun, 31 Dec 2023 22:56:10 -0600 Subject: [PATCH 1/9] refine conversation states --- .../IConversationStateService.cs | 2 +- .../Models/ConversationHistoryState.cs | 22 ++++++ .../Models/HistoryStateKeyValue.cs | 30 +++++++ .../Repositories/IBotSharpRepository.cs | 4 +- .../Services/ConversationService.cs | 2 +- .../Services/ConversationStateService.cs | 78 ++++++++++++++----- .../Repository/BotSharpDbContext.cs | 4 +- .../Repository/FileRepository.cs | 53 ++++++------- .../Controllers/ConversationController.cs | 10 +-- .../Controllers/InstructModeController.cs | 8 +- .../ChatbotUiController.cs | 10 +-- .../Collections/ConversationDocument.cs | 3 - .../Collections/ConversationStateDocument.cs | 9 +++ .../Collections/LlmCompletionLogDocument.cs | 2 +- ...LogElement.cs => PromptLogMongoElement.cs} | 2 +- .../Models/StateMongoElement.cs | 54 +++++++++++++ .../MongoDbContext.cs | 3 + .../Repository/MongoRepository.cs | 71 +++++++++++------ 18 files changed, 272 insertions(+), 95 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs rename src/Plugins/BotSharp.Plugin.MongoStorage/Models/{PromptLogElement.cs => PromptLogMongoElement.cs} (88%) create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs index eeb39f73..65570dc4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs @@ -12,7 +12,7 @@ public interface IConversationStateService string GetState(string name, string defaultValue = ""); bool ContainsState(string name); ConversationState GetStates(); - IConversationStateService SetState(string name, T value); + IConversationStateService SetState(string name, T value, bool isConst = false); void SaveStateByArgs(JsonDocument args); void CleanState(); void Save(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs new file mode 100644 index 00000000..224fe735 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs @@ -0,0 +1,22 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class ConversationHistoryState : Dictionary> +{ + public ConversationHistoryState() + { + + } + + public ConversationHistoryState(List pairs) + { + foreach (var pair in pairs) + { + this[pair.Key] = pair.Values; + } + } + + //public List ToKeyValueList() + //{ + // return this.Select(x => new StateKeyValue(x.Key, x.Value)).ToList(); + //} +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs new file mode 100644 index 00000000..2134f9e8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs @@ -0,0 +1,30 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class HistoryStateKeyValue +{ + public string Key { get; set; } + public List Values { get; set; } = new List(); + + public HistoryStateKeyValue() + { + + } + + public HistoryStateKeyValue(string key, List values) + { + Key = key; + Values = values; + } +} + +public class HistoryStateValue +{ + public string? MessageId { get; set; } + public string Data { get; set; } + public DateTime UpdateTime { get; set; } + + public HistoryStateValue() + { + + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 1834dfd5..860de96b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -33,8 +33,8 @@ public interface IBotSharpRepository List GetConversationDialogs(string conversationId); void UpdateConversationDialogElements(string conversationId, List updateElements); void AppendConversationDialogs(string conversationId, List dialogs); - List GetConversationStates(string conversationId); - void UpdateConversationStates(string conversationId, List states); + List GetConversationStates(string conversationId); + void UpdateConversationStates(string conversationId, List states); void UpdateConversationStatus(string conversationId, string status); Conversation GetConversation(string conversationId); List GetConversations(ConversationFilter filter); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index d6a15072..a39af217 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -120,6 +120,6 @@ public partial class ConversationService : IConversationService { _conversationId = conversationId; _state.Load(_conversationId); - states.ForEach(x => _state.SetState(x.Split('=')[0], x.Split('=')[1])); + states.ForEach(x => _state.SetState(x.Split('=')[0], x.Split('=')[1], true)); } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 23e589b1..1b560f43 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Repositories; -using System.IO; namespace BotSharp.Core.Conversations.Services; @@ -11,26 +10,33 @@ public class ConversationStateService : IConversationStateService, IDisposable private readonly ILogger _logger; private readonly IServiceProvider _services; private ConversationState _states; - private BotSharpDatabaseSettings _dbSettings; + private ConversationHistoryState _historyStates; private string _conversationId; private readonly IBotSharpRepository _db; - private List _savedStates; public ConversationStateService(ILogger logger, - IServiceProvider services, - BotSharpDatabaseSettings dbSettings, + IServiceProvider services, IBotSharpRepository db) { _logger = logger; _services = services; - _dbSettings = dbSettings; _db = db; _states = new ConversationState(); + _historyStates = new ConversationHistoryState(); } public string GetConversationId() => _conversationId; - public IConversationStateService SetState(string name, T value) + + /// + /// Set conversation state + /// + /// + /// + /// + /// whether the state is related to message or not + /// + public IConversationStateService SetState(string name, T value, bool isConst = false) { if (value == null) { @@ -40,6 +46,7 @@ public class ConversationStateService : IConversationStateService, IDisposable var currentValue = value.ToString(); var hooks = _services.GetServices(); string preValue = _states.ContainsKey(name) ? _states[name] : ""; + if (!_states.ContainsKey(name) || _states[name] != currentValue) { _states[name] = currentValue; @@ -48,6 +55,28 @@ public class ConversationStateService : IConversationStateService, IDisposable { hook.OnStateChanged(name, preValue, currentValue).Wait(); } + + var historyStateValue = new HistoryStateValue + { + Data = currentValue, + UpdateTime = DateTime.UtcNow + }; + + if (!_historyStates.ContainsKey(name)) + { + _historyStates[name] = new List(); + } + + if (isConst) + { + _historyStates[name] = new List { historyStateValue }; + } + else + { + var messageId = GetCurrentMessageId(); + historyStateValue.MessageId = messageId ?? string.Empty; + _historyStates[name].Add(historyStateValue); + } } return this; @@ -57,14 +86,16 @@ public class ConversationStateService : IConversationStateService, IDisposable { _conversationId = conversationId; - _savedStates = _db.GetConversationStates(_conversationId).ToList(); + var savedStates = _db.GetConversationStates(_conversationId).ToList(); + _historyStates = new ConversationHistoryState(savedStates); - if (!_savedStates.IsNullOrEmpty()) + if (!savedStates.IsNullOrEmpty()) { - foreach (var data in _savedStates) + foreach (var state in savedStates) { - _states[data.Key] = data.Value; - _logger.LogInformation($"[STATE] {data.Key} : {data.Value}"); + var value = state.Values.LastOrDefault()?.Data ?? string.Empty; + _states[state.Key] = value; + _logger.LogInformation($"[STATE] {state.Key} : {value}"); } } @@ -85,24 +116,23 @@ public class ConversationStateService : IConversationStateService, IDisposable return; } - var states = new List(); + var historyStates = new List(); - foreach (var dic in _states) + foreach (var dic in _historyStates) { - states.Add(new StateKeyValue(dic.Key, dic.Value)); + historyStates.Add(new HistoryStateKeyValue(dic.Key, dic.Value)); } - _db.UpdateConversationStates(_conversationId, states); - _logger.LogInformation($"Saved state {_conversationId}"); + _db.UpdateConversationStates(_conversationId, historyStates); + _logger.LogInformation($"Saved states of conversation {_conversationId}"); } public void CleanState() { - //File.Delete(_file); + } - public ConversationState GetStates() - => _states; + public ConversationState GetStates() => _states; public string GetState(string name, string defaultValue = "") { @@ -142,4 +172,12 @@ public class ConversationStateService : IConversationStateService, IDisposable } } } + + private string? GetCurrentMessageId() + { + if (string.IsNullOrEmpty(_conversationId)) return null; + + var dialogs = _db.GetConversationDialogs(_conversationId); + return dialogs.LastOrDefault()?.MetaData?.MessageId; + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index ec8ac5f6..901440d5 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -152,7 +152,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public List GetConversationStates(string conversationId) + public List GetConversationStates(string conversationId) { throw new NotImplementedException(); } @@ -165,7 +165,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository { throw new NotImplementedException(); } - public void UpdateConversationStates(string conversationId, List states) + public void UpdateConversationStates(string conversationId, List states) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 0df9ed98..34e0c8b0 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -644,7 +644,7 @@ public class FileRepository : IBotSharpRepository var stateDir = Path.Combine(dir, "state.dict"); if (!File.Exists(stateDir)) { - File.WriteAllText(stateDir, string.Empty); + File.WriteAllText(stateDir, "[]"); } } @@ -730,33 +730,31 @@ public class FileRepository : IBotSharpRepository } } } - public List GetConversationStates(string conversationId) + public List GetConversationStates(string conversationId) { - var curStates = new List(); + var curStates = new List(); var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var stateDir = Path.Combine(convDir, "state.dict"); - curStates = CollectConversationStates(stateDir); + var stateFile = Path.Combine(convDir, "state.dict"); + curStates = CollectConversationStates(stateFile); } return curStates; } - public void UpdateConversationStates(string conversationId, List states) + public void UpdateConversationStates(string conversationId, List states) { - var localStates = new List(); + if (states.IsNullOrEmpty()) return; + var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var stateDir = Path.Combine(convDir, "state.dict"); - if (File.Exists(stateDir)) + var stateFile = Path.Combine(convDir, "state.dict"); + if (File.Exists(stateFile)) { - foreach (var data in states) - { - localStates.Add($"{data.Key}={data.Value}"); - } - File.WriteAllLines(stateDir, localStates); + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); } } } @@ -796,8 +794,13 @@ public class FileRepository : IBotSharpRepository var stateFile = Path.Combine(convDir, "state.dict"); if (record != null) { - var states = CollectConversationStates(stateFile); - record.States = new ConversationState(states); + var historyStates = CollectConversationStates(stateFile); + var recentStates = historyStates.Select(x => new StateKeyValue + { + Key = x.Key, + Value = x.Values.LastOrDefault()?.Data ?? string.Empty + }).ToList(); + record.States = new ConversationState(recentStates); } return record; @@ -1082,18 +1085,16 @@ public class FileRepository : IBotSharpRepository return dialogTexts; } - private List CollectConversationStates(string stateDir) + private List CollectConversationStates(string stateFile) { - var states = new List(); - if (!File.Exists(stateDir)) return states; + var states = new List(); + if (!File.Exists(stateFile)) return states; - var dict = File.ReadAllLines(stateDir); - foreach (var line in dict) - { - var data = line.Split('='); - states.Add(new StateKeyValue(data[0], data[1])); - } - return states; + var stateStr = File.ReadAllText(stateFile); + if (string.IsNullOrEmpty(stateStr)) return states; + + states = JsonSerializer.Deserialize>(stateStr, _options); + return states ?? new List(); } private int GetNextLlmCompletionLogIndex(string logDir, string id) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 505f794f..4d5fed54 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -117,11 +117,11 @@ public class ConversationController : ControllerBase { var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, input.States); - conv.States.SetState("channel", input.Channel) - .SetState("provider", input.Provider) - .SetState("model", input.Model) - .SetState("temperature", input.Temperature) - .SetState("sampling_factor", input.SamplingFactor); + conv.States.SetState("channel", input.Channel, true) + .SetState("provider", input.Provider, true) + .SetState("model", input.Model, true) + .SetState("temperature", input.Temperature, true) + .SetState("sampling_factor", input.SamplingFactor, true); var response = new ChatResponseModel(); var inputMsg = new RoleDialogModel(AgentRole.User, input.Text); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 750719b8..5bd8e1c2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -22,10 +22,10 @@ public class InstructModeController : ControllerBase [FromBody] InstructMessageModel input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Split('=')[0], x.Split('=')[1])); - state.SetState("provider", input.Provider) - .SetState("model", input.Model) - .SetState("instruction", input.Instruction) + input.States.ForEach(x => state.SetState(x.Split('=')[0], x.Split('=')[1], true)); + state.SetState("provider", input.Provider, true) + .SetState("model", input.Model, true) + .SetState("instruction", input.Instruction, true) .SetState("input_text", input.Text); var instructor = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs index 9c422a7a..b6f3b0b2 100644 --- a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs +++ b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs @@ -75,11 +75,11 @@ public class ChatbotUiController : ControllerBase var conv = _services.GetRequiredService(); conv.SetConversationId(input.ConversationId, input.States); - conv.States.SetState("channel", input.Channel) - .SetState("provider", input.Provider) - .SetState("model", input.Model) - .SetState("temperature", input.Temperature) - .SetState("sampling_factor", input.SamplingFactor); + conv.States.SetState("channel", input.Channel, true) + .SetState("provider", input.Provider, true) + .SetState("model", input.Model, true) + .SetState("temperature", input.Temperature, true) + .SetState("sampling_factor", input.SamplingFactor, true); var result = await conv.SendMessage(input.AgentId, message, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index 4606974d..877d1c96 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Models; - namespace BotSharp.Plugin.MongoStorage.Collections; public class ConversationDocument : MongoBase @@ -9,7 +7,6 @@ public class ConversationDocument : MongoBase public string Title { get; set; } public string Channel { get; set; } public string Status { get; set; } - public List States { get; set; } public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs new file mode 100644 index 00000000..8f5edfa5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs @@ -0,0 +1,9 @@ +using BotSharp.Plugin.MongoStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class ConversationStateDocument : MongoBase +{ + public string ConversationId { get; set; } + public List States { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs index e90714cb..f04cb46c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs @@ -5,5 +5,5 @@ namespace BotSharp.Plugin.MongoStorage.Collections; public class LlmCompletionLogDocument : MongoBase { public string ConversationId { get; set; } - public List Logs { get; set; } + public List Logs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs rename to src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs index 91c5e4d1..ced9030d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; -public class PromptLogElement +public class PromptLogMongoElement { public string MessageId { get; set; } public string AgentId { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs new file mode 100644 index 00000000..4053d2e7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs @@ -0,0 +1,54 @@ +using BotSharp.Abstraction.Conversations.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class StateMongoElement +{ + public string Key { get; set; } + public List Values { get; set; } + + public static StateMongoElement ToMongoElement(HistoryStateKeyValue state) + { + return new StateMongoElement + { + Key = state.Key, + Values = state.Values?.Select(x => StateValueMongoElement.ToMongoElement(x))?.ToList() ?? new List() + }; + } + + public static HistoryStateKeyValue ToDomainElement(StateMongoElement state) + { + return new HistoryStateKeyValue + { + Key = state.Key, + Values = state.Values?.Select(x => StateValueMongoElement.ToDomainElement(x))?.ToList() ?? new List() + }; + } +} + +public class StateValueMongoElement +{ + public string? MessageId { get; set; } + public string Data { get; set; } + public DateTime UpdateTime { get; set; } + + public static StateValueMongoElement ToMongoElement(HistoryStateValue element) + { + return new StateValueMongoElement + { + MessageId = element.MessageId, + Data = element.Data, + UpdateTime = element.UpdateTime + }; + } + + public static HistoryStateValue ToDomainElement(StateValueMongoElement element) + { + return new HistoryStateValue + { + MessageId = element.MessageId, + Data = element.Data, + UpdateTime = element.UpdateTime + }; + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 11a1af33..1f5cbb92 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -37,6 +37,9 @@ public class MongoDbContext public IMongoCollection ConversationDialogs => Database.GetCollection($"{_collectionPrefix}_ConversationDialogs"); + public IMongoCollection ConversationStates + => Database.GetCollection($"{_collectionPrefix}_ConversationStates"); + public IMongoCollection ExectionLogs => Database.GetCollection($"{_collectionPrefix}_ExecutionLogs"); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 4b94911d..005f2830 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -447,7 +447,7 @@ public class MongoRepository : IBotSharpRepository { query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower()); } - + if (filter.Disabled.HasValue) { query = query.Where(x => x.Disabled == filter.Disabled); @@ -514,9 +514,9 @@ public class MongoRepository : IBotSharpRepository public List GetAgentsByUser(string userId) { var agentIds = (from ua in _dc.UserAgents.AsQueryable() - join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id - where ua.UserId == userId || u.ExternalId == userId - select ua.AgentId).ToList(); + join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id + where ua.UserId == userId || u.ExternalId == userId + select ua.AgentId).ToList(); var filter = new AgentFilter { @@ -608,7 +608,7 @@ public class MongoRepository : IBotSharpRepository { return false; } - + } #endregion @@ -625,7 +625,6 @@ public class MongoRepository : IBotSharpRepository Title = conversation.Title, Channel = conversation.Channel, Status = conversation.Status, - States = conversation.States?.ToKeyValueList() ?? new List(), CreatedTime = DateTime.UtcNow, UpdatedTime = DateTime.UtcNow, }; @@ -637,8 +636,16 @@ public class MongoRepository : IBotSharpRepository Dialogs = new List() }; + var states = new ConversationStateDocument + { + Id = Guid.NewGuid().ToString(), + ConversationId = conv.Id, + States = new List() + }; + _dc.Conversations.InsertOne(conv); _dc.ConversationDialogs.InsertOne(dialog); + _dc.ConversationStates.InsertOne(states); } public bool DeleteConversation(string conversationId) @@ -647,14 +654,16 @@ public class MongoRepository : IBotSharpRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); + var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; } @@ -688,7 +697,7 @@ public class MongoRepository : IBotSharpRepository } return x; }).ToList(); - + _dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog); } @@ -727,30 +736,37 @@ public class MongoRepository : IBotSharpRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } - public List GetConversationStates(string conversationId) + public List GetConversationStates(string conversationId) { - var states = new List(); + var states = new List(); if (string.IsNullOrEmpty(conversationId)) return states; - var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConversation = _dc.Conversations.Find(filter).FirstOrDefault(); - var savedStates = foundConversation?.States ?? new List(); + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(filter).FirstOrDefault(); + if (foundStates == null || foundStates.States.IsNullOrEmpty()) return states; + + var savedStates = foundStates.States.Select(x => StateMongoElement.ToDomainElement(x)).ToList(); return savedStates; } - public void UpdateConversationStates(string conversationId, List states) + public void UpdateConversationStates(string conversationId, List states) { - if (string.IsNullOrEmpty(conversationId)) return; + if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; - var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filter).FirstOrDefault(); + var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); + var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault(); if (foundConv == null) return; - var update = Builders.Update - .Set(x => x.States, states) - .Set(x => x.UpdatedTime, DateTime.UtcNow); + var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(filterStates).FirstOrDefault(); + if (foundStates == null) return; - _dc.Conversations.UpdateOne(filter, update); + var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); + var updateStates = Builders.Update.Set(x => x.States, saveStates); + var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.ConversationStates.UpdateOne(filterStates, updateStates); + _dc.Conversations.UpdateOne(filterConv, updateConv); } public void UpdateConversationStatus(string conversationId, string status) @@ -774,14 +790,21 @@ public class MongoRepository : IBotSharpRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); var conv = _dc.Conversations.Find(filterConv).FirstOrDefault(); var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); + var states = _dc.ConversationStates.Find(filterState).FirstOrDefault(); if (conv == null) return null; var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); - + var recentStates = states.States?.Select(x => new StateKeyValue + { + Key = x.Key, + Value = x.Values.LastOrDefault()?.Data ?? string.Empty + })?.ToList() ?? new List(); + return new Conversation { Id = conv.Id.ToString(), @@ -791,7 +814,7 @@ public class MongoRepository : IBotSharpRepository Channel = conv.Channel, Status = conv.Status, Dialogs = dialogElements, - States = new ConversationState(conv.States ?? new List()), + States = new ConversationState(recentStates), CreatedTime = conv.CreatedTime, UpdatedTime = conv.UpdatedTime }; @@ -938,7 +961,7 @@ public class MongoRepository : IBotSharpRepository var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); - var logElement = new PromptLogElement + var logElement = new PromptLogMongoElement { MessageId = messageId, AgentId = log.AgentId, From a534a93d35bd3faa711c4e948113f1e290bc8147 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jan 2024 13:07:42 -0600 Subject: [PATCH 2/9] remove is const --- .../Conversations/IConversationStateService.cs | 2 +- .../Conversations/Services/ConversationService.cs | 2 +- .../Services/ConversationStateService.cs | 14 +++----------- .../Controllers/ConversationController.cs | 10 +++++----- .../Controllers/InstructModeController.cs | 8 ++++---- .../ChatbotUiController.cs | 10 +++++----- 6 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs index 65570dc4..eeb39f73 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs @@ -12,7 +12,7 @@ public interface IConversationStateService string GetState(string name, string defaultValue = ""); bool ContainsState(string name); ConversationState GetStates(); - IConversationStateService SetState(string name, T value, bool isConst = false); + IConversationStateService SetState(string name, T value); void SaveStateByArgs(JsonDocument args); void CleanState(); void Save(); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index a39af217..d6a15072 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -120,6 +120,6 @@ public partial class ConversationService : IConversationService { _conversationId = conversationId; _state.Load(_conversationId); - states.ForEach(x => _state.SetState(x.Split('=')[0], x.Split('=')[1], true)); + states.ForEach(x => _state.SetState(x.Split('=')[0], x.Split('=')[1])); } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 1b560f43..1975e53d 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -36,7 +36,7 @@ public class ConversationStateService : IConversationStateService, IDisposable /// /// whether the state is related to message or not /// - public IConversationStateService SetState(string name, T value, bool isConst = false) + public IConversationStateService SetState(string name, T value) { if (value == null) { @@ -58,6 +58,7 @@ public class ConversationStateService : IConversationStateService, IDisposable var historyStateValue = new HistoryStateValue { + MessageId = GetCurrentMessageId(), Data = currentValue, UpdateTime = DateTime.UtcNow }; @@ -67,16 +68,7 @@ public class ConversationStateService : IConversationStateService, IDisposable _historyStates[name] = new List(); } - if (isConst) - { - _historyStates[name] = new List { historyStateValue }; - } - else - { - var messageId = GetCurrentMessageId(); - historyStateValue.MessageId = messageId ?? string.Empty; - _historyStates[name].Add(historyStateValue); - } + _historyStates[name].Add(historyStateValue); } return this; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 4d5fed54..505f794f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -117,11 +117,11 @@ public class ConversationController : ControllerBase { var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, input.States); - conv.States.SetState("channel", input.Channel, true) - .SetState("provider", input.Provider, true) - .SetState("model", input.Model, true) - .SetState("temperature", input.Temperature, true) - .SetState("sampling_factor", input.SamplingFactor, true); + conv.States.SetState("channel", input.Channel) + .SetState("provider", input.Provider) + .SetState("model", input.Model) + .SetState("temperature", input.Temperature) + .SetState("sampling_factor", input.SamplingFactor); var response = new ChatResponseModel(); var inputMsg = new RoleDialogModel(AgentRole.User, input.Text); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 5bd8e1c2..750719b8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -22,10 +22,10 @@ public class InstructModeController : ControllerBase [FromBody] InstructMessageModel input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Split('=')[0], x.Split('=')[1], true)); - state.SetState("provider", input.Provider, true) - .SetState("model", input.Model, true) - .SetState("instruction", input.Instruction, true) + input.States.ForEach(x => state.SetState(x.Split('=')[0], x.Split('=')[1])); + state.SetState("provider", input.Provider) + .SetState("model", input.Model) + .SetState("instruction", input.Instruction) .SetState("input_text", input.Text); var instructor = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs index b6f3b0b2..9c422a7a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs +++ b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs @@ -75,11 +75,11 @@ public class ChatbotUiController : ControllerBase var conv = _services.GetRequiredService(); conv.SetConversationId(input.ConversationId, input.States); - conv.States.SetState("channel", input.Channel, true) - .SetState("provider", input.Provider, true) - .SetState("model", input.Model, true) - .SetState("temperature", input.Temperature, true) - .SetState("sampling_factor", input.SamplingFactor, true); + conv.States.SetState("channel", input.Channel) + .SetState("provider", input.Provider) + .SetState("model", input.Model) + .SetState("temperature", input.Temperature) + .SetState("sampling_factor", input.SamplingFactor); var result = await conv.SendMessage(input.AgentId, message, From a523cf50479cee98a02d33c48012f43eaf4116fa Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jan 2024 13:53:05 -0600 Subject: [PATCH 3/9] remove message id in state history add isNeedVersion --- .../IConversationStateService.cs | 2 +- .../Models/HistoryStateKeyValue.cs | 1 - .../Services/ConversationStateService.cs | 23 +++++++------------ .../Conversations/Services/TokenStatistics.cs | 6 ++--- .../Models/StateMongoElement.cs | 3 --- 5 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs index eeb39f73..c6af0438 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs @@ -12,7 +12,7 @@ public interface IConversationStateService string GetState(string name, string defaultValue = ""); bool ContainsState(string name); ConversationState GetStates(); - IConversationStateService SetState(string name, T value); + IConversationStateService SetState(string name, T value, bool isNeedVersion = true); void SaveStateByArgs(JsonDocument args); void CleanState(); void Save(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs index 2134f9e8..e5368f42 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs @@ -19,7 +19,6 @@ public class HistoryStateKeyValue public class HistoryStateValue { - public string? MessageId { get; set; } public string Data { get; set; } public DateTime UpdateTime { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 1975e53d..fec0dcdb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -34,9 +34,9 @@ public class ConversationStateService : IConversationStateService, IDisposable /// /// /// - /// whether the state is related to message or not + /// whether the state is related to message or not /// - public IConversationStateService SetState(string name, T value) + public IConversationStateService SetState(string name, T value, bool isNeedVersion = true) { if (value == null) { @@ -58,17 +58,18 @@ public class ConversationStateService : IConversationStateService, IDisposable var historyStateValue = new HistoryStateValue { - MessageId = GetCurrentMessageId(), Data = currentValue, UpdateTime = DateTime.UtcNow }; - if (!_historyStates.ContainsKey(name)) + if (!_historyStates.ContainsKey(name) || !isNeedVersion) { - _historyStates[name] = new List(); + _historyStates[name] = new List { historyStateValue }; + } + else + { + _historyStates[name].Add(historyStateValue); } - - _historyStates[name].Add(historyStateValue); } return this; @@ -164,12 +165,4 @@ public class ConversationStateService : IConversationStateService, IDisposable } } } - - private string? GetCurrentMessageId() - { - if (string.IsNullOrEmpty(_conversationId)) return null; - - var dialogs = _db.GetConversationDialogs(_conversationId); - return dialogs.LastOrDefault()?.MetaData?.MessageId; - } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 62b52f53..1f824dd8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -47,14 +47,14 @@ public class TokenStatistics : ITokenStatistics // Accumulated Token var stat = _services.GetRequiredService(); var inputCount = int.Parse(stat.GetState("prompt_total", "0")); - stat.SetState("prompt_total", stats.PromptCount + inputCount); + stat.SetState("prompt_total", stats.PromptCount + inputCount, false); var outputCount = int.Parse(stat.GetState("completion_total", "0")); - stat.SetState("completion_total", stats.CompletionCount + outputCount); + stat.SetState("completion_total", stats.CompletionCount + outputCount, false); // Total cost var total_cost = float.Parse(stat.GetState("llm_total_cost", "0")); total_cost += Cost; - stat.SetState("llm_total_cost", total_cost); + stat.SetState("llm_total_cost", total_cost, false); } public void PrintStatistics() diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs index 4053d2e7..ee4228fd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs @@ -28,7 +28,6 @@ public class StateMongoElement public class StateValueMongoElement { - public string? MessageId { get; set; } public string Data { get; set; } public DateTime UpdateTime { get; set; } @@ -36,7 +35,6 @@ public class StateValueMongoElement { return new StateValueMongoElement { - MessageId = element.MessageId, Data = element.Data, UpdateTime = element.UpdateTime }; @@ -46,7 +44,6 @@ public class StateValueMongoElement { return new HistoryStateValue { - MessageId = element.MessageId, Data = element.Data, UpdateTime = element.UpdateTime }; From 9980ca3e1d98b148af480b5abe11c28378c1274a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jan 2024 13:56:56 -0600 Subject: [PATCH 4/9] clean code --- .../Conversations/Models/ConversationHistoryState.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs index 224fe735..2b8816a3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs @@ -14,9 +14,4 @@ public class ConversationHistoryState : Dictionary ToKeyValueList() - //{ - // return this.Select(x => new StateKeyValue(x.Key, x.Value)).ToList(); - //} } From 66578bf4f30c1a33e1b821afff9d7c2fd6ad2e95 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jan 2024 14:03:23 -0600 Subject: [PATCH 5/9] clean code --- .../Repository/MongoRepository.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 005f2830..649f9013 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -753,20 +753,14 @@ public class MongoRepository : IBotSharpRepository { if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; - var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault(); - if (foundConv == null) return; - var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var foundStates = _dc.ConversationStates.Find(filterStates).FirstOrDefault(); if (foundStates == null) return; var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); var updateStates = Builders.Update.Set(x => x.States, saveStates); - var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); - _dc.Conversations.UpdateOne(filterConv, updateConv); } public void UpdateConversationStatus(string conversationId, string status) From 2bde3d147d3a74bc6530c75295b58de212cfe717 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jan 2024 20:36:54 -0600 Subject: [PATCH 6/9] rename state file --- .../Repository/FileRepository.cs | 78 +++++++++++-------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 34e0c8b0..aa364f34 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -9,6 +9,7 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Routing.Settings; using BotSharp.Abstraction.Evaluations.Settings; +using System.Text.Encodings.Web; namespace BotSharp.Core.Repository; @@ -20,6 +21,17 @@ public class FileRepository : IBotSharpRepository private readonly ConversationSetting _conversationSettings; private JsonSerializerOptions _options; + private const string AGENT_FILE = "agent.json"; + private const string AGENT_INSTRUCTION_FILE = "instruction"; + private const string AGENT_FUNCTIONS_FILE = "functions.json"; + private const string AGENT_SAMPLES_FILE = "samples.txt"; + private const string USER_FILE = "user.json"; + private const string USER_AGENT_FILE = "agents.json"; + private const string CONVERSATION_FILE = "conversation.json"; + private const string DIALOG_FILE = "dialogs.txt"; + private const string STATE_FILE = "state.json"; + private const string EXECUTION_LOG_FILE = "execution.log"; + public FileRepository( IServiceProvider services, BotSharpDatabaseSettings dbSettings, @@ -35,7 +47,9 @@ public class FileRepository : IBotSharpRepository { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true + WriteIndented = true, + AllowTrailingCommas = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; } @@ -59,7 +73,7 @@ public class FileRepository : IBotSharpRepository { foreach (var d in Directory.GetDirectories(dir)) { - var userFile = Path.Combine(d, "user.json"); + var userFile = Path.Combine(d, USER_FILE); if (!Directory.Exists(d) || !File.Exists(userFile)) continue; @@ -86,7 +100,7 @@ public class FileRepository : IBotSharpRepository { foreach (var d in Directory.GetDirectories(dir)) { - var file = Path.Combine(d, "agent.json"); + var file = Path.Combine(d, AGENT_FILE); if (!Directory.Exists(d) || !File.Exists(file)) continue; @@ -122,7 +136,7 @@ public class FileRepository : IBotSharpRepository { foreach (var d in Directory.GetDirectories(dir)) { - var file = Path.Combine(d, "agents.json"); + var file = Path.Combine(d, USER_AGENT_FILE); if (!Directory.Exists(d) || !File.Exists(file)) continue; @@ -171,7 +185,7 @@ public class FileRepository : IBotSharpRepository { Directory.CreateDirectory(dir); } - var path = Path.Combine(dir, "agent.json"); + var path = Path.Combine(dir, AGENT_FILE); File.WriteAllText(path, JsonSerializer.Serialize(agent, _options)); } } @@ -184,7 +198,7 @@ public class FileRepository : IBotSharpRepository { Directory.CreateDirectory(dir); } - var path = Path.Combine(dir, "user.json"); + var path = Path.Combine(dir, USER_FILE); File.WriteAllText(path, JsonSerializer.Serialize(user, _options)); } } @@ -198,7 +212,7 @@ public class FileRepository : IBotSharpRepository if (agents.Any()) { var dir = Path.Combine(_dbSettings.FileRepository, "users", uid); - var path = Path.Combine(dir, "agents.json"); + var path = Path.Combine(dir, USER_AGENT_FILE); File.WriteAllText(path, JsonSerializer.Serialize(agents, _options)); } }); @@ -357,7 +371,7 @@ public class FileRepository : IBotSharpRepository if (agent == null) return; var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, $"instruction.{_agentSettings.TemplateFormat}"); + agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); File.WriteAllText(instructionFile, instruction); } @@ -370,7 +384,7 @@ public class FileRepository : IBotSharpRepository if (agent == null) return; var functionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, "functions.json"); + agentId, AGENT_FUNCTIONS_FILE); var functionText = JsonSerializer.Serialize(inputFunctions, _options); File.WriteAllText(functionFile, functionText); @@ -436,7 +450,7 @@ public class FileRepository : IBotSharpRepository var (agent, agentFile) = GetAgentFromFile(agentId); if (agent == null) return; - var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "samples.txt"); + var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_SAMPLES_FILE); File.WriteAllLines(file, samples); } @@ -501,7 +515,7 @@ public class FileRepository : IBotSharpRepository if (!string.IsNullOrEmpty(dir)) { - var json = File.ReadAllText(Path.Combine(dir, "agent.json")); + var json = File.ReadAllText(Path.Combine(dir, AGENT_FILE)); if (string.IsNullOrEmpty(json)) return null; var record = JsonSerializer.Deserialize(json, _options); @@ -629,19 +643,19 @@ public class FileRepository : IBotSharpRepository Directory.CreateDirectory(dir); } - var convDir = Path.Combine(dir, "conversation.json"); + var convDir = Path.Combine(dir, CONVERSATION_FILE); if (!File.Exists(convDir)) { File.WriteAllText(convDir, JsonSerializer.Serialize(conversation, _options)); } - var dialogDir = Path.Combine(dir, "dialogs.txt"); + var dialogDir = Path.Combine(dir, DIALOG_FILE); if (!File.Exists(dialogDir)) { File.WriteAllText(dialogDir, string.Empty); } - var stateDir = Path.Combine(dir, "state.dict"); + var stateDir = Path.Combine(dir, STATE_FILE); if (!File.Exists(stateDir)) { File.WriteAllText(stateDir, "[]"); @@ -665,7 +679,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var dialogDir = Path.Combine(convDir, "dialogs.txt"); + var dialogDir = Path.Combine(convDir, DIALOG_FILE); dialogs = CollectDialogElements(dialogDir); } @@ -680,7 +694,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var dialogDir = Path.Combine(convDir, "dialogs.txt"); + var dialogDir = Path.Combine(convDir, DIALOG_FILE); if (File.Exists(dialogDir)) { var updated = dialogElements.Select((x, idx) => @@ -704,7 +718,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var dialogDir = Path.Combine(convDir, "dialogs.txt"); + var dialogDir = Path.Combine(convDir, DIALOG_FILE); if (File.Exists(dialogDir)) { var texts = ParseDialogElements(dialogs); @@ -719,7 +733,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var convFile = Path.Combine(convDir, "conversation.json"); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); var content = File.ReadAllText(convFile); var record = JsonSerializer.Deserialize(content, _options); if (record != null) @@ -736,7 +750,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var stateFile = Path.Combine(convDir, "state.dict"); + var stateFile = Path.Combine(convDir, STATE_FILE); curStates = CollectConversationStates(stateFile); } @@ -750,7 +764,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var stateFile = Path.Combine(convDir, "state.dict"); + var stateFile = Path.Combine(convDir, STATE_FILE); if (File.Exists(stateFile)) { var stateStr = JsonSerializer.Serialize(states, _options); @@ -764,7 +778,7 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { - var convFile = Path.Combine(convDir, "conversation.json"); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); if (File.Exists(convFile)) { var json = File.ReadAllText(convFile); @@ -781,17 +795,17 @@ public class FileRepository : IBotSharpRepository var convDir = FindConversationDirectory(conversationId); if (string.IsNullOrEmpty(convDir)) return null; - var convFile = Path.Combine(convDir, "conversation.json"); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); var content = File.ReadAllText(convFile); var record = JsonSerializer.Deserialize(content, _options); - var dialogFile = Path.Combine(convDir, "dialogs.txt"); + var dialogFile = Path.Combine(convDir, DIALOG_FILE); if (record != null) { record.Dialogs = CollectDialogElements(dialogFile); } - var stateFile = Path.Combine(convDir, "state.dict"); + var stateFile = Path.Combine(convDir, STATE_FILE); if (record != null) { var historyStates = CollectConversationStates(stateFile); @@ -813,7 +827,7 @@ public class FileRepository : IBotSharpRepository foreach (var d in Directory.GetDirectories(dir)) { - var path = Path.Combine(d, "conversation.json"); + var path = Path.Combine(d, CONVERSATION_FILE); if (!File.Exists(path)) continue; var json = File.ReadAllText(path); @@ -841,7 +855,7 @@ public class FileRepository : IBotSharpRepository foreach (var d in Directory.GetDirectories(dir)) { - var path = Path.Combine(d, "conversation.json"); + var path = Path.Combine(d, CONVERSATION_FILE); if (!File.Exists(path)) continue; var json = File.ReadAllText(path); @@ -892,7 +906,7 @@ public class FileRepository : IBotSharpRepository Directory.CreateDirectory(dir); } - var file = Path.Combine(dir, "execution.log"); + var file = Path.Combine(dir, EXECUTION_LOG_FILE); File.AppendAllLines(file, logs); } @@ -904,7 +918,7 @@ public class FileRepository : IBotSharpRepository var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); if (!Directory.Exists(dir)) return logs; - var file = Path.Combine(dir, "execution.log"); + var file = Path.Combine(dir, EXECUTION_LOG_FILE); logs = File.ReadAllLines(file)?.ToList() ?? new List(); return logs; } @@ -952,7 +966,7 @@ public class FileRepository : IBotSharpRepository private (Agent?, string) GetAgentFromFile(string agentId) { var dir = GetAgentDataDir(agentId); - var agentFile = Path.Combine(dir, "agent.json"); + var agentFile = Path.Combine(dir, AGENT_FILE); if (!File.Exists(agentFile)) return (null, string.Empty); var json = File.ReadAllText(agentFile); @@ -962,7 +976,7 @@ public class FileRepository : IBotSharpRepository private string FetchInstruction(string fileDir) { - var file = Path.Combine(fileDir, $"instruction.{_agentSettings.TemplateFormat}"); + var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); if (!File.Exists(file)) return string.Empty; var instruction = File.ReadAllText(file); @@ -971,7 +985,7 @@ public class FileRepository : IBotSharpRepository private List FetchFunctions(string fileDir) { - var file = Path.Combine(fileDir, "functions.json"); + var file = Path.Combine(fileDir, AGENT_FUNCTIONS_FILE); if (!File.Exists(file)) return new List(); var functionsJson = File.ReadAllText(file); @@ -981,7 +995,7 @@ public class FileRepository : IBotSharpRepository private List FetchSamples(string fileDir) { - var file = Path.Combine(fileDir, "samples.txt"); + var file = Path.Combine(fileDir, AGENT_SAMPLES_FILE); if (!File.Exists(file)) return new List(); return File.ReadAllLines(file)?.ToList() ?? new List(); From f5a9f37dd4b7537f6f4a64e5cda9b9567ea6353d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 2 Jan 2024 11:40:08 -0600 Subject: [PATCH 7/9] union conversation states --- .../IConversationStateService.cs | 4 +- .../Conversations/Models/Conversation.cs | 2 +- .../Models/ConversationHistoryState.cs | 17 ------ .../Conversations/Models/ConversationState.cs | 11 +--- .../Models/HistoryStateKeyValue.cs | 29 --------- .../Conversations/Models/StateKeyValue.cs | 17 +++++- .../Instructs/Models/InstructResult.cs | 2 +- .../Repositories/IBotSharpRepository.cs | 4 +- .../Records/ConversationRecord.cs | 5 -- .../Services/ConversationStateService.cs | 58 +++++++++++------- .../Repository/BotSharpDbContext.cs | 4 +- .../Repository/FileRepository.cs | 61 +++++++++++-------- .../Controllers/AgentController.cs | 7 +++ .../Conversations/ConversationViewModel.cs | 2 +- .../Models/StateMongoElement.cs | 14 ++--- .../Repository/MongoRepository.cs | 44 +++++++------ 16 files changed, 138 insertions(+), 143 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs index c6af0438..d4e88991 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs @@ -8,10 +8,10 @@ namespace BotSharp.Abstraction.Conversations; public interface IConversationStateService { string GetConversationId(); - ConversationState Load(string conversationId); + Dictionary Load(string conversationId); string GetState(string name, string defaultValue = ""); bool ContainsState(string name); - ConversationState GetStates(); + Dictionary GetStates(); IConversationStateService SetState(string name, T value, bool isNeedVersion = true); void SaveStateByArgs(JsonDocument args); void CleanState(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index d98032fe..1903e913 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -13,7 +13,7 @@ public class Conversation public List Dialogs { get; set; } = new List(); [JsonIgnore] - public ConversationState States { get; set; } = new ConversationState(); + public Dictionary States { get; set; } = new Dictionary(); public string Status { get; set; } = ConversationStatus.Open; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs deleted file mode 100644 index 2b8816a3..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationHistoryState.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace BotSharp.Abstraction.Conversations.Models; - -public class ConversationHistoryState : Dictionary> -{ - public ConversationHistoryState() - { - - } - - public ConversationHistoryState(List pairs) - { - foreach (var pair in pairs) - { - this[pair.Key] = pair.Values; - } - } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs index f957f184..82cbcc02 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs @@ -1,22 +1,17 @@ namespace BotSharp.Abstraction.Conversations.Models; -public class ConversationState : Dictionary +public class ConversationState : Dictionary> { public ConversationState() { - + } public ConversationState(List pairs) { foreach (var pair in pairs) { - this[pair.Key] = pair.Value; + this[pair.Key] = pair.Values; } } - - public List ToKeyValueList() - { - return this.Select(x => new StateKeyValue(x.Key, x.Value)).ToList(); - } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs deleted file mode 100644 index e5368f42..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/HistoryStateKeyValue.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace BotSharp.Abstraction.Conversations.Models; - -public class HistoryStateKeyValue -{ - public string Key { get; set; } - public List Values { get; set; } = new List(); - - public HistoryStateKeyValue() - { - - } - - public HistoryStateKeyValue(string key, List values) - { - Key = key; - Values = values; - } -} - -public class HistoryStateValue -{ - public string Data { get; set; } - public DateTime UpdateTime { get; set; } - - public HistoryStateValue() - { - - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs index ad02628f..3ad34ac0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs @@ -3,16 +3,27 @@ namespace BotSharp.Abstraction.Conversations.Models; public class StateKeyValue { public string Key { get; set; } - public string Value { get; set; } + public List Values { get; set; } = new List(); public StateKeyValue() { } - public StateKeyValue(string key, string value) + public StateKeyValue(string key, List values) { Key = key; - Value = value; + Values = values; } } + +public class StateValue +{ + public string Data { get; set; } + public DateTime UpdateTime { get; set; } + + public StateValue() + { + + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs index 6790d866..34b9f76d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs @@ -6,5 +6,5 @@ public class InstructResult : ITrackableMessage public string MessageId { get; set; } public string Text { get; set; } public object Data { get; set; } - public ConversationState States { get; set; } + public Dictionary States { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 860de96b..1834dfd5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -33,8 +33,8 @@ public interface IBotSharpRepository List GetConversationDialogs(string conversationId); void UpdateConversationDialogElements(string conversationId, List updateElements); void AppendConversationDialogs(string conversationId, List dialogs); - List GetConversationStates(string conversationId); - void UpdateConversationStates(string conversationId, List states); + List GetConversationStates(string conversationId); + void UpdateConversationStates(string conversationId, List states); void UpdateConversationStatus(string conversationId, string status); Conversation GetConversation(string conversationId); List GetConversations(ConversationFilter filter); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs index 7a2f7dde..5375859d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.Abstraction.Repositories.Records; public class ConversationRecord : RecordBase @@ -18,9 +16,6 @@ public class ConversationRecord : RecordBase [JsonIgnore] public string Dialog { get; set; } - [JsonIgnore] - public List States { get; set; } - [Required] public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index fec0dcdb..a6c4642e 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Repositories; +using System.Linq; namespace BotSharp.Core.Conversations.Services; @@ -10,7 +11,6 @@ public class ConversationStateService : IConversationStateService, IDisposable private readonly ILogger _logger; private readonly IServiceProvider _services; private ConversationState _states; - private ConversationHistoryState _historyStates; private string _conversationId; private readonly IBotSharpRepository _db; @@ -22,7 +22,6 @@ public class ConversationStateService : IConversationStateService, IDisposable _services = services; _db = db; _states = new ConversationState(); - _historyStates = new ConversationHistoryState(); } public string GetConversationId() => _conversationId; @@ -43,51 +42,56 @@ public class ConversationStateService : IConversationStateService, IDisposable return this; } + var preValue = string.Empty; var currentValue = value.ToString(); var hooks = _services.GetServices(); - string preValue = _states.ContainsKey(name) ? _states[name] : ""; - if (!_states.ContainsKey(name) || _states[name] != currentValue) + if (_states.TryGetValue(name, out var values)) + { + preValue = values.LastOrDefault()?.Data ?? string.Empty; + } + + if (!_states.ContainsKey(name) || preValue != currentValue) { - _states[name] = currentValue; _logger.LogInformation($"[STATE] {name} = {value}"); foreach (var hook in hooks) { hook.OnStateChanged(name, preValue, currentValue).Wait(); } - var historyStateValue = new HistoryStateValue + var stateValue = new StateValue { Data = currentValue, UpdateTime = DateTime.UtcNow }; - if (!_historyStates.ContainsKey(name) || !isNeedVersion) + if (!_states.ContainsKey(name) || !isNeedVersion) { - _historyStates[name] = new List { historyStateValue }; + _states[name] = new List { stateValue }; } else { - _historyStates[name].Add(historyStateValue); + _states[name].Add(stateValue); } } return this; } - public ConversationState Load(string conversationId) + public Dictionary Load(string conversationId) { _conversationId = conversationId; var savedStates = _db.GetConversationStates(_conversationId).ToList(); - _historyStates = new ConversationHistoryState(savedStates); + _states = new ConversationState(savedStates); + var curStates = new Dictionary(); if (!savedStates.IsNullOrEmpty()) { foreach (var state in savedStates) { var value = state.Values.LastOrDefault()?.Data ?? string.Empty; - _states[state.Key] = value; + curStates[state.Key] = value; _logger.LogInformation($"[STATE] {state.Key} : {value}"); } } @@ -99,7 +103,7 @@ public class ConversationStateService : IConversationStateService, IDisposable hook.OnStateLoaded(_states).Wait(); } - return _states; + return curStates; } public void Save() @@ -109,32 +113,40 @@ public class ConversationStateService : IConversationStateService, IDisposable return; } - var historyStates = new List(); + var states = new List(); - foreach (var dic in _historyStates) + foreach (var dic in _states) { - historyStates.Add(new HistoryStateKeyValue(dic.Key, dic.Value)); + states.Add(new StateKeyValue(dic.Key, dic.Value)); } - _db.UpdateConversationStates(_conversationId, historyStates); + _db.UpdateConversationStates(_conversationId, states); _logger.LogInformation($"Saved states of conversation {_conversationId}"); } public void CleanState() { - + _states.Clear(); } - public ConversationState GetStates() => _states; + public Dictionary GetStates() + { + var curStates = new Dictionary(); + foreach (var state in _states) + { + curStates[state.Key] = state.Value.LastOrDefault()?.Data ?? string.Empty; + } + return curStates; + } public string GetState(string name, string defaultValue = "") { - if (!_states.ContainsKey(name)) + if (!_states.ContainsKey(name) || _states[name].IsNullOrEmpty()) { return defaultValue; } - return _states[name]; + return _states[name].Last().Data; } public void Dispose() @@ -144,7 +156,9 @@ public class ConversationStateService : IConversationStateService, IDisposable public bool ContainsState(string name) { - return _states.ContainsKey(name) && !string.IsNullOrEmpty(_states[name]); + return _states.ContainsKey(name) + && !_states[name].IsNullOrEmpty() + && !string.IsNullOrEmpty(_states[name].Last().Data); } public void SaveStateByArgs(JsonDocument args) diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 901440d5..ec8ac5f6 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -152,7 +152,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public List GetConversationStates(string conversationId) + public List GetConversationStates(string conversationId) { throw new NotImplementedException(); } @@ -165,7 +165,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository { throw new NotImplementedException(); } - public void UpdateConversationStates(string conversationId, List states) + public void UpdateConversationStates(string conversationId, List states) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index aa364f34..030ac91a 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -643,22 +643,31 @@ public class FileRepository : IBotSharpRepository Directory.CreateDirectory(dir); } - var convDir = Path.Combine(dir, CONVERSATION_FILE); - if (!File.Exists(convDir)) + var convFile = Path.Combine(dir, CONVERSATION_FILE); + if (!File.Exists(convFile)) { - File.WriteAllText(convDir, JsonSerializer.Serialize(conversation, _options)); + File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); } - var dialogDir = Path.Combine(dir, DIALOG_FILE); - if (!File.Exists(dialogDir)) + var dialogFile = Path.Combine(dir, DIALOG_FILE); + if (!File.Exists(dialogFile)) { - File.WriteAllText(dialogDir, string.Empty); + File.WriteAllText(dialogFile, string.Empty); } - var stateDir = Path.Combine(dir, STATE_FILE); - if (!File.Exists(stateDir)) + var stateFile = Path.Combine(dir, STATE_FILE); + if (!File.Exists(stateFile)) { - File.WriteAllText(stateDir, "[]"); + var states = conversation.States ?? new Dictionary(); + var initialStates = states.Select(x => new StateKeyValue + { + Key = x.Key, + Values = new List + { + new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow } + } + }).ToList(); + File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options)); } } @@ -725,9 +734,8 @@ public class FileRepository : IBotSharpRepository File.AppendAllLines(dialogDir, texts); } } - - return; } + public void UpdateConversationTitle(string conversationId, string title) { var convDir = FindConversationDirectory(conversationId); @@ -744,20 +752,21 @@ public class FileRepository : IBotSharpRepository } } } - public List GetConversationStates(string conversationId) + + public List GetConversationStates(string conversationId) { - var curStates = new List(); + var states = new List(); var convDir = FindConversationDirectory(conversationId); if (!string.IsNullOrEmpty(convDir)) { var stateFile = Path.Combine(convDir, STATE_FILE); - curStates = CollectConversationStates(stateFile); + states = CollectConversationStates(stateFile); } - return curStates; + return states; } - public void UpdateConversationStates(string conversationId, List states) + public void UpdateConversationStates(string conversationId, List states) { if (states.IsNullOrEmpty()) return; @@ -808,13 +817,13 @@ public class FileRepository : IBotSharpRepository var stateFile = Path.Combine(convDir, STATE_FILE); if (record != null) { - var historyStates = CollectConversationStates(stateFile); - var recentStates = historyStates.Select(x => new StateKeyValue + var states = CollectConversationStates(stateFile); + var curStates = new Dictionary(); + states.ForEach(x => { - Key = x.Key, - Value = x.Values.LastOrDefault()?.Data ?? string.Empty - }).ToList(); - record.States = new ConversationState(recentStates); + curStates[x.Key] = x.Values.LastOrDefault()?.Data ?? string.Empty; + }); + record.States = curStates; } return record; @@ -1099,16 +1108,16 @@ public class FileRepository : IBotSharpRepository return dialogTexts; } - private List CollectConversationStates(string stateFile) + private List CollectConversationStates(string stateFile) { - var states = new List(); + var states = new List(); if (!File.Exists(stateFile)) return states; var stateStr = File.ReadAllText(stateFile); if (string.IsNullOrEmpty(stateStr)) return states; - states = JsonSerializer.Deserialize>(stateStr, _options); - return states ?? new List(); + states = JsonSerializer.Deserialize>(stateStr, _options); + return states ?? new List(); } private int GetNextLlmCompletionLogIndex(string logDir, string id) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 36c48476..de1060dd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -15,6 +15,13 @@ public class AgentController : ControllerBase _services = services; } + [HttpGet("/test")] + public async Task Test() + { + + } + + [HttpGet("/agent/settings")] public AgentSettings GetSettings() { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs index 556991ab..a863763e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs @@ -20,7 +20,7 @@ public class ConversationViewModel public string Channel { get; set; } = ConversationChannel.OpenAPI; public string Status { get; set; } - public ConversationState States { get; set; } + public Dictionary States { get; set; } [JsonPropertyName("updated_time")] public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs index ee4228fd..03f0bda1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs @@ -7,7 +7,7 @@ public class StateMongoElement public string Key { get; set; } public List Values { get; set; } - public static StateMongoElement ToMongoElement(HistoryStateKeyValue state) + public static StateMongoElement ToMongoElement(StateKeyValue state) { return new StateMongoElement { @@ -16,12 +16,12 @@ public class StateMongoElement }; } - public static HistoryStateKeyValue ToDomainElement(StateMongoElement state) + public static StateKeyValue ToDomainElement(StateMongoElement state) { - return new HistoryStateKeyValue + return new StateKeyValue { Key = state.Key, - Values = state.Values?.Select(x => StateValueMongoElement.ToDomainElement(x))?.ToList() ?? new List() + Values = state.Values?.Select(x => StateValueMongoElement.ToDomainElement(x))?.ToList() ?? new List() }; } } @@ -31,7 +31,7 @@ public class StateValueMongoElement public string Data { get; set; } public DateTime UpdateTime { get; set; } - public static StateValueMongoElement ToMongoElement(HistoryStateValue element) + public static StateValueMongoElement ToMongoElement(StateValue element) { return new StateValueMongoElement { @@ -40,9 +40,9 @@ public class StateValueMongoElement }; } - public static HistoryStateValue ToDomainElement(StateValueMongoElement element) + public static StateValue ToDomainElement(StateValueMongoElement element) { - return new HistoryStateValue + return new StateValue { Data = element.Data, UpdateTime = element.UpdateTime diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 649f9013..7c57ec43 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -617,7 +617,7 @@ public class MongoRepository : IBotSharpRepository { if (conversation == null) return; - var conv = new ConversationDocument + var convDoc = new ConversationDocument { Id = !string.IsNullOrEmpty(conversation.Id) ? conversation.Id : Guid.NewGuid().ToString(), AgentId = conversation.AgentId, @@ -629,23 +629,33 @@ public class MongoRepository : IBotSharpRepository UpdatedTime = DateTime.UtcNow, }; - var dialog = new ConversationDialogDocument + var dialogDoc = new ConversationDialogDocument { Id = Guid.NewGuid().ToString(), - ConversationId = conv.Id, + ConversationId = convDoc.Id, Dialogs = new List() }; - var states = new ConversationStateDocument + var states = conversation.States ?? new Dictionary(); + var initialStates = states.Select(x => new StateMongoElement + { + Key = x.Key, + Values = new List + { + new StateValueMongoElement { Data = x.Value, UpdateTime = DateTime.UtcNow } + } + }).ToList(); + + var stateDoc = new ConversationStateDocument { Id = Guid.NewGuid().ToString(), - ConversationId = conv.Id, - States = new List() + ConversationId = convDoc.Id, + States = initialStates }; - _dc.Conversations.InsertOne(conv); - _dc.ConversationDialogs.InsertOne(dialog); - _dc.ConversationStates.InsertOne(states); + _dc.Conversations.InsertOne(convDoc); + _dc.ConversationDialogs.InsertOne(dialogDoc); + _dc.ConversationStates.InsertOne(stateDoc); } public bool DeleteConversation(string conversationId) @@ -736,9 +746,9 @@ public class MongoRepository : IBotSharpRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } - public List GetConversationStates(string conversationId) + public List GetConversationStates(string conversationId) { - var states = new List(); + var states = new List(); if (string.IsNullOrEmpty(conversationId)) return states; var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); @@ -749,7 +759,7 @@ public class MongoRepository : IBotSharpRepository return savedStates; } - public void UpdateConversationStates(string conversationId, List states) + public void UpdateConversationStates(string conversationId, List states) { if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; @@ -793,11 +803,11 @@ public class MongoRepository : IBotSharpRepository if (conv == null) return null; var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); - var recentStates = states.States?.Select(x => new StateKeyValue + var curStates = new Dictionary(); + states.States.ForEach(x => { - Key = x.Key, - Value = x.Values.LastOrDefault()?.Data ?? string.Empty - })?.ToList() ?? new List(); + curStates[x.Key] = x.Values.LastOrDefault()?.Data ?? string.Empty; + }); return new Conversation { @@ -808,7 +818,7 @@ public class MongoRepository : IBotSharpRepository Channel = conv.Channel, Status = conv.Status, Dialogs = dialogElements, - States = new ConversationState(recentStates), + States = curStates, CreatedTime = conv.CreatedTime, UpdatedTime = conv.UpdatedTime }; From 538be13bee4242381f23d61b0563d3e4b9ffc0b7 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 2 Jan 2024 11:50:07 -0600 Subject: [PATCH 8/9] refine null --- .../Repositories/IBotSharpRepository.cs | 2 +- .../Services/ConversationStateService.cs | 13 ++++++------- .../BotSharp.Core/Repository/BotSharpDbContext.cs | 2 +- .../BotSharp.Core/Repository/FileRepository.cs | 6 +++--- .../Repository/MongoRepository.cs | 8 ++++---- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 1834dfd5..b8ca82a7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -33,7 +33,7 @@ public interface IBotSharpRepository List GetConversationDialogs(string conversationId); void UpdateConversationDialogElements(string conversationId, List updateElements); void AppendConversationDialogs(string conversationId, List dialogs); - List GetConversationStates(string conversationId); + ConversationState GetConversationStates(string conversationId); void UpdateConversationStates(string conversationId, List states); void UpdateConversationStatus(string conversationId, string status); Conversation GetConversation(string conversationId); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index a6c4642e..0c106612 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -48,7 +48,7 @@ public class ConversationStateService : IConversationStateService, IDisposable if (_states.TryGetValue(name, out var values)) { - preValue = values.LastOrDefault()?.Data ?? string.Empty; + preValue = values?.LastOrDefault()?.Data ?? string.Empty; } if (!_states.ContainsKey(name) || preValue != currentValue) @@ -82,15 +82,14 @@ public class ConversationStateService : IConversationStateService, IDisposable { _conversationId = conversationId; - var savedStates = _db.GetConversationStates(_conversationId).ToList(); - _states = new ConversationState(savedStates); + _states = _db.GetConversationStates(_conversationId); var curStates = new Dictionary(); - if (!savedStates.IsNullOrEmpty()) + if (!_states.IsNullOrEmpty()) { - foreach (var state in savedStates) + foreach (var state in _states) { - var value = state.Values.LastOrDefault()?.Data ?? string.Empty; + var value = state.Value?.LastOrDefault()?.Data ?? string.Empty; curStates[state.Key] = value; _logger.LogInformation($"[STATE] {state.Key} : {value}"); } @@ -134,7 +133,7 @@ public class ConversationStateService : IConversationStateService, IDisposable var curStates = new Dictionary(); foreach (var state in _states) { - curStates[state.Key] = state.Value.LastOrDefault()?.Data ?? string.Empty; + curStates[state.Key] = state.Value?.LastOrDefault()?.Data ?? string.Empty; } return curStates; } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index ec8ac5f6..c3e5ad5e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -152,7 +152,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } - public List GetConversationStates(string conversationId) + public ConversationState GetConversationStates(string conversationId) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 030ac91a..fb20db1f 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -753,7 +753,7 @@ public class FileRepository : IBotSharpRepository } } - public List GetConversationStates(string conversationId) + public ConversationState GetConversationStates(string conversationId) { var states = new List(); var convDir = FindConversationDirectory(conversationId); @@ -763,7 +763,7 @@ public class FileRepository : IBotSharpRepository states = CollectConversationStates(stateFile); } - return states; + return new ConversationState(states); } public void UpdateConversationStates(string conversationId, List states) @@ -821,7 +821,7 @@ public class FileRepository : IBotSharpRepository var curStates = new Dictionary(); states.ForEach(x => { - curStates[x.Key] = x.Values.LastOrDefault()?.Data ?? string.Empty; + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; }); record.States = curStates; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 7c57ec43..5baf65c2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -746,9 +746,9 @@ public class MongoRepository : IBotSharpRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } - public List GetConversationStates(string conversationId) + public ConversationState GetConversationStates(string conversationId) { - var states = new List(); + var states = new ConversationState(); if (string.IsNullOrEmpty(conversationId)) return states; var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); @@ -756,7 +756,7 @@ public class MongoRepository : IBotSharpRepository if (foundStates == null || foundStates.States.IsNullOrEmpty()) return states; var savedStates = foundStates.States.Select(x => StateMongoElement.ToDomainElement(x)).ToList(); - return savedStates; + return new ConversationState(savedStates); } public void UpdateConversationStates(string conversationId, List states) @@ -806,7 +806,7 @@ public class MongoRepository : IBotSharpRepository var curStates = new Dictionary(); states.States.ForEach(x => { - curStates[x.Key] = x.Values.LastOrDefault()?.Data ?? string.Empty; + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; }); return new Conversation From 9d47f41109e8c5c499e48a9f96b6569c148977ce Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 2 Jan 2024 13:02:07 -0600 Subject: [PATCH 9/9] clean code --- .../Controllers/AgentController.cs | 7 ------- .../Repository/MongoRepository.cs | 19 ++----------------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index de1060dd..36c48476 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -15,13 +15,6 @@ public class AgentController : ControllerBase _services = services; } - [HttpGet("/test")] - public async Task Test() - { - - } - - [HttpGet("/agent/settings")] public AgentSettings GetSettings() { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 5baf65c2..3efc6cb3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -716,13 +716,7 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId)) return; var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault(); - if (foundConv == null) return; - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); - if (foundDialog == null) return; - var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -736,9 +730,6 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId)) return; var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault(); - if (foundConv == null) return; - var updateConv = Builders.Update .Set(x => x.UpdatedTime, DateTime.UtcNow) .Set(x => x.Title, title); @@ -764,9 +755,6 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var foundStates = _dc.ConversationStates.Find(filterStates).FirstOrDefault(); - if (foundStates == null) return; - var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); var updateStates = Builders.Update.Set(x => x.States, saveStates); @@ -778,9 +766,6 @@ public class MongoRepository : IBotSharpRepository if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(status)) return; var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var foundConv = _dc.Conversations.Find(filter).FirstOrDefault(); - if (foundConv == null) return; - var update = Builders.Update .Set(x => x.Status, status) .Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -938,8 +923,8 @@ public class MongoRepository : IBotSharpRepository var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); var update = Builders.Update - .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .PushEach(x => x.Logs, logs); + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) + .PushEach(x => x.Logs, logs); _dc.ExectionLogs.UpdateOne(filter, update, _options); }