From 6c42e4c65189ad7971d670284649b84623f6bad3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Jan 2024 20:13:38 -0600 Subject: [PATCH] add mongo plugin actions --- .../Repository/FileRepository.cs | 1078 ----------------- .../FileRepository/FileRepository.Agent.cs | 422 +++++++ .../FileRepository.Conversation.cs | 324 +++++ .../FileRepository/FileRepository.Log.cs | 84 ++ .../FileRepository.Plugin.cs | 1 + .../FileRepository.Transaction.cs | 0 .../FileRepository/FileRepository.User.cs | 31 + .../FileRepository/FileRepository.cs | 253 ++++ .../Collections/PluginDocument.cs | 6 + .../MongoDbContext.cs | 3 + .../Repository/MongoRepository.Agent.cs | 450 +++++++ .../MongoRepository.Conversation.cs | 257 ++++ .../Repository/MongoRepository.Log.cs | 61 + .../Repository/MongoRepository.Plugin.cs | 35 + .../Repository/MongoRepository.Transaction.cs | 151 +++ .../Repository/MongoRepository.User.cs | 63 + .../Repository/MongoRepository.cs | 965 +-------------- 17 files changed, 2143 insertions(+), 2041 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs rename src/Infrastructure/BotSharp.Core/Repository/{ => FileRepository}/FileRepository.Plugin.cs (97%) rename src/Infrastructure/BotSharp.Core/Repository/{ => FileRepository}/FileRepository.Transaction.cs (100%) create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/PluginDocument.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs deleted file mode 100644 index bba73858..00000000 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ /dev/null @@ -1,1078 +0,0 @@ -using BotSharp.Abstraction.Repositories; -using System.IO; -using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef; -using BotSharp.Abstraction.Users.Models; -using BotSharp.Abstraction.Agents.Models; -using MongoDB.Driver; -using BotSharp.Abstraction.Routing.Models; -using BotSharp.Abstraction.Repositories.Filters; -using BotSharp.Abstraction.Repositories.Models; -using BotSharp.Abstraction.Routing.Settings; -using BotSharp.Abstraction.Evaluations.Settings; -using System.Text.Encodings.Web; -using BotSharp.Abstraction.Plugins.Models; - -namespace BotSharp.Core.Repository; - -public partial class FileRepository : IBotSharpRepository -{ - private readonly IServiceProvider _services; - private readonly BotSharpDatabaseSettings _dbSettings; - private readonly AgentSettings _agentSettings; - private readonly ConversationSetting _conversationSettings; - private JsonSerializerOptions _options; - - private const string AGENT_FILE = "agent.json"; - private const string AGENT_INSTRUCTION_FILE = "instruction"; - private const string AGENT_FUNCTIONS_FILE = "functions.json"; - private const string AGENT_SAMPLES_FILE = "samples.txt"; - private const string USER_FILE = "user.json"; - private const string USER_AGENT_FILE = "agents.json"; - private const string CONVERSATION_FILE = "conversation.json"; - private const string DIALOG_FILE = "dialogs.txt"; - private const string STATE_FILE = "state.json"; - private const string EXECUTION_LOG_FILE = "execution.log"; - private const string PLUGIN_CONFIG_FILE = "config.json"; - - public FileRepository( - IServiceProvider services, - BotSharpDatabaseSettings dbSettings, - AgentSettings agentSettings, - ConversationSetting conversationSettings) - { - _services = services; - _dbSettings = dbSettings; - _agentSettings = agentSettings; - _conversationSettings = conversationSettings; - - _options = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true, - AllowTrailingCommas = true, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping - }; - - _dbSettings.FileRepository = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _dbSettings.FileRepository); - } - - private List _users = new List(); - private List _agents = new List(); - private List _userAgents = new List(); - private List _conversations = new List(); - private PluginConfig? _pluginConfig = null; - - private IQueryable Users - { - get - { - if (!_users.IsNullOrEmpty()) - { - return _users.AsQueryable(); - } - - var dir = Path.Combine(_dbSettings.FileRepository, "users"); - _users = new List(); - if (Directory.Exists(dir)) - { - foreach (var d in Directory.GetDirectories(dir)) - { - var userFile = Path.Combine(d, USER_FILE); - if (!Directory.Exists(d) || !File.Exists(userFile)) - continue; - - var json = File.ReadAllText(userFile); - _users.Add(JsonSerializer.Deserialize(json, _options)); - } - } - return _users.AsQueryable(); - } - } - - private IQueryable Agents - { - get - { - if (!_agents.IsNullOrEmpty()) - { - return _agents.AsQueryable(); - } - - var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir); - _agents = new List(); - if (Directory.Exists(dir)) - { - foreach (var d in Directory.GetDirectories(dir)) - { - var file = Path.Combine(d, AGENT_FILE); - if (!Directory.Exists(d) || !File.Exists(file)) - continue; - - var json = File.ReadAllText(file); - var agent = JsonSerializer.Deserialize(json, _options); - if (agent != null) - { - agent = agent.SetInstruction(FetchInstruction(d)) - .SetTemplates(FetchTemplates(d)) - .SetFunctions(FetchFunctions(d)) - .SetResponses(FetchResponses(d)) - .SetSamples(FetchSamples(d)); - _agents.Add(agent); - } - } - } - return _agents.AsQueryable(); - } - } - - private IQueryable UserAgents - { - get - { - if (!_userAgents.IsNullOrEmpty()) - { - return _userAgents.AsQueryable(); - } - - var dir = Path.Combine(_dbSettings.FileRepository, "users"); - _userAgents = new List(); - if (Directory.Exists(dir)) - { - foreach (var d in Directory.GetDirectories(dir)) - { - var file = Path.Combine(d, USER_AGENT_FILE); - if (!Directory.Exists(d) || !File.Exists(file)) - continue; - - var json = File.ReadAllText(file); - _userAgents.AddRange(JsonSerializer.Deserialize>(json, _options)); - } - } - return _userAgents.AsQueryable(); - } - } - - #region Agent - public void UpdateAgent(Agent agent, AgentField field) - { - if (agent == null || string.IsNullOrEmpty(agent.Id)) return; - - switch (field) - { - case AgentField.Name: - UpdateAgentName(agent.Id, agent.Name); - break; - case AgentField.Description: - UpdateAgentDescription(agent.Id, agent.Description); - break; - case AgentField.IsPublic: - UpdateAgentIsPublic(agent.Id, agent.IsPublic); - break; - case AgentField.Disabled: - UpdateAgentDisabled(agent.Id, agent.Disabled); - break; - case AgentField.AllowRouting: - UpdateAgentAllowRouting(agent.Id, agent.AllowRouting); - break; - case AgentField.Profiles: - UpdateAgentProfiles(agent.Id, agent.Profiles); - break; - case AgentField.RoutingRule: - UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); - break; - case AgentField.Instruction: - UpdateAgentInstruction(agent.Id, agent.Instruction); - break; - case AgentField.Function: - UpdateAgentFunctions(agent.Id, agent.Functions); - break; - case AgentField.Template: - UpdateAgentTemplates(agent.Id, agent.Templates); - break; - case AgentField.Response: - UpdateAgentResponses(agent.Id, agent.Responses); - break; - case AgentField.Sample: - UpdateAgentSamples(agent.Id, agent.Samples); - break; - case AgentField.LlmConfig: - UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); - break; - case AgentField.All: - UpdateAgentAllFields(agent); - break; - default: - break; - } - } - - #region Update Agent Fields - private void UpdateAgentName(string agentId, string name) - { - if (string.IsNullOrEmpty(name)) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.Name = name; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentDescription(string agentId, string description) - { - if (string.IsNullOrEmpty(description)) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.Description = description; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentIsPublic(string agentId, bool isPublic) - { - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.IsPublic = isPublic; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentDisabled(string agentId, bool disabled) - { - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.Disabled = disabled; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentAllowRouting(string agentId, bool allowRouting) - { - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.AllowRouting = allowRouting; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentProfiles(string agentId, List profiles) - { - if (profiles.IsNullOrEmpty()) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.Profiles = profiles; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentRoutingRules(string agentId, List rules) - { - if (rules.IsNullOrEmpty()) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.RoutingRules = rules; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentInstruction(string agentId, string instruction) - { - if (string.IsNullOrEmpty(instruction)) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); - - File.WriteAllText(instructionFile, instruction); - } - - private void UpdateAgentFunctions(string agentId, List inputFunctions) - { - if (inputFunctions.IsNullOrEmpty()) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - var functionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, AGENT_FUNCTIONS_FILE); - - var functionText = JsonSerializer.Serialize(inputFunctions, _options); - File.WriteAllText(functionFile, functionText); - } - - private void UpdateAgentTemplates(string agentId, List templates) - { - if (templates.IsNullOrEmpty()) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - var templateDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "templates"); - - if (!Directory.Exists(templateDir)) - { - Directory.CreateDirectory(templateDir); - } - - foreach (var file in Directory.GetFiles(templateDir)) - { - File.Delete(file); - } - - foreach (var template in templates) - { - var file = Path.Combine(templateDir, $"{template.Name}.{_agentSettings.TemplateFormat}"); - File.WriteAllText(file, template.Content); - } - } - - private void UpdateAgentResponses(string agentId, List responses) - { - if (responses.IsNullOrEmpty()) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - var responseDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "responses"); - if (!Directory.Exists(responseDir)) - { - Directory.CreateDirectory(responseDir); - } - - foreach (var file in Directory.GetFiles(responseDir)) - { - File.Delete(file); - } - - for (int i = 0; i < responses.Count; i++) - { - var response = responses[i]; - var fileName = $"{response.Prefix}.{response.Intent}.{i}.{_agentSettings.TemplateFormat}"; - var file = Path.Combine(responseDir, fileName); - File.WriteAllText(file, response.Content); - } - } - - private void UpdateAgentSamples(string agentId, List samples) - { - if (samples.IsNullOrEmpty()) return; - - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_SAMPLES_FILE); - File.WriteAllLines(file, samples); - } - - private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) - { - var (agent, agentFile) = GetAgentFromFile(agentId); - if (agent == null) return; - - agent.LlmConfig = config; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - } - - private void UpdateAgentAllFields(Agent inputAgent) - { - var (agent, agentFile) = GetAgentFromFile(inputAgent.Id); - if (agent == null) return; - - agent.Name = inputAgent.Name; - agent.Description = inputAgent.Description; - agent.IsPublic = inputAgent.IsPublic; - agent.Disabled = inputAgent.Disabled; - agent.AllowRouting = inputAgent.AllowRouting; - agent.Profiles = inputAgent.Profiles; - agent.RoutingRules = inputAgent.RoutingRules; - agent.UpdatedDateTime = DateTime.UtcNow; - var json = JsonSerializer.Serialize(agent, _options); - File.WriteAllText(agentFile, json); - - UpdateAgentInstruction(inputAgent.Id, inputAgent.Instruction); - UpdateAgentResponses(inputAgent.Id, inputAgent.Responses); - UpdateAgentTemplates(inputAgent.Id, inputAgent.Templates); - UpdateAgentFunctions(inputAgent.Id, inputAgent.Functions); - UpdateAgentSamples(inputAgent.Id, inputAgent.Samples); - } - #endregion - - public List GetAgentResponses(string agentId, string prefix, string intent) - { - var responses = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "responses"); - if (!Directory.Exists(dir)) return responses; - - foreach (var file in Directory.GetFiles(dir)) - { - if (file.Split(Path.DirectorySeparatorChar) - .Last() - .StartsWith(prefix + "." + intent)) - { - responses.Add(File.ReadAllText(file)); - } - } - - return responses; - } - - public Agent? GetAgent(string agentId) - { - var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir); - var dir = Directory.GetDirectories(agentDir).FirstOrDefault(x => x.Split(Path.DirectorySeparatorChar).Last() == agentId); - - if (!string.IsNullOrEmpty(dir)) - { - var json = File.ReadAllText(Path.Combine(dir, AGENT_FILE)); - if (string.IsNullOrEmpty(json)) return null; - - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) return null; - - var instruction = FetchInstruction(dir); - var functions = FetchFunctions(dir); - var samples = FetchSamples(dir); - var templates = FetchTemplates(dir); - var responses = FetchResponses(dir); - return record.SetInstruction(instruction) - .SetFunctions(functions) - .SetSamples(samples) - .SetTemplates(templates) - .SetResponses(responses); - } - - return null; - } - - public List GetAgents(AgentFilter filter) - { - var query = Agents; - if (!string.IsNullOrEmpty(filter.AgentName)) - { - query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower()); - } - - if (filter.Disabled.HasValue) - { - query = query.Where(x => x.Disabled == filter.Disabled); - } - - if (filter.AllowRouting.HasValue) - { - query = query.Where(x => x.AllowRouting == filter.AllowRouting); - } - - if (filter.IsPublic.HasValue) - { - query = query.Where(x => x.IsPublic == filter.IsPublic); - } - - if (filter.IsRouter.HasValue) - { - var route = _services.GetRequiredService(); - query = filter.IsRouter.Value ? - query.Where(x => x.Id == route.AgentId) : - query.Where(x => x.Id != route.AgentId); - } - - if (filter.IsEvaluator.HasValue) - { - var evaluate = _services.GetRequiredService(); - query = filter.IsEvaluator.Value ? - query.Where(x => x.Id == evaluate.AgentId) : - query.Where(x => x.Id != evaluate.AgentId); - } - - if (filter.AgentIds != null) - { - query = query.Where(x => filter.AgentIds.Contains(x.Id)); - } - - return query.ToList(); - } - - public List GetAgentsByUser(string userId) - { - var agentIds = (from ua in UserAgents - join u in Users on ua.UserId equals u.Id - where ua.UserId == userId || u.ExternalId == userId - select ua.AgentId).ToList(); - - var filter = new AgentFilter - { - IsPublic = true, - AgentIds = agentIds - }; - var agents = GetAgents(filter); - return agents; - } - - - public string GetAgentTemplate(string agentId, string templateName) - { - var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "templates"); - if (!Directory.Exists(dir)) return string.Empty; - - foreach (var file in Directory.GetFiles(dir)) - { - var fileName = file.Split(Path.DirectorySeparatorChar).Last(); - var splits = ParseFileNameByPath(fileName.ToLower()); - var name = splits[0]; - var extension = splits[1]; - if (name.IsEqualTo(templateName) && extension.IsEqualTo(_agentSettings.TemplateFormat)) - { - return File.ReadAllText(file); - } - } - - return string.Empty; - } - - public void BulkInsertAgents(List agents) - { - } - - public void BulkInsertUserAgents(List userAgents) - { - } - - public bool DeleteAgents() - { - return false; - } - #endregion - - #region Conversation - public void CreateNewConversation(Conversation conversation) - { - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - var convFile = Path.Combine(dir, CONVERSATION_FILE); - if (!File.Exists(convFile)) - { - File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); - } - - var dialogFile = Path.Combine(dir, DIALOG_FILE); - if (!File.Exists(dialogFile)) - { - File.WriteAllText(dialogFile, string.Empty); - } - - var stateFile = Path.Combine(dir, STATE_FILE); - if (!File.Exists(stateFile)) - { - var states = conversation.States ?? new Dictionary(); - var initialStates = states.Select(x => new StateKeyValue - { - Key = x.Key, - Values = new List - { - new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow } - } - }).ToList(); - File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options)); - } - } - - public bool DeleteConversation(string conversationId) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - - Directory.Delete(convDir, true); - return true; - } - - public List GetConversationDialogs(string conversationId) - { - var dialogs = new List(); - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - dialogs = CollectDialogElements(dialogDir); - } - - return dialogs; - } - - public void UpdateConversationDialogElements(string conversationId, List updateElements) - { - var dialogElements = GetConversationDialogs(conversationId); - if (dialogElements.IsNullOrEmpty() || updateElements.IsNullOrEmpty()) return; - - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - if (File.Exists(dialogDir)) - { - var updated = dialogElements.Select((x, idx) => - { - var found = updateElements.FirstOrDefault(e => e.Index == idx); - if (found != null) - { - x.Content = found.UpdateContent; - } - return x; - }).ToList(); - - var texts = ParseDialogElements(updated); - File.WriteAllLines(dialogDir, texts); - } - } - } - - public void AppendConversationDialogs(string conversationId, List dialogs) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - if (File.Exists(dialogDir)) - { - var texts = ParseDialogElements(dialogs); - File.AppendAllLines(dialogDir, texts); - } - } - } - - public void UpdateConversationTitle(string conversationId, string title) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - if (record != null) - { - record.Title = title; - record.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); - } - } - } - - public ConversationState GetConversationStates(string conversationId) - { - var states = new List(); - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var stateFile = Path.Combine(convDir, STATE_FILE); - states = CollectConversationStates(stateFile); - } - - return new ConversationState(states); - } - - public void UpdateConversationStates(string conversationId, List states) - { - if (states.IsNullOrEmpty()) return; - - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var stateFile = Path.Combine(convDir, STATE_FILE); - if (File.Exists(stateFile)) - { - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateFile, stateStr); - } - } - } - - public void UpdateConversationStatus(string conversationId, string status) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (File.Exists(convFile)) - { - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - conv.Status = status; - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - } - } - - public Conversation GetConversation(string conversationId) - { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return null; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - if (record != null) - { - record.Dialogs = CollectDialogElements(dialogFile); - } - - var stateFile = Path.Combine(convDir, STATE_FILE); - if (record != null) - { - var states = CollectConversationStates(stateFile); - var curStates = new Dictionary(); - states.ForEach(x => - { - curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; - }); - record.States = curStates; - } - - return record; - } - - public List GetConversations(ConversationFilter filter) - { - var records = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - - foreach (var d in Directory.GetDirectories(dir)) - { - var path = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(path)) continue; - - var json = File.ReadAllText(path); - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) continue; - - var matched = true; - if(filter.Id != null) matched = matched && record.Id == filter.Id; - if (filter.AgentId != null) matched = matched && record.AgentId == filter.AgentId; - if (filter.Status != null) matched = matched && record.Status == filter.Status; - if (filter.Channel != null) matched = matched && record.Channel == filter.Channel; - if (filter.UserId != null) matched = matched && record.UserId == filter.UserId; - - if (!matched) continue; - records.Add(record); - } - - return records; - } - - public List GetLastConversations() - { - var records = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - - foreach (var d in Directory.GetDirectories(dir)) - { - var path = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(path)) continue; - - var json = File.ReadAllText(path); - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) continue; - - records.Add(record); - } - return records.GroupBy(r => r.UserId) - .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) - .ToList(); - } - #endregion - - #region User - public User? GetUserByEmail(string email) - { - return Users.FirstOrDefault(x => x.Email == email); - } - - public User? GetUserById(string id = null) - { - return Users.FirstOrDefault(x => x.ExternalId == id || x.Id == id); - } - - public void CreateUser(User user) - { - var userId = Guid.NewGuid().ToString(); - user.Id = userId; - var dir = Path.Combine(_dbSettings.FileRepository, "users", userId); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - var path = Path.Combine(dir, "user.json"); - File.WriteAllText(path, JsonSerializer.Serialize(user, _options)); - } - #endregion - - #region Execution Log - public void AddExecutionLogs(string conversationId, List logs) - { - if (string.IsNullOrEmpty(conversationId) || logs.IsNullOrEmpty()) return; - - var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - var file = Path.Combine(dir, EXECUTION_LOG_FILE); - File.AppendAllLines(file, logs); - } - - public List GetExecutionLogs(string conversationId) - { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; - - var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); - if (!Directory.Exists(dir)) return logs; - - var file = Path.Combine(dir, EXECUTION_LOG_FILE); - logs = File.ReadAllLines(file)?.ToList() ?? new List(); - return logs; - } - #endregion - - #region LLM Completion Log - public void SaveLlmCompletionLog(LlmCompletionLog log) - { - if (log == null) return; - - log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); - log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); - - var convDir = FindConversationDirectory(log.ConversationId); - if (string.IsNullOrEmpty(convDir)) - { - convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); - Directory.CreateDirectory(convDir); - } - - var logDir = Path.Combine(convDir, "llm_prompt_log"); - if (!Directory.Exists(logDir)) - { - Directory.CreateDirectory(logDir); - } - - var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); - var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); - File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); - } - #endregion - - - #region Private methods - private string GetAgentDataDir(string agentId) - { - var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); - if (!Directory.Exists(dir)) - { - dir = string.Empty; - } - return dir; - } - - private (Agent?, string) GetAgentFromFile(string agentId) - { - var dir = GetAgentDataDir(agentId); - var agentFile = Path.Combine(dir, AGENT_FILE); - if (!File.Exists(agentFile)) return (null, string.Empty); - - var json = File.ReadAllText(agentFile); - var agent = JsonSerializer.Deserialize(json, _options); - return (agent, agentFile); - } - - private string FetchInstruction(string fileDir) - { - var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); - if (!File.Exists(file)) return string.Empty; - - var instruction = File.ReadAllText(file); - return instruction; - } - - private List FetchFunctions(string fileDir) - { - var file = Path.Combine(fileDir, AGENT_FUNCTIONS_FILE); - if (!File.Exists(file)) return new List(); - - var functionsJson = File.ReadAllText(file); - var functions = JsonSerializer.Deserialize>(functionsJson, _options); - return functions; - } - - private List FetchSamples(string fileDir) - { - var file = Path.Combine(fileDir, AGENT_SAMPLES_FILE); - if (!File.Exists(file)) return new List(); - - return File.ReadAllLines(file)?.ToList() ?? new List(); - } - - private List FetchTemplates(string fileDir) - { - var templates = new List(); - var templateDir = Path.Combine(fileDir, "templates"); - if (!Directory.Exists(templateDir)) return templates; - - foreach (var file in Directory.GetFiles(templateDir)) - { - var fileName = file.Split(Path.DirectorySeparatorChar).Last(); - var splits = fileName.ToLower().Split('.'); - var name = string.Join('.', splits.Take(splits.Length - 1)); - var extension = splits.Last(); - if (extension.Equals(_agentSettings.TemplateFormat, StringComparison.OrdinalIgnoreCase)) - { - var content = File.ReadAllText(file); - templates.Add(new AgentTemplate(name, content)); - } - } - - return templates; - } - - private List FetchResponses(string fileDir) - { - var responses = new List(); - var responseDir = Path.Combine(fileDir, "responses"); - if (!Directory.Exists(responseDir)) return responses; - - foreach (var file in Directory.GetFiles(responseDir)) - { - var fileName = file.Split(Path.DirectorySeparatorChar).Last(); - var splits = fileName.ToLower().Split('.'); - var prefix = splits[0]; - var intent = splits[1]; - var content = File.ReadAllText(file); - responses.Add(new AgentResponse(prefix, intent, content)); - } - - return responses; - } - - private string? FindConversationDirectory(string conversationId) - { - if (string.IsNullOrEmpty(conversationId)) return null; - - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); - if (!Directory.Exists(dir)) return null; - - return dir; - } - - private List CollectDialogElements(string dialogDir) - { - var dialogs = new List(); - - if (!File.Exists(dialogDir)) return dialogs; - - var rawDialogs = File.ReadAllLines(dialogDir); - if (!rawDialogs.IsNullOrEmpty()) - { - for (int i = 0; i < rawDialogs.Count(); i += 2) - { - var blocks = rawDialogs[i].Split("|"); - var content = rawDialogs[i + 1]; - var trimmed = content.Substring(4); - var meta = new DialogMeta - { - Role = blocks[1], - AgentId = blocks[2], - MessageId = blocks[3], - FunctionName = blocks[1] == AgentRole.Function ? blocks[4] : null, - SenderId = blocks[1] == AgentRole.Function ? null : blocks[4], - CreateTime = DateTime.Parse(blocks[0]) - }; - dialogs.Add(new DialogElement(meta, trimmed)); - } - } - return dialogs; - } - - private List ParseDialogElements(List dialogs) - { - var dialogTexts = new List(); - if (dialogs.IsNullOrEmpty()) return dialogTexts; - - foreach (var element in dialogs) - { - var meta = element.MetaData; - var source = meta.FunctionName ?? meta.SenderId; - var metaStr = $"{meta.CreateTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{source}"; - dialogTexts.Add(metaStr); - var content = $" - {element.Content}"; - dialogTexts.Add(content); - } - - return dialogTexts; - } - - private List CollectConversationStates(string stateFile) - { - var states = new List(); - if (!File.Exists(stateFile)) return states; - - var stateStr = File.ReadAllText(stateFile); - if (string.IsNullOrEmpty(stateStr)) return states; - - states = JsonSerializer.Deserialize>(stateStr, _options); - return states ?? new List(); - } - - private int GetNextLlmCompletionLogIndex(string logDir, string id) - { - var files = Directory.GetFiles(logDir); - if (files.IsNullOrEmpty()) - return 0; - - var logIndexes = files.Where(file => - { - var fileName = ParseFileNameByPath(file); - return fileName[0].IsEqualTo(id); - }).Select(file => - { - var fileName = ParseFileNameByPath(file); - return int.Parse(fileName[1]); - }).ToList(); - - return logIndexes.IsNullOrEmpty() ? 0 : logIndexes.Max() + 1; - } - - private string[] ParseFileNameByPath(string path, string separator = ".") - { - var name = path.Split(Path.DirectorySeparatorChar).Last(); - return name.Split(separator); - } - #endregion -} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs new file mode 100644 index 00000000..ba979e84 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -0,0 +1,422 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Evaluations.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Routing.Models; +using BotSharp.Abstraction.Routing.Settings; +using System.IO; + +namespace BotSharp.Core.Repository +{ + public partial class FileRepository + { + public void UpdateAgent(Agent agent, AgentField field) + { + if (agent == null || string.IsNullOrEmpty(agent.Id)) return; + + switch (field) + { + case AgentField.Name: + UpdateAgentName(agent.Id, agent.Name); + break; + case AgentField.Description: + UpdateAgentDescription(agent.Id, agent.Description); + break; + case AgentField.IsPublic: + UpdateAgentIsPublic(agent.Id, agent.IsPublic); + break; + case AgentField.Disabled: + UpdateAgentDisabled(agent.Id, agent.Disabled); + break; + case AgentField.AllowRouting: + UpdateAgentAllowRouting(agent.Id, agent.AllowRouting); + break; + case AgentField.Profiles: + UpdateAgentProfiles(agent.Id, agent.Profiles); + break; + case AgentField.RoutingRule: + UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); + break; + case AgentField.Instruction: + UpdateAgentInstruction(agent.Id, agent.Instruction); + break; + case AgentField.Function: + UpdateAgentFunctions(agent.Id, agent.Functions); + break; + case AgentField.Template: + UpdateAgentTemplates(agent.Id, agent.Templates); + break; + case AgentField.Response: + UpdateAgentResponses(agent.Id, agent.Responses); + break; + case AgentField.Sample: + UpdateAgentSamples(agent.Id, agent.Samples); + break; + case AgentField.LlmConfig: + UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); + break; + case AgentField.All: + UpdateAgentAllFields(agent); + break; + default: + break; + } + } + + #region Update Agent Fields + private void UpdateAgentName(string agentId, string name) + { + if (string.IsNullOrEmpty(name)) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.Name = name; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentDescription(string agentId, string description) + { + if (string.IsNullOrEmpty(description)) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.Description = description; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentIsPublic(string agentId, bool isPublic) + { + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.IsPublic = isPublic; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentDisabled(string agentId, bool disabled) + { + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.Disabled = disabled; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentAllowRouting(string agentId, bool allowRouting) + { + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.AllowRouting = allowRouting; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentProfiles(string agentId, List profiles) + { + if (profiles.IsNullOrEmpty()) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.Profiles = profiles; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentRoutingRules(string agentId, List rules) + { + if (rules.IsNullOrEmpty()) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.RoutingRules = rules; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentInstruction(string agentId, string instruction) + { + if (string.IsNullOrEmpty(instruction)) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, + agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); + + File.WriteAllText(instructionFile, instruction); + } + + private void UpdateAgentFunctions(string agentId, List inputFunctions) + { + if (inputFunctions.IsNullOrEmpty()) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + var functionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, + agentId, AGENT_FUNCTIONS_FILE); + + var functionText = JsonSerializer.Serialize(inputFunctions, _options); + File.WriteAllText(functionFile, functionText); + } + + private void UpdateAgentTemplates(string agentId, List templates) + { + if (templates.IsNullOrEmpty()) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + var templateDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "templates"); + + if (!Directory.Exists(templateDir)) + { + Directory.CreateDirectory(templateDir); + } + + foreach (var file in Directory.GetFiles(templateDir)) + { + File.Delete(file); + } + + foreach (var template in templates) + { + var file = Path.Combine(templateDir, $"{template.Name}.{_agentSettings.TemplateFormat}"); + File.WriteAllText(file, template.Content); + } + } + + private void UpdateAgentResponses(string agentId, List responses) + { + if (responses.IsNullOrEmpty()) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + var responseDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "responses"); + if (!Directory.Exists(responseDir)) + { + Directory.CreateDirectory(responseDir); + } + + foreach (var file in Directory.GetFiles(responseDir)) + { + File.Delete(file); + } + + for (int i = 0; i < responses.Count; i++) + { + var response = responses[i]; + var fileName = $"{response.Prefix}.{response.Intent}.{i}.{_agentSettings.TemplateFormat}"; + var file = Path.Combine(responseDir, fileName); + File.WriteAllText(file, response.Content); + } + } + + private void UpdateAgentSamples(string agentId, List samples) + { + if (samples.IsNullOrEmpty()) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_SAMPLES_FILE); + File.WriteAllLines(file, samples); + } + + private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) + { + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.LlmConfig = config; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + + private void UpdateAgentAllFields(Agent inputAgent) + { + var (agent, agentFile) = GetAgentFromFile(inputAgent.Id); + if (agent == null) return; + + agent.Name = inputAgent.Name; + agent.Description = inputAgent.Description; + agent.IsPublic = inputAgent.IsPublic; + agent.Disabled = inputAgent.Disabled; + agent.AllowRouting = inputAgent.AllowRouting; + agent.Profiles = inputAgent.Profiles; + agent.RoutingRules = inputAgent.RoutingRules; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + + UpdateAgentInstruction(inputAgent.Id, inputAgent.Instruction); + UpdateAgentResponses(inputAgent.Id, inputAgent.Responses); + UpdateAgentTemplates(inputAgent.Id, inputAgent.Templates); + UpdateAgentFunctions(inputAgent.Id, inputAgent.Functions); + UpdateAgentSamples(inputAgent.Id, inputAgent.Samples); + } + #endregion + + public List GetAgentResponses(string agentId, string prefix, string intent) + { + var responses = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "responses"); + if (!Directory.Exists(dir)) return responses; + + foreach (var file in Directory.GetFiles(dir)) + { + if (file.Split(Path.DirectorySeparatorChar) + .Last() + .StartsWith(prefix + "." + intent)) + { + responses.Add(File.ReadAllText(file)); + } + } + + return responses; + } + + public Agent? GetAgent(string agentId) + { + var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir); + var dir = Directory.GetDirectories(agentDir).FirstOrDefault(x => x.Split(Path.DirectorySeparatorChar).Last() == agentId); + + if (!string.IsNullOrEmpty(dir)) + { + var json = File.ReadAllText(Path.Combine(dir, AGENT_FILE)); + if (string.IsNullOrEmpty(json)) return null; + + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) return null; + + var instruction = FetchInstruction(dir); + var functions = FetchFunctions(dir); + var samples = FetchSamples(dir); + var templates = FetchTemplates(dir); + var responses = FetchResponses(dir); + return record.SetInstruction(instruction) + .SetFunctions(functions) + .SetSamples(samples) + .SetTemplates(templates) + .SetResponses(responses); + } + + return null; + } + + public List GetAgents(AgentFilter filter) + { + var query = Agents; + if (!string.IsNullOrEmpty(filter.AgentName)) + { + query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower()); + } + + if (filter.Disabled.HasValue) + { + query = query.Where(x => x.Disabled == filter.Disabled); + } + + if (filter.AllowRouting.HasValue) + { + query = query.Where(x => x.AllowRouting == filter.AllowRouting); + } + + if (filter.IsPublic.HasValue) + { + query = query.Where(x => x.IsPublic == filter.IsPublic); + } + + if (filter.IsRouter.HasValue) + { + var route = _services.GetRequiredService(); + query = filter.IsRouter.Value ? + query.Where(x => x.Id == route.AgentId) : + query.Where(x => x.Id != route.AgentId); + } + + if (filter.IsEvaluator.HasValue) + { + var evaluate = _services.GetRequiredService(); + query = filter.IsEvaluator.Value ? + query.Where(x => x.Id == evaluate.AgentId) : + query.Where(x => x.Id != evaluate.AgentId); + } + + if (filter.AgentIds != null) + { + query = query.Where(x => filter.AgentIds.Contains(x.Id)); + } + + return query.ToList(); + } + + public List GetAgentsByUser(string userId) + { + var agentIds = (from ua in UserAgents + join u in Users on ua.UserId equals u.Id + where ua.UserId == userId || u.ExternalId == userId + select ua.AgentId).ToList(); + + var filter = new AgentFilter + { + IsPublic = true, + AgentIds = agentIds + }; + var agents = GetAgents(filter); + return agents; + } + + + public string GetAgentTemplate(string agentId, string templateName) + { + var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "templates"); + if (!Directory.Exists(dir)) return string.Empty; + + foreach (var file in Directory.GetFiles(dir)) + { + var fileName = file.Split(Path.DirectorySeparatorChar).Last(); + var splits = ParseFileNameByPath(fileName.ToLower()); + var name = splits[0]; + var extension = splits[1]; + if (name.IsEqualTo(templateName) && extension.IsEqualTo(_agentSettings.TemplateFormat)) + { + return File.ReadAllText(file); + } + } + + return string.Empty; + } + + public void BulkInsertAgents(List agents) + { + } + + public void BulkInsertUserAgents(List userAgents) + { + } + + public bool DeleteAgents() + { + return false; + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs new file mode 100644 index 00000000..da6b9c9d --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -0,0 +1,324 @@ +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; +using System.IO; + +namespace BotSharp.Core.Repository +{ + public partial class FileRepository + { + public void CreateNewConversation(Conversation conversation) + { + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var convFile = Path.Combine(dir, CONVERSATION_FILE); + if (!File.Exists(convFile)) + { + File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); + } + + var dialogFile = Path.Combine(dir, DIALOG_FILE); + if (!File.Exists(dialogFile)) + { + File.WriteAllText(dialogFile, string.Empty); + } + + var stateFile = Path.Combine(dir, STATE_FILE); + if (!File.Exists(stateFile)) + { + var states = conversation.States ?? new Dictionary(); + var initialStates = states.Select(x => new StateKeyValue + { + Key = x.Key, + Values = new List + { + new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow } + } + }).ToList(); + File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options)); + } + } + + public bool DeleteConversation(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + Directory.Delete(convDir, true); + return true; + } + + public List GetConversationDialogs(string conversationId) + { + var dialogs = new List(); + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + dialogs = CollectDialogElements(dialogDir); + } + + return dialogs; + } + + public void UpdateConversationDialogElements(string conversationId, List updateElements) + { + var dialogElements = GetConversationDialogs(conversationId); + if (dialogElements.IsNullOrEmpty() || updateElements.IsNullOrEmpty()) return; + + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + if (File.Exists(dialogDir)) + { + var updated = dialogElements.Select((x, idx) => + { + var found = updateElements.FirstOrDefault(e => e.Index == idx); + if (found != null) + { + x.Content = found.UpdateContent; + } + return x; + }).ToList(); + + var texts = ParseDialogElements(updated); + File.WriteAllLines(dialogDir, texts); + } + } + } + + public void AppendConversationDialogs(string conversationId, List dialogs) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + if (File.Exists(dialogDir)) + { + var texts = ParseDialogElements(dialogs); + File.AppendAllLines(dialogDir, texts); + } + } + } + + public void UpdateConversationTitle(string conversationId, string title) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); + if (record != null) + { + record.Title = title; + record.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); + } + } + } + + public ConversationState GetConversationStates(string conversationId) + { + var states = new List(); + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var stateFile = Path.Combine(convDir, STATE_FILE); + states = CollectConversationStates(stateFile); + } + + return new ConversationState(states); + } + + public void UpdateConversationStates(string conversationId, List states) + { + if (states.IsNullOrEmpty()) return; + + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) + { + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); + } + } + } + + public void UpdateConversationStatus(string conversationId, string status) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (File.Exists(convFile)) + { + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + conv.Status = status; + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } + } + } + + public Conversation GetConversation(string conversationId) + { + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return null; + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); + + var dialogFile = Path.Combine(convDir, DIALOG_FILE); + if (record != null) + { + record.Dialogs = CollectDialogElements(dialogFile); + } + + var stateFile = Path.Combine(convDir, STATE_FILE); + if (record != null) + { + var states = CollectConversationStates(stateFile); + var curStates = new Dictionary(); + states.ForEach(x => + { + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; + }); + record.States = curStates; + } + + return record; + } + + public List GetConversations(ConversationFilter filter) + { + var records = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + + foreach (var d in Directory.GetDirectories(dir)) + { + var path = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(path)) continue; + + var json = File.ReadAllText(path); + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) continue; + + var matched = true; + if (filter.Id != null) matched = matched && record.Id == filter.Id; + if (filter.AgentId != null) matched = matched && record.AgentId == filter.AgentId; + if (filter.Status != null) matched = matched && record.Status == filter.Status; + if (filter.Channel != null) matched = matched && record.Channel == filter.Channel; + if (filter.UserId != null) matched = matched && record.UserId == filter.UserId; + + if (!matched) continue; + records.Add(record); + } + + return records; + } + + public List GetLastConversations() + { + var records = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + + foreach (var d in Directory.GetDirectories(dir)) + { + var path = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(path)) continue; + + var json = File.ReadAllText(path); + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) continue; + + records.Add(record); + } + return records.GroupBy(r => r.UserId) + .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) + .ToList(); + } + + + #region Private methods + private string? FindConversationDirectory(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return null; + + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); + if (!Directory.Exists(dir)) return null; + + return dir; + } + + private List CollectDialogElements(string dialogDir) + { + var dialogs = new List(); + + if (!File.Exists(dialogDir)) return dialogs; + + var rawDialogs = File.ReadAllLines(dialogDir); + if (!rawDialogs.IsNullOrEmpty()) + { + for (int i = 0; i < rawDialogs.Count(); i += 2) + { + var blocks = rawDialogs[i].Split("|"); + var content = rawDialogs[i + 1]; + var trimmed = content.Substring(4); + var meta = new DialogMeta + { + Role = blocks[1], + AgentId = blocks[2], + MessageId = blocks[3], + FunctionName = blocks[1] == AgentRole.Function ? blocks[4] : null, + SenderId = blocks[1] == AgentRole.Function ? null : blocks[4], + CreateTime = DateTime.Parse(blocks[0]) + }; + dialogs.Add(new DialogElement(meta, trimmed)); + } + } + return dialogs; + } + + private List ParseDialogElements(List dialogs) + { + var dialogTexts = new List(); + if (dialogs.IsNullOrEmpty()) return dialogTexts; + + foreach (var element in dialogs) + { + var meta = element.MetaData; + var source = meta.FunctionName ?? meta.SenderId; + var metaStr = $"{meta.CreateTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{source}"; + dialogTexts.Add(metaStr); + var content = $" - {element.Content}"; + dialogTexts.Add(content); + } + + return dialogTexts; + } + + private List CollectConversationStates(string stateFile) + { + var states = new List(); + if (!File.Exists(stateFile)) return states; + + var stateStr = File.ReadAllText(stateFile); + if (string.IsNullOrEmpty(stateStr)) return states; + + states = JsonSerializer.Deserialize>(stateStr, _options); + return states ?? new List(); + } + #endregion + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs new file mode 100644 index 00000000..78e8f9b1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -0,0 +1,84 @@ +using System.IO; + +namespace BotSharp.Core.Repository +{ + public partial class FileRepository + { + #region Execution Log + public void AddExecutionLogs(string conversationId, List logs) + { + if (string.IsNullOrEmpty(conversationId) || logs.IsNullOrEmpty()) return; + + var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var file = Path.Combine(dir, EXECUTION_LOG_FILE); + File.AppendAllLines(file, logs); + } + + public List GetExecutionLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); + if (!Directory.Exists(dir)) return logs; + + var file = Path.Combine(dir, EXECUTION_LOG_FILE); + logs = File.ReadAllLines(file)?.ToList() ?? new List(); + return logs; + } + #endregion + + #region LLM Completion Log + public void SaveLlmCompletionLog(LlmCompletionLog log) + { + if (log == null) return; + + log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var convDir = FindConversationDirectory(log.ConversationId); + if (string.IsNullOrEmpty(convDir)) + { + convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + Directory.CreateDirectory(convDir); + } + + var logDir = Path.Combine(convDir, "llm_prompt_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + #endregion + + #region Private methods + private int GetNextLlmCompletionLogIndex(string logDir, string id) + { + var files = Directory.GetFiles(logDir); + if (files.IsNullOrEmpty()) + return 0; + + var logIndexes = files.Where(file => + { + var fileName = ParseFileNameByPath(file); + return fileName[0].IsEqualTo(id); + }).Select(file => + { + var fileName = ParseFileNameByPath(file); + return int.Parse(fileName[1]); + }).ToList(); + + return logIndexes.IsNullOrEmpty() ? 0 : logIndexes.Max() + 1; + } + #endregion + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.Plugin.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Plugin.cs similarity index 97% rename from src/Infrastructure/BotSharp.Core/Repository/FileRepository.Plugin.cs rename to src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Plugin.cs index 75ba0e0b..787c57b7 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.Plugin.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Plugin.cs @@ -34,5 +34,6 @@ public partial class FileRepository { var configFile = Path.Combine(_dbSettings.FileRepository, "plugins", "config.json"); File.WriteAllText(configFile, JsonSerializer.Serialize(config, _options)); + _pluginConfig = null; } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.Transaction.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Transaction.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Repository/FileRepository.Transaction.cs rename to src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Transaction.cs diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs new file mode 100644 index 00000000..64c46ad9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.User.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Users.Models; +using System.IO; + +namespace BotSharp.Core.Repository +{ + public partial class FileRepository + { + public User? GetUserByEmail(string email) + { + return Users.FirstOrDefault(x => x.Email == email); + } + + public User? GetUserById(string id = null) + { + return Users.FirstOrDefault(x => x.ExternalId == id || x.Id == id); + } + + public void CreateUser(User user) + { + var userId = Guid.NewGuid().ToString(); + user.Id = userId; + var dir = Path.Combine(_dbSettings.FileRepository, "users", userId); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + var path = Path.Combine(dir, "user.json"); + File.WriteAllText(path, JsonSerializer.Serialize(user, _options)); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs new file mode 100644 index 00000000..6874ed02 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -0,0 +1,253 @@ +using BotSharp.Abstraction.Repositories; +using System.IO; +using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef; +using BotSharp.Abstraction.Users.Models; +using BotSharp.Abstraction.Agents.Models; +using MongoDB.Driver; +using BotSharp.Abstraction.Routing.Models; +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; +using BotSharp.Abstraction.Routing.Settings; +using BotSharp.Abstraction.Evaluations.Settings; +using System.Text.Encodings.Web; +using BotSharp.Abstraction.Plugins.Models; + +namespace BotSharp.Core.Repository; + +public partial class FileRepository : IBotSharpRepository +{ + private readonly IServiceProvider _services; + private readonly BotSharpDatabaseSettings _dbSettings; + private readonly AgentSettings _agentSettings; + private readonly ConversationSetting _conversationSettings; + private JsonSerializerOptions _options; + + private const string AGENT_FILE = "agent.json"; + private const string AGENT_INSTRUCTION_FILE = "instruction"; + private const string AGENT_FUNCTIONS_FILE = "functions.json"; + private const string AGENT_SAMPLES_FILE = "samples.txt"; + private const string USER_FILE = "user.json"; + private const string USER_AGENT_FILE = "agents.json"; + private const string CONVERSATION_FILE = "conversation.json"; + private const string DIALOG_FILE = "dialogs.txt"; + private const string STATE_FILE = "state.json"; + private const string EXECUTION_LOG_FILE = "execution.log"; + private const string PLUGIN_CONFIG_FILE = "config.json"; + + public FileRepository( + IServiceProvider services, + BotSharpDatabaseSettings dbSettings, + AgentSettings agentSettings, + ConversationSetting conversationSettings) + { + _services = services; + _dbSettings = dbSettings; + _agentSettings = agentSettings; + _conversationSettings = conversationSettings; + + _options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + AllowTrailingCommas = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + _dbSettings.FileRepository = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _dbSettings.FileRepository); + } + + private List _users = new List(); + private List _agents = new List(); + private List _userAgents = new List(); + private List _conversations = new List(); + private PluginConfig? _pluginConfig = null; + + private IQueryable Users + { + get + { + if (!_users.IsNullOrEmpty()) + { + return _users.AsQueryable(); + } + + var dir = Path.Combine(_dbSettings.FileRepository, "users"); + _users = new List(); + if (Directory.Exists(dir)) + { + foreach (var d in Directory.GetDirectories(dir)) + { + var userFile = Path.Combine(d, USER_FILE); + if (!Directory.Exists(d) || !File.Exists(userFile)) + continue; + + var json = File.ReadAllText(userFile); + _users.Add(JsonSerializer.Deserialize(json, _options)); + } + } + return _users.AsQueryable(); + } + } + + private IQueryable Agents + { + get + { + if (!_agents.IsNullOrEmpty()) + { + return _agents.AsQueryable(); + } + + var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir); + _agents = new List(); + if (Directory.Exists(dir)) + { + foreach (var d in Directory.GetDirectories(dir)) + { + var file = Path.Combine(d, AGENT_FILE); + if (!Directory.Exists(d) || !File.Exists(file)) + continue; + + var json = File.ReadAllText(file); + var agent = JsonSerializer.Deserialize(json, _options); + if (agent != null) + { + agent = agent.SetInstruction(FetchInstruction(d)) + .SetTemplates(FetchTemplates(d)) + .SetFunctions(FetchFunctions(d)) + .SetResponses(FetchResponses(d)) + .SetSamples(FetchSamples(d)); + _agents.Add(agent); + } + } + } + return _agents.AsQueryable(); + } + } + + private IQueryable UserAgents + { + get + { + if (!_userAgents.IsNullOrEmpty()) + { + return _userAgents.AsQueryable(); + } + + var dir = Path.Combine(_dbSettings.FileRepository, "users"); + _userAgents = new List(); + if (Directory.Exists(dir)) + { + foreach (var d in Directory.GetDirectories(dir)) + { + var file = Path.Combine(d, USER_AGENT_FILE); + if (!Directory.Exists(d) || !File.Exists(file)) + continue; + + var json = File.ReadAllText(file); + _userAgents.AddRange(JsonSerializer.Deserialize>(json, _options)); + } + } + return _userAgents.AsQueryable(); + } + } + + + #region Private methods + private string GetAgentDataDir(string agentId) + { + var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); + if (!Directory.Exists(dir)) + { + dir = string.Empty; + } + return dir; + } + + private (Agent?, string) GetAgentFromFile(string agentId) + { + var dir = GetAgentDataDir(agentId); + var agentFile = Path.Combine(dir, AGENT_FILE); + if (!File.Exists(agentFile)) return (null, string.Empty); + + var json = File.ReadAllText(agentFile); + var agent = JsonSerializer.Deserialize(json, _options); + return (agent, agentFile); + } + + private string FetchInstruction(string fileDir) + { + var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); + if (!File.Exists(file)) return string.Empty; + + var instruction = File.ReadAllText(file); + return instruction; + } + + private List FetchFunctions(string fileDir) + { + var file = Path.Combine(fileDir, AGENT_FUNCTIONS_FILE); + if (!File.Exists(file)) return new List(); + + var functionsJson = File.ReadAllText(file); + var functions = JsonSerializer.Deserialize>(functionsJson, _options); + return functions; + } + + private List FetchSamples(string fileDir) + { + var file = Path.Combine(fileDir, AGENT_SAMPLES_FILE); + if (!File.Exists(file)) return new List(); + + return File.ReadAllLines(file)?.ToList() ?? new List(); + } + + private List FetchTemplates(string fileDir) + { + var templates = new List(); + var templateDir = Path.Combine(fileDir, "templates"); + if (!Directory.Exists(templateDir)) return templates; + + foreach (var file in Directory.GetFiles(templateDir)) + { + var fileName = file.Split(Path.DirectorySeparatorChar).Last(); + var splits = fileName.ToLower().Split('.'); + var name = string.Join('.', splits.Take(splits.Length - 1)); + var extension = splits.Last(); + if (extension.Equals(_agentSettings.TemplateFormat, StringComparison.OrdinalIgnoreCase)) + { + var content = File.ReadAllText(file); + templates.Add(new AgentTemplate(name, content)); + } + } + + return templates; + } + + private List FetchResponses(string fileDir) + { + var responses = new List(); + var responseDir = Path.Combine(fileDir, "responses"); + if (!Directory.Exists(responseDir)) return responses; + + foreach (var file in Directory.GetFiles(responseDir)) + { + var fileName = file.Split(Path.DirectorySeparatorChar).Last(); + var splits = fileName.ToLower().Split('.'); + var prefix = splits[0]; + var intent = splits[1]; + var content = File.ReadAllText(file); + responses.Add(new AgentResponse(prefix, intent, content)); + } + + return responses; + } + + private string[] ParseFileNameByPath(string path, string separator = ".") + { + var name = path.Split(Path.DirectorySeparatorChar).Last(); + return name.Split(separator); + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/PluginDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/PluginDocument.cs new file mode 100644 index 00000000..a4c045c4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/PluginDocument.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class PluginDocument : MongoBase +{ + public List EnabledPlugins { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index 1f5cbb92..a3e994b4 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -51,4 +51,7 @@ public class MongoDbContext public IMongoCollection LlmCompletionLogs => Database.GetCollection($"{_collectionPrefix}_Llm_Completion_Logs"); + + public IMongoCollection Plugins + => Database.GetCollection($"{_collectionPrefix}_Plugins"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs new file mode 100644 index 00000000..8f9e4ce8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -0,0 +1,450 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Evaluations.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Routing.Models; +using BotSharp.Abstraction.Routing.Settings; +using BotSharp.Plugin.MongoStorage.Collections; +using BotSharp.Plugin.MongoStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public void UpdateAgent(Agent agent, AgentField field) + { + if (agent == null || string.IsNullOrEmpty(agent.Id)) return; + + switch (field) + { + case AgentField.Name: + UpdateAgentName(agent.Id, agent.Name); + break; + case AgentField.Description: + UpdateAgentDescription(agent.Id, agent.Description); + break; + case AgentField.IsPublic: + UpdateAgentIsPublic(agent.Id, agent.IsPublic); + break; + case AgentField.Disabled: + UpdateAgentDisabled(agent.Id, agent.Disabled); + break; + case AgentField.AllowRouting: + UpdateAgentAllowRouting(agent.Id, agent.AllowRouting); + break; + case AgentField.Profiles: + UpdateAgentProfiles(agent.Id, agent.Profiles); + break; + case AgentField.RoutingRule: + UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); + break; + case AgentField.Instruction: + UpdateAgentInstruction(agent.Id, agent.Instruction); + break; + case AgentField.Function: + UpdateAgentFunctions(agent.Id, agent.Functions); + break; + case AgentField.Template: + UpdateAgentTemplates(agent.Id, agent.Templates); + break; + case AgentField.Response: + UpdateAgentResponses(agent.Id, agent.Responses); + break; + case AgentField.Sample: + UpdateAgentSamples(agent.Id, agent.Samples); + break; + case AgentField.LlmConfig: + UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); + break; + case AgentField.All: + UpdateAgentAllFields(agent); + break; + default: + break; + } + } + + #region Update Agent Fields + private void UpdateAgentName(string agentId, string name) + { + if (string.IsNullOrEmpty(name)) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Name, name) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentDescription(string agentId, string description) + { + if (string.IsNullOrEmpty(description)) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Description, description) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentIsPublic(string agentId, bool isPublic) + { + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.IsPublic, isPublic) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentDisabled(string agentId, bool disabled) + { + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Disabled, disabled) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentAllowRouting(string agentId, bool allowRouting) + { + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.AllowRouting, allowRouting) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentProfiles(string agentId, List profiles) + { + if (profiles.IsNullOrEmpty()) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Profiles, profiles) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentRoutingRules(string agentId, List rules) + { + if (rules.IsNullOrEmpty()) return; + + var ruleElements = rules.Select(x => RoutingRuleMongoElement.ToMongoElement(x)).ToList(); + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.RoutingRules, ruleElements) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentInstruction(string agentId, string instruction) + { + if (string.IsNullOrEmpty(instruction)) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Instruction, instruction) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentFunctions(string agentId, List functions) + { + if (functions.IsNullOrEmpty()) return; + + var functionsToUpdate = functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList(); + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Functions, functionsToUpdate) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentTemplates(string agentId, List templates) + { + if (templates.IsNullOrEmpty()) return; + + var templatesToUpdate = templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList(); + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Templates, templatesToUpdate) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentResponses(string agentId, List responses) + { + if (responses.IsNullOrEmpty()) return; + + var responsesToUpdate = responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList(); + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Responses, responsesToUpdate) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentSamples(string agentId, List samples) + { + if (samples.IsNullOrEmpty()) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Samples, samples) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) + { + var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config); + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.LlmConfig, llmConfig) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + + private void UpdateAgentAllFields(Agent agent) + { + var filter = Builders.Filter.Eq(x => x.Id, agent.Id); + var update = Builders.Update + .Set(x => x.Name, agent.Name) + .Set(x => x.Description, agent.Description) + .Set(x => x.Disabled, agent.Disabled) + .Set(x => x.AllowRouting, agent.AllowRouting) + .Set(x => x.Profiles, agent.Profiles) + .Set(x => x.RoutingRules, agent.RoutingRules.Select(r => RoutingRuleMongoElement.ToMongoElement(r)).ToList()) + .Set(x => x.Instruction, agent.Instruction) + .Set(x => x.Templates, agent.Templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList()) + .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) + .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) + .Set(x => x.Samples, agent.Samples) + .Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig)) + .Set(x => x.IsPublic, agent.IsPublic) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + var res = _dc.Agents.UpdateOne(filter, update); + Console.WriteLine(); + } + #endregion + + + public Agent? GetAgent(string agentId) + { + var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); + if (agent == null) return null; + + return new Agent + { + Id = agent.Id, + Name = agent.Name, + Description = agent.Description, + Instruction = agent.Instruction, + Templates = !agent.Templates.IsNullOrEmpty() ? agent.Templates + .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) + .ToList() : new List(), + Functions = !agent.Functions.IsNullOrEmpty() ? agent.Functions + .Select(f => FunctionDefMongoElement.ToDomainElement(f)) + .ToList() : new List(), + Responses = !agent.Responses.IsNullOrEmpty() ? agent.Responses + .Select(r => AgentResponseMongoElement.ToDomainElement(r)) + .ToList() : new List(), + Samples = agent.Samples ?? new List(), + IsPublic = agent.IsPublic, + Disabled = agent.Disabled, + AllowRouting = agent.AllowRouting, + Profiles = agent.Profiles, + RoutingRules = !agent.RoutingRules.IsNullOrEmpty() ? agent.RoutingRules + .Select(r => RoutingRuleMongoElement.ToDomainElement(agent.Id, agent.Name, r)) + .ToList() : new List(), + LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agent.LlmConfig) + }; + } + + public List GetAgents(AgentFilter filter) + { + var agents = new List(); + IQueryable query = _dc.Agents.AsQueryable(); + + if (!string.IsNullOrEmpty(filter.AgentName)) + { + query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower()); + } + + if (filter.Disabled.HasValue) + { + query = query.Where(x => x.Disabled == filter.Disabled); + } + + if (filter.AllowRouting.HasValue) + { + query = query.Where(x => x.AllowRouting == filter.AllowRouting); + } + + if (filter.IsPublic.HasValue) + { + query = query.Where(x => x.IsPublic == filter.IsPublic); + } + + if (filter.IsRouter.HasValue) + { + var route = _services.GetRequiredService(); + query = filter.IsRouter.Value ? + query.Where(x => x.Id == route.AgentId) : + query.Where(x => x.Id != route.AgentId); + } + + if (filter.IsEvaluator.HasValue) + { + var evaluate = _services.GetRequiredService(); + query = filter.IsEvaluator.Value ? + query.Where(x => x.Id == evaluate.AgentId) : + query.Where(x => x.Id != evaluate.AgentId); + } + + if (filter.AgentIds != null) + { + query = query.Where(x => filter.AgentIds.Contains(x.Id)); + } + + return query.ToList().Select(x => new Agent + { + Id = x.Id, + Name = x.Name, + Description = x.Description, + Instruction = x.Instruction, + Templates = !x.Templates.IsNullOrEmpty() ? x.Templates + .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) + .ToList() : new List(), + Functions = !x.Functions.IsNullOrEmpty() ? x.Functions + .Select(f => FunctionDefMongoElement.ToDomainElement(f)) + .ToList() : new List(), + Responses = !x.Responses.IsNullOrEmpty() ? x.Responses + .Select(r => AgentResponseMongoElement.ToDomainElement(r)) + .ToList() : new List(), + Samples = x.Samples ?? new List(), + IsPublic = x.IsPublic, + Disabled = x.Disabled, + AllowRouting = x.AllowRouting, + Profiles = x.Profiles, + RoutingRules = !x.RoutingRules.IsNullOrEmpty() ? x.RoutingRules + .Select(r => RoutingRuleMongoElement.ToDomainElement(x.Id, x.Name, r)) + .ToList() : new List(), + LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(x.LlmConfig) + }).ToList(); + } + + public List GetAgentsByUser(string userId) + { + var agentIds = (from ua in _dc.UserAgents.AsQueryable() + join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id + where ua.UserId == userId || u.ExternalId == userId + select ua.AgentId).ToList(); + + var filter = new AgentFilter + { + AgentIds = agentIds, + IsPublic = true + }; + var agents = GetAgents(filter); + return agents; + } + + public List GetAgentResponses(string agentId, string prefix, string intent) + { + var responses = new List(); + var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); + if (agent == null) return responses; + + return agent.Responses.Where(x => x.Prefix == prefix && x.Intent == intent).Select(x => x.Content).ToList(); + } + + public string GetAgentTemplate(string agentId, string templateName) + { + var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); + if (agent == null) return string.Empty; + + return agent.Templates?.FirstOrDefault(x => x.Name == templateName.ToLower())?.Content ?? string.Empty; + } + + public void BulkInsertAgents(List agents) + { + if (agents.IsNullOrEmpty()) return; + + var agentDocs = agents.Select(x => new AgentDocument + { + Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), + Name = x.Name, + Description = x.Description, + Instruction = x.Instruction, + Templates = x.Templates? + .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? + .ToList() ?? new List(), + Functions = x.Functions? + .Select(f => FunctionDefMongoElement.ToMongoElement(f))? + .ToList() ?? new List(), + Responses = x.Responses? + .Select(r => AgentResponseMongoElement.ToMongoElement(r))? + .ToList() ?? new List(), + Samples = x.Samples ?? new List(), + IsPublic = x.IsPublic, + AllowRouting = x.AllowRouting, + Disabled = x.Disabled, + Profiles = x.Profiles, + RoutingRules = x.RoutingRules? + .Select(r => RoutingRuleMongoElement.ToMongoElement(r))? + .ToList() ?? new List(), + LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), + CreatedTime = x.CreatedDateTime, + UpdatedTime = x.UpdatedDateTime + }).ToList(); + + _dc.Agents.InsertMany(agentDocs); + } + + public void BulkInsertUserAgents(List userAgents) + { + if (userAgents.IsNullOrEmpty()) return; + + var userAgentDocs = userAgents.Select(x => new UserAgentDocument + { + Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), + AgentId = x.AgentId, + UserId = !string.IsNullOrEmpty(x.UserId) ? x.UserId : string.Empty, + Editable = x.Editable, + CreatedTime = x.CreatedTime, + UpdatedTime = x.UpdatedTime + }).ToList(); + + _dc.UserAgents.InsertMany(userAgentDocs); + } + + public bool DeleteAgents() + { + try + { + _dc.UserAgents.DeleteMany(Builders.Filter.Empty); + _dc.Agents.DeleteMany(Builders.Filter.Empty); + return true; + } + catch + { + return false; + } + + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs new file mode 100644 index 00000000..ada3ee5a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -0,0 +1,257 @@ +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; +using BotSharp.Plugin.MongoStorage.Collections; +using BotSharp.Plugin.MongoStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public void CreateNewConversation(Conversation conversation) + { + if (conversation == null) return; + + var convDoc = new ConversationDocument + { + Id = !string.IsNullOrEmpty(conversation.Id) ? conversation.Id : Guid.NewGuid().ToString(), + AgentId = conversation.AgentId, + UserId = !string.IsNullOrEmpty(conversation.UserId) ? conversation.UserId : string.Empty, + Title = conversation.Title, + Channel = conversation.Channel, + Status = conversation.Status, + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow, + }; + + var dialogDoc = new ConversationDialogDocument + { + Id = Guid.NewGuid().ToString(), + ConversationId = convDoc.Id, + Dialogs = new List() + }; + + var states = conversation.States ?? new Dictionary(); + var initialStates = states.Select(x => new StateMongoElement + { + Key = x.Key, + Values = new List + { + new StateValueMongoElement { Data = x.Value, UpdateTime = DateTime.UtcNow } + } + }).ToList(); + + var stateDoc = new ConversationStateDocument + { + Id = Guid.NewGuid().ToString(), + ConversationId = convDoc.Id, + States = initialStates + }; + + _dc.Conversations.InsertOne(convDoc); + _dc.ConversationDialogs.InsertOne(dialogDoc); + _dc.ConversationStates.InsertOne(stateDoc); + } + + public bool DeleteConversation(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); + var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + + var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); + var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); + var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); + var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); + var convDeleted = _dc.Conversations.DeleteMany(filterConv); + return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 + || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; + } + + public List GetConversationDialogs(string conversationId) + { + var dialogs = new List(); + if (string.IsNullOrEmpty(conversationId)) return dialogs; + + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundDialog = _dc.ConversationDialogs.Find(filter).FirstOrDefault(); + if (foundDialog == null) return dialogs; + + var formattedDialog = foundDialog.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList(); + return formattedDialog ?? new List(); + } + + public void UpdateConversationDialogElements(string conversationId, List updateElements) + { + if (string.IsNullOrEmpty(conversationId) || updateElements.IsNullOrEmpty()) return; + + var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); + if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) return; + + foundDialog.Dialogs = foundDialog.Dialogs.Select((x, idx) => + { + var found = updateElements.FirstOrDefault(e => e.Index == idx); + if (found != null) + { + x.Content = found.UpdateContent; + } + return x; + }).ToList(); + + _dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog); + } + + public void AppendConversationDialogs(string conversationId, List dialogs) + { + if (string.IsNullOrEmpty(conversationId)) return; + + var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); + var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); + var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); + var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.ConversationDialogs.UpdateOne(filterDialog, updateDialog); + _dc.Conversations.UpdateOne(filterConv, updateConv); + } + + public void UpdateConversationTitle(string conversationId, string title) + { + if (string.IsNullOrEmpty(conversationId)) return; + + var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); + var updateConv = Builders.Update + .Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.Title, title); + + _dc.Conversations.UpdateOne(filterConv, updateConv); + } + + public ConversationState GetConversationStates(string conversationId) + { + var states = new ConversationState(); + if (string.IsNullOrEmpty(conversationId)) return states; + + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(filter).FirstOrDefault(); + if (foundStates == null || foundStates.States.IsNullOrEmpty()) return states; + + var savedStates = foundStates.States.Select(x => StateMongoElement.ToDomainElement(x)).ToList(); + return new ConversationState(savedStates); + } + + public void UpdateConversationStates(string conversationId, List states) + { + if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; + + var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); + var updateStates = Builders.Update.Set(x => x.States, saveStates); + + _dc.ConversationStates.UpdateOne(filterStates, updateStates); + } + + public void UpdateConversationStatus(string conversationId, string status) + { + if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(status)) return; + + var filter = Builders.Filter.Eq(x => x.Id, conversationId); + var update = Builders.Update + .Set(x => x.Status, status) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Conversations.UpdateOne(filter, update); + } + + public Conversation GetConversation(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return null; + + var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); + var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); + + var conv = _dc.Conversations.Find(filterConv).FirstOrDefault(); + var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); + var states = _dc.ConversationStates.Find(filterState).FirstOrDefault(); + + if (conv == null) return null; + + var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); + var curStates = new Dictionary(); + states.States.ForEach(x => + { + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; + }); + + return new Conversation + { + Id = conv.Id.ToString(), + AgentId = conv.AgentId.ToString(), + UserId = conv.UserId.ToString(), + Title = conv.Title, + Channel = conv.Channel, + Status = conv.Status, + Dialogs = dialogElements, + States = curStates, + CreatedTime = conv.CreatedTime, + UpdatedTime = conv.UpdatedTime + }; + } + + public List GetConversations(ConversationFilter filter) + { + var records = new List(); + var builder = Builders.Filter; + var filters = new List>(); + + if (!string.IsNullOrEmpty(filter.AgentId)) filters.Add(builder.Eq(x => x.AgentId, filter.AgentId)); + if (!string.IsNullOrEmpty(filter.Status)) filters.Add(builder.Eq(x => x.Status, filter.Status)); + if (!string.IsNullOrEmpty(filter.Channel)) filters.Add(builder.Eq(x => x.Channel, filter.Channel)); + if (!string.IsNullOrEmpty(filter.UserId)) filters.Add(builder.Eq(x => x.UserId, filter.UserId)); + + var conversations = _dc.Conversations.Find(builder.And(filters)).ToList(); + + foreach (var conv in conversations) + { + var convId = conv.Id.ToString(); + records.Add(new Conversation + { + Id = convId, + AgentId = conv.AgentId.ToString(), + UserId = conv.UserId.ToString(), + Title = conv.Title, + Channel = conv.Channel, + Status = conv.Status, + CreatedTime = conv.CreatedTime, + UpdatedTime = conv.UpdatedTime + }); + } + + return records; + } + + public List GetLastConversations() + { + var records = new List(); + var conversations = _dc.Conversations.Aggregate() + .Group(c => c.UserId, g => g.OrderByDescending(x => x.CreatedTime).First()) + .ToList(); + return conversations.Select(c => new Conversation() + { + Id = c.Id.ToString(), + AgentId = c.AgentId.ToString(), + UserId = c.UserId.ToString(), + Title = c.Title, + Channel = c.Channel, + Status = c.Status, + CreatedTime = c.CreatedTime, + UpdatedTime = c.UpdatedTime + }).ToList(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs new file mode 100644 index 00000000..20266450 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -0,0 +1,61 @@ +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Plugin.MongoStorage.Collections; +using BotSharp.Plugin.MongoStorage.Models; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + #region Execution Log + public void AddExecutionLogs(string conversationId, List logs) + { + if (string.IsNullOrEmpty(conversationId) || logs.IsNullOrEmpty()) return; + + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var update = Builders.Update + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) + .PushEach(x => x.Logs, logs); + + _dc.ExectionLogs.UpdateOne(filter, update, _options); + } + + public List GetExecutionLogs(string conversationId) + { + var logs = new List(); + if (string.IsNullOrEmpty(conversationId)) return logs; + + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var logCollection = _dc.ExectionLogs.Find(filter).FirstOrDefault(); + + logs = logCollection?.Logs ?? new List(); + return logs; + } + #endregion + + #region LLM Completion Log + public void SaveLlmCompletionLog(LlmCompletionLog log) + { + if (log == null) return; + + var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); + + var logElement = new PromptLogMongoElement + { + MessageId = messageId, + AgentId = log.AgentId, + Prompt = log.Prompt, + Response = log.Response, + CreateDateTime = log.CreateDateTime + }; + + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var update = Builders.Update + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) + .Push(x => x.Logs, logElement); + + _dc.LlmCompletionLogs.UpdateOne(filter, update, _options); + } + + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs new file mode 100644 index 00000000..c6bc60b3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs @@ -0,0 +1,35 @@ +using BotSharp.Abstraction.Plugins.Models; +using BotSharp.Plugin.MongoStorage.Collections; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + #region Plugin + public PluginConfig GetPluginConfig() + { + var config = new PluginConfig(); + var found = _dc.Plugins.AsQueryable().FirstOrDefault(); + if (found != null) + { + config = new PluginConfig() + { + EnabledPlugins = found.EnabledPlugins + }; + } + return config; + } + + public void SavePluginConfig(PluginConfig config) + { + if (config == null || config.EnabledPlugins == null) return; + + var filter = Builders.Filter.Empty; + var update = Builders.Update + .Set(x => x.EnabledPlugins, config.EnabledPlugins) + .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()); + + _dc.Plugins.UpdateOne(filter, update, _options); + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs new file mode 100644 index 00000000..aa65f4c8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs @@ -0,0 +1,151 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Users.Models; +using BotSharp.Plugin.MongoStorage.Collections; +using BotSharp.Plugin.MongoStorage.Models; + + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public void Add(object entity) + { + if (entity is Agent agent) + { + _agents.Add(agent); + _changedTableNames.Add(nameof(Agent)); + } + else if (entity is User user) + { + _users.Add(user); + _changedTableNames.Add(nameof(User)); + } + else if (entity is UserAgent userAgent) + { + _userAgents.Add(userAgent); + _changedTableNames.Add(nameof(UserAgent)); + } + } + + public int Transaction(Action action) + { + _changedTableNames.Clear(); + action(); + + foreach (var table in _changedTableNames) + { + if (table == nameof(Agent)) + { + var agents = _agents.Select(x => new AgentDocument + { + Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), + Name = x.Name, + Description = x.Description, + Instruction = x.Instruction, + Templates = x.Templates? + .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? + .ToList() ?? new List(), + Functions = x.Functions? + .Select(f => FunctionDefMongoElement.ToMongoElement(f))? + .ToList() ?? new List(), + Responses = x.Responses? + .Select(r => AgentResponseMongoElement.ToMongoElement(r))? + .ToList() ?? new List(), + Samples = x.Samples ?? new List(), + IsPublic = x.IsPublic, + AllowRouting = x.AllowRouting, + Disabled = x.Disabled, + Profiles = x.Profiles, + RoutingRules = x.RoutingRules? + .Select(r => RoutingRuleMongoElement.ToMongoElement(r))? + .ToList() ?? new List(), + LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), + CreatedTime = x.CreatedDateTime, + UpdatedTime = x.UpdatedDateTime + }).ToList(); + + foreach (var agent in agents) + { + var filter = Builders.Filter.Eq(x => x.Id, agent.Id); + var update = Builders.Update + .Set(x => x.Name, agent.Name) + .Set(x => x.Description, agent.Description) + .Set(x => x.Instruction, agent.Instruction) + .Set(x => x.Templates, agent.Templates) + .Set(x => x.Functions, agent.Functions) + .Set(x => x.Responses, agent.Responses) + .Set(x => x.Samples, agent.Samples) + .Set(x => x.IsPublic, agent.IsPublic) + .Set(x => x.AllowRouting, agent.AllowRouting) + .Set(x => x.Disabled, agent.Disabled) + .Set(x => x.Profiles, agent.Profiles) + .Set(x => x.RoutingRules, agent.RoutingRules) + .Set(x => x.LlmConfig, agent.LlmConfig) + .Set(x => x.CreatedTime, agent.CreatedTime) + .Set(x => x.UpdatedTime, agent.UpdatedTime); + _dc.Agents.UpdateOne(filter, update, _options); + } + } + else if (table == nameof(User)) + { + var users = _users.Select(x => new UserDocument + { + Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), + UserName = x.UserName, + FirstName = x.FirstName, + LastName = x.LastName, + Salt = x.Salt, + Password = x.Password, + Email = x.Email, + ExternalId = x.ExternalId, + Role = x.Role, + CreatedTime = x.CreatedTime, + UpdatedTime = x.UpdatedTime + }).ToList(); + + foreach (var user in users) + { + var filter = Builders.Filter.Eq(x => x.Id, user.Id); + var update = Builders.Update + .Set(x => x.UserName, user.UserName) + .Set(x => x.FirstName, user.FirstName) + .Set(x => x.LastName, user.LastName) + .Set(x => x.Email, user.Email) + .Set(x => x.Salt, user.Salt) + .Set(x => x.Password, user.Password) + .Set(x => x.ExternalId, user.ExternalId) + .Set(x => x.Role, user.Role) + .Set(x => x.CreatedTime, user.CreatedTime) + .Set(x => x.UpdatedTime, user.UpdatedTime); + _dc.Users.UpdateOne(filter, update, _options); + } + } + else if (table == nameof(UserAgent)) + { + var userAgents = _userAgents.Select(x => new UserAgentDocument + { + Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), + AgentId = x.AgentId, + UserId = !string.IsNullOrEmpty(x.UserId) ? x.UserId : string.Empty, + Editable = x.Editable, + CreatedTime = x.CreatedTime, + UpdatedTime = x.UpdatedTime + }).ToList(); + + foreach (var userAgent in userAgents) + { + var filter = Builders.Filter.Eq(x => x.Id, userAgent.Id); + var update = Builders.Update + .Set(x => x.AgentId, userAgent.AgentId) + .Set(x => x.UserId, userAgent.UserId) + .Set(x => x.Editable, userAgent.Editable) + .Set(x => x.CreatedTime, userAgent.CreatedTime) + .Set(x => x.UpdatedTime, userAgent.UpdatedTime); + _dc.UserAgents.UpdateOne(filter, update, _options); + } + } + } + + return _changedTableNames.Count; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs new file mode 100644 index 00000000..c9fc4539 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -0,0 +1,63 @@ +using BotSharp.Abstraction.Users.Models; +using BotSharp.Plugin.MongoStorage.Collections; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public User? GetUserByEmail(string email) + { + var user = _dc.Users.AsQueryable().FirstOrDefault(x => x.Email == email); + return user != null ? new User + { + Id = user.Id, + UserName = user.UserName, + FirstName = user.FirstName, + LastName = user.LastName, + Email = user.Email, + Password = user.Password, + Salt = user.Salt, + ExternalId = user.ExternalId, + Role = user.Role + } : null; + } + + public User? GetUserById(string id) + { + var user = _dc.Users.AsQueryable().FirstOrDefault(x => x.Id == id || x.ExternalId == id); + return user != null ? new User + { + Id = user.Id, + UserName = user.UserName, + FirstName = user.FirstName, + LastName = user.LastName, + Email = user.Email, + Password = user.Password, + Salt = user.Salt, + ExternalId = user.ExternalId, + Role = user.Role + } : null; + } + + public void CreateUser(User user) + { + if (user == null) return; + + var userCollection = new UserDocument + { + Id = Guid.NewGuid().ToString(), + UserName = user.UserName, + FirstName = user.FirstName, + LastName = user.LastName, + Salt = user.Salt, + Password = user.Password, + Email = user.Email, + ExternalId = user.ExternalId, + Role = user.Role, + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow + }; + + _dc.Users.InsertOne(userCollection); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 9ad633ed..02d21c43 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -1,19 +1,10 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Evaluations.Settings; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Repositories.Filters; -using BotSharp.Abstraction.Repositories.Models; -using BotSharp.Abstraction.Routing.Models; -using BotSharp.Abstraction.Routing.Settings; using BotSharp.Abstraction.Users.Models; -using BotSharp.Plugin.MongoStorage.Collections; -using BotSharp.Plugin.MongoStorage.Models; namespace BotSharp.Plugin.MongoStorage.Repository; -public class MongoRepository : IBotSharpRepository +public partial class MongoRepository : IBotSharpRepository { private readonly MongoDbContext _dc; private readonly IServiceProvider _services; @@ -33,957 +24,5 @@ public class MongoRepository : IBotSharpRepository private List _users = new List(); private List _userAgents = new List(); private List _conversations = new List(); - List _changedTableNames = new List(); - - public void Add(object entity) - { - if (entity is Agent agent) - { - _agents.Add(agent); - _changedTableNames.Add(nameof(Agent)); - } - else if (entity is User user) - { - _users.Add(user); - _changedTableNames.Add(nameof(User)); - } - else if (entity is UserAgent userAgent) - { - _userAgents.Add(userAgent); - _changedTableNames.Add(nameof(UserAgent)); - } - } - - public int Transaction(Action action) - { - _changedTableNames.Clear(); - action(); - - foreach (var table in _changedTableNames) - { - if (table == nameof(Agent)) - { - var agents = _agents.Select(x => new AgentDocument - { - Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), - Name = x.Name, - Description = x.Description, - Instruction = x.Instruction, - Templates = x.Templates? - .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? - .ToList() ?? new List(), - Functions = x.Functions? - .Select(f => FunctionDefMongoElement.ToMongoElement(f))? - .ToList() ?? new List(), - Responses = x.Responses? - .Select(r => AgentResponseMongoElement.ToMongoElement(r))? - .ToList() ?? new List(), - Samples = x.Samples ?? new List(), - IsPublic = x.IsPublic, - AllowRouting = x.AllowRouting, - Disabled = x.Disabled, - Profiles = x.Profiles, - RoutingRules = x.RoutingRules? - .Select(r => RoutingRuleMongoElement.ToMongoElement(r))? - .ToList() ?? new List(), - LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), - CreatedTime = x.CreatedDateTime, - UpdatedTime = x.UpdatedDateTime - }).ToList(); - - foreach (var agent in agents) - { - var filter = Builders.Filter.Eq(x => x.Id, agent.Id); - var update = Builders.Update - .Set(x => x.Name, agent.Name) - .Set(x => x.Description, agent.Description) - .Set(x => x.Instruction, agent.Instruction) - .Set(x => x.Templates, agent.Templates) - .Set(x => x.Functions, agent.Functions) - .Set(x => x.Responses, agent.Responses) - .Set(x => x.Samples, agent.Samples) - .Set(x => x.IsPublic, agent.IsPublic) - .Set(x => x.AllowRouting, agent.AllowRouting) - .Set(x => x.Disabled, agent.Disabled) - .Set(x => x.Profiles, agent.Profiles) - .Set(x => x.RoutingRules, agent.RoutingRules) - .Set(x => x.LlmConfig, agent.LlmConfig) - .Set(x => x.CreatedTime, agent.CreatedTime) - .Set(x => x.UpdatedTime, agent.UpdatedTime); - _dc.Agents.UpdateOne(filter, update, _options); - } - } - else if (table == nameof(User)) - { - var users = _users.Select(x => new UserDocument - { - Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), - UserName = x.UserName, - FirstName = x.FirstName, - LastName = x.LastName, - Salt = x.Salt, - Password = x.Password, - Email = x.Email, - ExternalId = x.ExternalId, - Role = x.Role, - CreatedTime = x.CreatedTime, - UpdatedTime = x.UpdatedTime - }).ToList(); - - foreach (var user in users) - { - var filter = Builders.Filter.Eq(x => x.Id, user.Id); - var update = Builders.Update - .Set(x => x.UserName, user.UserName) - .Set(x => x.FirstName, user.FirstName) - .Set(x => x.LastName, user.LastName) - .Set(x => x.Email, user.Email) - .Set(x => x.Salt, user.Salt) - .Set(x => x.Password, user.Password) - .Set(x => x.ExternalId, user.ExternalId) - .Set(x => x.Role, user.Role) - .Set(x => x.CreatedTime, user.CreatedTime) - .Set(x => x.UpdatedTime, user.UpdatedTime); - _dc.Users.UpdateOne(filter, update, _options); - } - } - else if (table == nameof(UserAgent)) - { - var userAgents = _userAgents.Select(x => new UserAgentDocument - { - Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), - AgentId = x.AgentId, - UserId = !string.IsNullOrEmpty(x.UserId) ? x.UserId : string.Empty, - Editable = x.Editable, - CreatedTime = x.CreatedTime, - UpdatedTime = x.UpdatedTime - }).ToList(); - - foreach (var userAgent in userAgents) - { - var filter = Builders.Filter.Eq(x => x.Id, userAgent.Id); - var update = Builders.Update - .Set(x => x.AgentId, userAgent.AgentId) - .Set(x => x.UserId, userAgent.UserId) - .Set(x => x.Editable, userAgent.Editable) - .Set(x => x.CreatedTime, userAgent.CreatedTime) - .Set(x => x.UpdatedTime, userAgent.UpdatedTime); - _dc.UserAgents.UpdateOne(filter, update, _options); - } - } - } - - return _changedTableNames.Count; - } - - #region Plugin - public PluginConfig GetPluginConfig() - { - return new PluginConfig(); - } - - public void SavePluginConfig(PluginConfig config) - { - - } - #endregion - - #region Agent - public void UpdateAgent(Agent agent, AgentField field) - { - if (agent == null || string.IsNullOrEmpty(agent.Id)) return; - - switch (field) - { - case AgentField.Name: - UpdateAgentName(agent.Id, agent.Name); - break; - case AgentField.Description: - UpdateAgentDescription(agent.Id, agent.Description); - break; - case AgentField.IsPublic: - UpdateAgentIsPublic(agent.Id, agent.IsPublic); - break; - case AgentField.Disabled: - UpdateAgentDisabled(agent.Id, agent.Disabled); - break; - case AgentField.AllowRouting: - UpdateAgentAllowRouting(agent.Id, agent.AllowRouting); - break; - case AgentField.Profiles: - UpdateAgentProfiles(agent.Id, agent.Profiles); - break; - case AgentField.RoutingRule: - UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); - break; - case AgentField.Instruction: - UpdateAgentInstruction(agent.Id, agent.Instruction); - break; - case AgentField.Function: - UpdateAgentFunctions(agent.Id, agent.Functions); - break; - case AgentField.Template: - UpdateAgentTemplates(agent.Id, agent.Templates); - break; - case AgentField.Response: - UpdateAgentResponses(agent.Id, agent.Responses); - break; - case AgentField.Sample: - UpdateAgentSamples(agent.Id, agent.Samples); - break; - case AgentField.LlmConfig: - UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); - break; - case AgentField.All: - UpdateAgentAllFields(agent); - break; - default: - break; - } - } - - #region Update Agent Fields - private void UpdateAgentName(string agentId, string name) - { - if (string.IsNullOrEmpty(name)) return; - - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Name, name) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentDescription(string agentId, string description) - { - if (string.IsNullOrEmpty(description)) return; - - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Description, description) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentIsPublic(string agentId, bool isPublic) - { - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.IsPublic, isPublic) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentDisabled(string agentId, bool disabled) - { - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Disabled, disabled) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentAllowRouting(string agentId, bool allowRouting) - { - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.AllowRouting, allowRouting) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentProfiles(string agentId, List profiles) - { - if (profiles.IsNullOrEmpty()) return; - - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Profiles, profiles) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentRoutingRules(string agentId, List rules) - { - if (rules.IsNullOrEmpty()) return; - - var ruleElements = rules.Select(x => RoutingRuleMongoElement.ToMongoElement(x)).ToList(); - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.RoutingRules, ruleElements) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentInstruction(string agentId, string instruction) - { - if (string.IsNullOrEmpty(instruction)) return; - - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Instruction, instruction) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentFunctions(string agentId, List functions) - { - if (functions.IsNullOrEmpty()) return; - - var functionsToUpdate = functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList(); - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Functions, functionsToUpdate) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentTemplates(string agentId, List templates) - { - if (templates.IsNullOrEmpty()) return; - - var templatesToUpdate = templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList(); - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Templates, templatesToUpdate) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentResponses(string agentId, List responses) - { - if (responses.IsNullOrEmpty()) return; - - var responsesToUpdate = responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList(); - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Responses, responsesToUpdate) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentSamples(string agentId, List samples) - { - if (samples.IsNullOrEmpty()) return; - - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.Samples, samples) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) - { - var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config); - var filter = Builders.Filter.Eq(x => x.Id, agentId); - var update = Builders.Update - .Set(x => x.LlmConfig, llmConfig) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Agents.UpdateOne(filter, update); - } - - private void UpdateAgentAllFields(Agent agent) - { - var filter = Builders.Filter.Eq(x => x.Id, agent.Id); - var update = Builders.Update - .Set(x => x.Name, agent.Name) - .Set(x => x.Description, agent.Description) - .Set(x => x.Disabled, agent.Disabled) - .Set(x => x.AllowRouting, agent.AllowRouting) - .Set(x => x.Profiles, agent.Profiles) - .Set(x => x.RoutingRules, agent.RoutingRules.Select(r => RoutingRuleMongoElement.ToMongoElement(r)).ToList()) - .Set(x => x.Instruction, agent.Instruction) - .Set(x => x.Templates, agent.Templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList()) - .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) - .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) - .Set(x => x.Samples, agent.Samples) - .Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig)) - .Set(x => x.IsPublic, agent.IsPublic) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - var res = _dc.Agents.UpdateOne(filter, update); - Console.WriteLine(); - } - #endregion - - - public Agent? GetAgent(string agentId) - { - var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); - if (agent == null) return null; - - return new Agent - { - Id = agent.Id, - Name = agent.Name, - Description = agent.Description, - Instruction = agent.Instruction, - Templates = !agent.Templates.IsNullOrEmpty() ? agent.Templates - .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) - .ToList() : new List(), - Functions = !agent.Functions.IsNullOrEmpty() ? agent.Functions - .Select(f => FunctionDefMongoElement.ToDomainElement(f)) - .ToList() : new List(), - Responses = !agent.Responses.IsNullOrEmpty() ? agent.Responses - .Select(r => AgentResponseMongoElement.ToDomainElement(r)) - .ToList() : new List(), - Samples = agent.Samples ?? new List(), - IsPublic = agent.IsPublic, - Disabled = agent.Disabled, - AllowRouting = agent.AllowRouting, - Profiles = agent.Profiles, - RoutingRules = !agent.RoutingRules.IsNullOrEmpty() ? agent.RoutingRules - .Select(r => RoutingRuleMongoElement.ToDomainElement(agent.Id, agent.Name, r)) - .ToList() : new List(), - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agent.LlmConfig) - }; - } - - public List GetAgents(AgentFilter filter) - { - var agents = new List(); - IQueryable query = _dc.Agents.AsQueryable(); - - if (!string.IsNullOrEmpty(filter.AgentName)) - { - query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower()); - } - - if (filter.Disabled.HasValue) - { - query = query.Where(x => x.Disabled == filter.Disabled); - } - - if (filter.AllowRouting.HasValue) - { - query = query.Where(x => x.AllowRouting == filter.AllowRouting); - } - - if (filter.IsPublic.HasValue) - { - query = query.Where(x => x.IsPublic == filter.IsPublic); - } - - if (filter.IsRouter.HasValue) - { - var route = _services.GetRequiredService(); - query = filter.IsRouter.Value ? - query.Where(x => x.Id == route.AgentId) : - query.Where(x => x.Id != route.AgentId); - } - - if (filter.IsEvaluator.HasValue) - { - var evaluate = _services.GetRequiredService(); - query = filter.IsEvaluator.Value ? - query.Where(x => x.Id == evaluate.AgentId) : - query.Where(x => x.Id != evaluate.AgentId); - } - - if (filter.AgentIds != null) - { - query = query.Where(x => filter.AgentIds.Contains(x.Id)); - } - - return query.ToList().Select(x => new Agent - { - Id = x.Id, - Name = x.Name, - Description = x.Description, - Instruction = x.Instruction, - Templates = !x.Templates.IsNullOrEmpty() ? x.Templates - .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) - .ToList() : new List(), - Functions = !x.Functions.IsNullOrEmpty() ? x.Functions - .Select(f => FunctionDefMongoElement.ToDomainElement(f)) - .ToList() : new List(), - Responses = !x.Responses.IsNullOrEmpty() ? x.Responses - .Select(r => AgentResponseMongoElement.ToDomainElement(r)) - .ToList() : new List(), - Samples = x.Samples ?? new List(), - IsPublic = x.IsPublic, - Disabled = x.Disabled, - AllowRouting = x.AllowRouting, - Profiles = x.Profiles, - RoutingRules = !x.RoutingRules.IsNullOrEmpty() ? x.RoutingRules - .Select(r => RoutingRuleMongoElement.ToDomainElement(x.Id, x.Name, r)) - .ToList() : new List(), - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(x.LlmConfig) - }).ToList(); - } - - public List GetAgentsByUser(string userId) - { - var agentIds = (from ua in _dc.UserAgents.AsQueryable() - join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id - where ua.UserId == userId || u.ExternalId == userId - select ua.AgentId).ToList(); - - var filter = new AgentFilter - { - AgentIds = agentIds, - IsPublic = true - }; - var agents = GetAgents(filter); - return agents; - } - - public List GetAgentResponses(string agentId, string prefix, string intent) - { - var responses = new List(); - var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); - if (agent == null) return responses; - - return agent.Responses.Where(x => x.Prefix == prefix && x.Intent == intent).Select(x => x.Content).ToList(); - } - - public string GetAgentTemplate(string agentId, string templateName) - { - var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); - if (agent == null) return string.Empty; - - return agent.Templates?.FirstOrDefault(x => x.Name == templateName.ToLower())?.Content ?? string.Empty; - } - - public void BulkInsertAgents(List agents) - { - if (agents.IsNullOrEmpty()) return; - - var agentDocs = agents.Select(x => new AgentDocument - { - Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), - Name = x.Name, - Description = x.Description, - Instruction = x.Instruction, - Templates = x.Templates? - .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? - .ToList() ?? new List(), - Functions = x.Functions? - .Select(f => FunctionDefMongoElement.ToMongoElement(f))? - .ToList() ?? new List(), - Responses = x.Responses? - .Select(r => AgentResponseMongoElement.ToMongoElement(r))? - .ToList() ?? new List(), - Samples = x.Samples ?? new List(), - IsPublic = x.IsPublic, - AllowRouting = x.AllowRouting, - Disabled = x.Disabled, - Profiles = x.Profiles, - RoutingRules = x.RoutingRules? - .Select(r => RoutingRuleMongoElement.ToMongoElement(r))? - .ToList() ?? new List(), - LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), - CreatedTime = x.CreatedDateTime, - UpdatedTime = x.UpdatedDateTime - }).ToList(); - - _dc.Agents.InsertMany(agentDocs); - } - - public void BulkInsertUserAgents(List userAgents) - { - if (userAgents.IsNullOrEmpty()) return; - - var userAgentDocs = userAgents.Select(x => new UserAgentDocument - { - Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), - AgentId = x.AgentId, - UserId = !string.IsNullOrEmpty(x.UserId) ? x.UserId : string.Empty, - Editable = x.Editable, - CreatedTime = x.CreatedTime, - UpdatedTime = x.UpdatedTime - }).ToList(); - - _dc.UserAgents.InsertMany(userAgentDocs); - } - - public bool DeleteAgents() - { - try - { - _dc.UserAgents.DeleteMany(Builders.Filter.Empty); - _dc.Agents.DeleteMany(Builders.Filter.Empty); - return true; - } - catch - { - return false; - } - - } - #endregion - - #region Conversation - public void CreateNewConversation(Conversation conversation) - { - if (conversation == null) return; - - var convDoc = new ConversationDocument - { - Id = !string.IsNullOrEmpty(conversation.Id) ? conversation.Id : Guid.NewGuid().ToString(), - AgentId = conversation.AgentId, - UserId = !string.IsNullOrEmpty(conversation.UserId) ? conversation.UserId : string.Empty, - Title = conversation.Title, - Channel = conversation.Channel, - Status = conversation.Status, - CreatedTime = DateTime.UtcNow, - UpdatedTime = DateTime.UtcNow, - }; - - var dialogDoc = new ConversationDialogDocument - { - Id = Guid.NewGuid().ToString(), - ConversationId = convDoc.Id, - Dialogs = new List() - }; - - var states = conversation.States ?? new Dictionary(); - var initialStates = states.Select(x => new StateMongoElement - { - Key = x.Key, - Values = new List - { - new StateValueMongoElement { Data = x.Value, UpdateTime = DateTime.UtcNow } - } - }).ToList(); - - var stateDoc = new ConversationStateDocument - { - Id = Guid.NewGuid().ToString(), - ConversationId = convDoc.Id, - States = initialStates - }; - - _dc.Conversations.InsertOne(convDoc); - _dc.ConversationDialogs.InsertOne(dialogDoc); - _dc.ConversationStates.InsertOne(stateDoc); - } - - public bool DeleteConversation(string conversationId) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterSates = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterExeLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterPromptLog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - - var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog); - var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog); - var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates); - var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog); - var convDeleted = _dc.Conversations.DeleteMany(filterConv); - return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0 - || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0; - } - - public List GetConversationDialogs(string conversationId) - { - var dialogs = new List(); - if (string.IsNullOrEmpty(conversationId)) return dialogs; - - var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var foundDialog = _dc.ConversationDialogs.Find(filter).FirstOrDefault(); - if (foundDialog == null) return dialogs; - - var formattedDialog = foundDialog.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList(); - return formattedDialog ?? new List(); - } - - public void UpdateConversationDialogElements(string conversationId, List updateElements) - { - if (string.IsNullOrEmpty(conversationId) || updateElements.IsNullOrEmpty()) return; - - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); - if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) return; - - foundDialog.Dialogs = foundDialog.Dialogs.Select((x, idx) => - { - var found = updateElements.FirstOrDefault(e => e.Index == idx); - if (found != null) - { - x.Content = found.UpdateContent; - } - return x; - }).ToList(); - - _dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog); - } - - public void AppendConversationDialogs(string conversationId, List dialogs) - { - if (string.IsNullOrEmpty(conversationId)) return; - - var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); - var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); - var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.ConversationDialogs.UpdateOne(filterDialog, updateDialog); - _dc.Conversations.UpdateOne(filterConv, updateConv); - } - - public void UpdateConversationTitle(string conversationId, string title) - { - if (string.IsNullOrEmpty(conversationId)) return; - - var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var updateConv = Builders.Update - .Set(x => x.UpdatedTime, DateTime.UtcNow) - .Set(x => x.Title, title); - - _dc.Conversations.UpdateOne(filterConv, updateConv); - } - - public ConversationState GetConversationStates(string conversationId) - { - var states = new ConversationState(); - if (string.IsNullOrEmpty(conversationId)) return states; - - var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var foundStates = _dc.ConversationStates.Find(filter).FirstOrDefault(); - if (foundStates == null || foundStates.States.IsNullOrEmpty()) return states; - - var savedStates = foundStates.States.Select(x => StateMongoElement.ToDomainElement(x)).ToList(); - return new ConversationState(savedStates); - } - - public void UpdateConversationStates(string conversationId, List states) - { - if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; - - var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); - var updateStates = Builders.Update.Set(x => x.States, saveStates); - - _dc.ConversationStates.UpdateOne(filterStates, updateStates); - } - - public void UpdateConversationStatus(string conversationId, string status) - { - if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(status)) return; - - var filter = Builders.Filter.Eq(x => x.Id, conversationId); - var update = Builders.Update - .Set(x => x.Status, status) - .Set(x => x.UpdatedTime, DateTime.UtcNow); - - _dc.Conversations.UpdateOne(filter, update); - } - - public Conversation GetConversation(string conversationId) - { - if (string.IsNullOrEmpty(conversationId)) return null; - - var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); - var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); - - var conv = _dc.Conversations.Find(filterConv).FirstOrDefault(); - var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); - var states = _dc.ConversationStates.Find(filterState).FirstOrDefault(); - - if (conv == null) return null; - - var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); - var curStates = new Dictionary(); - states.States.ForEach(x => - { - curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; - }); - - return new Conversation - { - Id = conv.Id.ToString(), - AgentId = conv.AgentId.ToString(), - UserId = conv.UserId.ToString(), - Title = conv.Title, - Channel = conv.Channel, - Status = conv.Status, - Dialogs = dialogElements, - States = curStates, - CreatedTime = conv.CreatedTime, - UpdatedTime = conv.UpdatedTime - }; - } - - public List GetConversations(ConversationFilter filter) - { - var records = new List(); - var builder = Builders.Filter; - var filters = new List>(); - - if (!string.IsNullOrEmpty(filter.AgentId)) filters.Add(builder.Eq(x => x.AgentId, filter.AgentId)); - if (!string.IsNullOrEmpty(filter.Status)) filters.Add(builder.Eq(x => x.Status, filter.Status)); - if (!string.IsNullOrEmpty(filter.Channel)) filters.Add(builder.Eq(x => x.Channel, filter.Channel)); - if (!string.IsNullOrEmpty(filter.UserId)) filters.Add(builder.Eq(x => x.UserId, filter.UserId)); - - var conversations = _dc.Conversations.Find(builder.And(filters)).ToList(); - - foreach (var conv in conversations) - { - var convId = conv.Id.ToString(); - records.Add(new Conversation - { - Id = convId, - AgentId = conv.AgentId.ToString(), - UserId = conv.UserId.ToString(), - Title = conv.Title, - Channel = conv.Channel, - Status = conv.Status, - CreatedTime = conv.CreatedTime, - UpdatedTime = conv.UpdatedTime - }); - } - - return records; - } - - public List GetLastConversations() - { - var records = new List(); - var conversations = _dc.Conversations.Aggregate() - .Group(c => c.UserId, g => g.OrderByDescending(x => x.CreatedTime).First()) - .ToList(); - return conversations.Select(c => new Conversation() - { - Id = c.Id.ToString(), - AgentId = c.AgentId.ToString(), - UserId = c.UserId.ToString(), - Title = c.Title, - Channel = c.Channel, - Status = c.Status, - CreatedTime = c.CreatedTime, - UpdatedTime = c.UpdatedTime - }).ToList(); - } - #endregion - - #region User - public User? GetUserByEmail(string email) - { - var user = _dc.Users.AsQueryable().FirstOrDefault(x => x.Email == email); - return user != null ? new User - { - Id = user.Id, - UserName = user.UserName, - FirstName = user.FirstName, - LastName = user.LastName, - Email = user.Email, - Password = user.Password, - Salt = user.Salt, - ExternalId = user.ExternalId, - Role = user.Role - } : null; - } - - public User? GetUserById(string id) - { - var user = _dc.Users.AsQueryable().FirstOrDefault(x => x.Id == id || x.ExternalId == id); - return user != null ? new User - { - Id = user.Id, - UserName = user.UserName, - FirstName = user.FirstName, - LastName = user.LastName, - Email = user.Email, - Password = user.Password, - Salt = user.Salt, - ExternalId = user.ExternalId, - Role = user.Role - } : null; - } - - public void CreateUser(User user) - { - if (user == null) return; - - var userCollection = new UserDocument - { - Id = Guid.NewGuid().ToString(), - UserName = user.UserName, - FirstName = user.FirstName, - LastName = user.LastName, - Salt = user.Salt, - Password = user.Password, - Email = user.Email, - ExternalId = user.ExternalId, - Role = user.Role, - CreatedTime = DateTime.UtcNow, - UpdatedTime = DateTime.UtcNow - }; - - _dc.Users.InsertOne(userCollection); - } - #endregion - - #region Execution Log - public void AddExecutionLogs(string conversationId, List logs) - { - if (string.IsNullOrEmpty(conversationId) || logs.IsNullOrEmpty()) return; - - var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var update = Builders.Update - .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .PushEach(x => x.Logs, logs); - - _dc.ExectionLogs.UpdateOne(filter, update, _options); - } - - public List GetExecutionLogs(string conversationId) - { - var logs = new List(); - if (string.IsNullOrEmpty(conversationId)) return logs; - - var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var logCollection = _dc.ExectionLogs.Find(filter).FirstOrDefault(); - - logs = logCollection?.Logs ?? new List(); - return logs; - } - #endregion - - #region LLM Completion Log - public void SaveLlmCompletionLog(LlmCompletionLog log) - { - if (log == null) return; - - var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); - var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString()); - - var logElement = new PromptLogMongoElement - { - MessageId = messageId, - AgentId = log.AgentId, - Prompt = log.Prompt, - Response = log.Response, - CreateDateTime = log.CreateDateTime - }; - - var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var update = Builders.Update - .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .Push(x => x.Logs, logElement); - - _dc.LlmCompletionLogs.UpdateOne(filter, update, _options); - } - - #endregion + List _changedTableNames = new List(); }