From 336a98d22a90784981e24ef5a434a02e89c4e916 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 17 Oct 2023 15:29:40 -0500 Subject: [PATCH] add bulk insert agents and user agents --- .../Agents/IAgentService.cs | 1 + .../Repositories/IBotSharpRepository.cs | 5 +- .../Utilities/ListExtenstions.cs | 4 +- .../Services/AgentService.RefreshAgents.cs | 51 ++++++++++++ .../Repository/BotSharpDbContext.cs | 17 +++- .../Repository/FileRepository.cs | 23 ++++-- .../Controllers/AgentController.cs | 6 ++ .../BotSharp.Plugin.MongoStorage.csproj | 1 + .../Repository/MongoRepository.cs | 77 +++++++++++++++++-- 9 files changed, 171 insertions(+), 14 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index 83564458..5a02fb64 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -6,6 +6,7 @@ namespace BotSharp.Abstraction.Agents; public interface IAgentService { Task CreateAgent(Agent agent); + Task RefreshAgents(); Task> GetAgents(); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 2a57193a..96a36721 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -13,13 +13,16 @@ public interface IBotSharpRepository void Add(object entity); #region User - User GetUserByEmail(string email); + User? GetUserByEmail(string email); void CreateUser(User user); #endregion #region Agent void UpdateAgent(Agent agent, AgentField field); Agent? GetAgent(string agentId); + void BulkInsertAgents(List agents); + void BulkInsertUserAgents(List userAgents); + bool DeleteAgents(); List GetAgentResponses(string agentId, string prefix, string intent); string GetAgentTemplate(string agentId, string templateName); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs index acba3215..6853f055 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs @@ -2,8 +2,8 @@ namespace BotSharp.Abstraction.Utilities; public static class ListExtenstions { - public static bool IsNullOrEmpty(this IEnumerable strList) + public static bool IsNullOrEmpty(this IEnumerable list) { - return strList == null || !strList.Any(); + return list == null || !list.Any(); } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs new file mode 100644 index 00000000..769ae7c1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -0,0 +1,51 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Repositories; +using System.IO; + +namespace BotSharp.Core.Agents.Services; + +public partial class AgentService +{ + public async Task RefreshAgents() + { + var isDeleted = _db.DeleteAgents(); + if (!isDeleted) return; + + var dbSettings = _services.GetRequiredService(); + var agentDir = Path.Combine(dbSettings.FileRepository, _agentSettings.DataDir); + var user = _db.Users.FirstOrDefault(x => x.Id == _user.Id || x.ExternalId == _user.Id); + var agents = new List(); + var userAgents = new List(); + + foreach (var dir in Directory.GetDirectories(agentDir)) + { + var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json")); + var agent = JsonSerializer.Deserialize(agentJson, _options); + if (agent == null) continue; + + var functions = FetchFunctionsFromFile(dir); + var instruction = FetchInstructionFromFile(dir); + var responses = FetchResponsesFromFile(dir); + var templates = FetchTemplatesFromFile(dir); + agent.SetInstruction(instruction) + .SetTemplates(templates) + .SetFunctions(functions) + .SetResponses(responses); + + var userAgent = new UserAgent + { + Id = Guid.NewGuid().ToString(), + UserId = user.Id, + AgentId = agent.Id, + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow + }; + + agents.Add(agent); + userAgents.Add(userAgent); + } + + _db.BulkInsertAgents(agents); + _db.BulkInsertUserAgents(userAgents); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 9819957f..2f0c85e2 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -86,6 +86,21 @@ public class BotSharpDbContext : Database, IBotSharpRepository { throw new NotImplementedException(); } + + public void BulkInsertAgents(List agents) + { + throw new NotImplementedException(); + } + + public void BulkInsertUserAgents(List userAgents) + { + throw new NotImplementedException(); + } + + public bool DeleteAgents() + { + throw new NotImplementedException(); + } #endregion @@ -128,7 +143,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository #region User - public User GetUserByEmail(string email) + public User? GetUserByEmail(string email) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index f5d19637..8958356e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -31,7 +31,7 @@ public class FileRepository : IBotSharpRepository }; } - private List _users; + private List _users = new List(); public IQueryable Users { get @@ -52,7 +52,7 @@ public class FileRepository : IBotSharpRepository } } - private List _agents; + private List _agents = new List(); public IQueryable Agents { get @@ -81,7 +81,7 @@ public class FileRepository : IBotSharpRepository } } - private List _userAgents; + private List _userAgents = new List(); public IQueryable UserAgents { get @@ -106,7 +106,7 @@ public class FileRepository : IBotSharpRepository } } - private List _conversations; + private List _conversations = new List(); public IQueryable Conversations { get @@ -534,6 +534,19 @@ public class FileRepository : IBotSharpRepository return string.Empty; } + + public void BulkInsertAgents(List agents) + { + } + + public void BulkInsertUserAgents(List userAgents) + { + } + + public bool DeleteAgents() + { + return false; + } #endregion #region Conversation @@ -684,7 +697,7 @@ public class FileRepository : IBotSharpRepository #endregion #region User - public User GetUserByEmail(string email) + public User? GetUserByEmail(string email) { return Users.FirstOrDefault(x => x.Email == email); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index c79bef48..b68a0bf4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -38,6 +38,12 @@ public class AgentController : ControllerBase, IApiAdapter return AgentViewModel.FromAgent(createdAgent); } + [HttpPost("/refresh-agents")] + public async Task RefreshAgents() + { + await _agentService.RefreshAgents(); + } + [HttpPut("/agent/file/{agentId}")] public async Task UpdateAgentFromFile([FromRoute] string agentId) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj index 028d6136..b7404548 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/BotSharp.Plugin.MongoStorage.csproj @@ -3,6 +3,7 @@ netstandard2.1 $(LangVersion) + enable $(BotSharpVersion) $(GeneratePackageOnBuild) True diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index f9cbd047..2d621c01 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Users.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; @@ -24,7 +25,7 @@ public class MongoRepository : IBotSharpRepository }; } - private List _agents; + private List _agents = new List(); public IQueryable Agents { get @@ -65,7 +66,7 @@ public class MongoRepository : IBotSharpRepository } } - private List _users; + private List _users = new List(); public IQueryable Users { get @@ -93,7 +94,7 @@ public class MongoRepository : IBotSharpRepository } } - private List _userAgents; + private List _userAgents = new List(); public IQueryable UserAgents { get @@ -117,7 +118,7 @@ public class MongoRepository : IBotSharpRepository } } - private List _conversations; + private List _conversations = new List(); public IQueryable Conversations { get @@ -539,6 +540,72 @@ public class MongoRepository : IBotSharpRepository return agent.Templates?.FirstOrDefault(x => x.Name == templateName.ToLower())?.Content ?? string.Empty; } + + public void BulkInsertAgents(List agents) + { + if (agents.IsNullOrEmpty()) return; + + var agentDocs = agents.Select(x => new AgentCollection + { + Id = string.IsNullOrEmpty(x.Id) ? Guid.NewGuid() : new Guid(x.Id), + Name = x.Name, + Description = x.Description, + Instruction = x.Instruction, + Templates = x.Templates? + .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? + .ToList() ?? new List(), + Functions = x.Functions? + .Select(f => FunctionDefMongoElement.ToMongoElement(f))? + .ToList() ?? new List(), + Responses = x.Responses? + .Select(r => AgentResponseMongoElement.ToMongoElement(r))? + .ToList() ?? new List(), + IsPublic = x.IsPublic, + AllowRouting = x.AllowRouting, + Disabled = x.Disabled, + Profiles = x.Profiles, + RoutingRules = x.RoutingRules? + .Select(r => RoutingRuleMongoElement.ToMongoElement(r))? + .ToList() ?? new List(), + CreatedTime = x.CreatedDateTime, + UpdatedTime = x.UpdatedDateTime + }).ToList(); + + _dc.Agents.InsertMany(agentDocs); + } + + public void BulkInsertUserAgents(List userAgents) + { + if (userAgents.IsNullOrEmpty()) return; + + var userAgentDocs = userAgents.Select(x => new UserAgentCollection + { + Id = string.IsNullOrEmpty(x.Id) ? Guid.NewGuid() : new Guid(x.Id), + AgentId = Guid.Parse(x.AgentId), + UserId = !string.IsNullOrEmpty(x.UserId) && Guid.TryParse(x.UserId, out var _) ? Guid.Parse(x.UserId) : Guid.Empty, + CreatedTime = x.CreatedTime, + UpdatedTime = x.UpdatedTime + }).ToList(); + + _dc.UserAgents.InsertMany(userAgentDocs); + } + + public bool DeleteAgents() + { + try + { + var userAgentFilter = Builders.Filter.Empty; + var agentfilter = Builders.Filter.Empty; + _dc.UserAgents.DeleteMany(userAgentFilter); + _dc.Agents.DeleteMany(agentfilter); + return true; + } + catch + { + return false; + } + + } #endregion #region Conversation @@ -676,7 +743,7 @@ public class MongoRepository : IBotSharpRepository #endregion #region User - public User GetUserByEmail(string email) + public User? GetUserByEmail(string email) { var user = Users.FirstOrDefault(x => x.Email == email); return user != null ? new User