From 1543dd6ce9865e2796a6822a8df7b76ee5a10891 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Wed, 9 Aug 2023 23:53:22 -0500 Subject: [PATCH] File repository. --- .../Conversations/IConversationStorage.cs | 1 - .../Settings/ConversationSetting.cs | 1 + .../Services/AgentService.CreateAgent.cs | 5 +- .../Agents/Services/AgentService.GetAgents.cs | 4 +- .../Services/AgentService.UpdateAgent.cs | 2 +- .../Agents/Services/AgentService.cs | 11 +- .../BotSharpServiceCollectionExtensions.cs | 10 +- .../Services/ConversationService.cs | 6 +- .../Services/ConversationStateService.cs | 14 +- .../Services/ConversationStorage.cs | 14 +- .../Repository/BotSharpDbContext.cs | 53 +++++- .../Repository/FileRepository.cs | 173 ++++++++++++++++++ .../Repository/IBotSharpRepository.cs | 11 ++ .../Repository/MyDatabaseSettings.cs | 1 + .../Users/Services/UserService.cs | 10 +- 15 files changed, 279 insertions(+), 37 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/IBotSharpRepository.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStorage.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStorage.cs index 95b63d22..65e523ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStorage.cs @@ -7,5 +7,4 @@ public interface IConversationStorage void InitStorage(string conversationId); void Append(string conversationId, RoleDialogModel dialog); List GetDialogs(string conversationId); - string GetConversationDataDir(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index 4f11ed8f..48e81061 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Conversations.Settings; public class ConversationSetting { + public string DataDir { get; set; } public string ChatCompletion { get; set; } public bool EnableKnowledgeBase { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index 8b8b8975..105470fb 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -6,11 +6,12 @@ public partial class AgentService { public async Task CreateAgent(Agent agent) { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var record = (from a in db.Agent join ua in db.UserAgent on a.Id equals ua.AgentId - where ua.UserId == _user.Id && a.Name == agent.Name + join u in db.User on ua.UserId equals u.Id + where (ua.UserId == _user.Id || u.ExternalId == _user.Id) && a.Name == agent.Name select a).FirstOrDefault(); if (record != null) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index e20b5bcc..dc9d417a 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -8,7 +8,7 @@ public partial class AgentService { public async Task> GetAgents() { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var query = from a in db.Agent join ua in db.UserAgent on a.Id equals ua.AgentId where ua.UserId == _user.Id @@ -18,7 +18,7 @@ public partial class AgentService public async Task GetAgent(string id) { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var query = from agent in db.Agent where agent.Id == id select agent.ToAgent(); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index dfbd1fd9..cd8f353b 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -7,7 +7,7 @@ public partial class AgentService { public async Task UpdateAgent(Agent agent) { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); db.Transaction(delegate { diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs index ae292090..a07c63ae 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs @@ -10,7 +10,10 @@ public partial class AgentService : IAgentService private readonly IUserIdentity _user; private readonly AgentSettings _settings; - public AgentService(IServiceProvider services, ILogger logger, IUserIdentity user, AgentSettings settings) + public AgentService(IServiceProvider services, + ILogger logger, + IUserIdentity user, + AgentSettings settings) { _services = services; _logger = logger; @@ -20,12 +23,14 @@ public partial class AgentService : IAgentService public string GetDataDir() { - return Path.Combine(_settings.DataDir); + var dbSettings = _services.GetRequiredService(); + return Path.Combine(dbSettings.FileRepository); } public string GetAgentDataDir(string agentId) { - var dir = Path.Combine(_settings.DataDir, "agents", agentId); + var dbSettings = _services.GetRequiredService(); + var dir = Path.Combine(dbSettings.FileRepository, _settings.DataDir, agentId); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 78b95eaf..37db512e 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -37,13 +37,17 @@ public static class BotSharpServiceCollectionExtensions var myDatabaseSettings = new MyDatabaseSettings(); config.Bind("Database", myDatabaseSettings); - services.AddSingleton((IServiceProvider x) => databaseSettings); + services.AddSingleton((IServiceProvider x) => myDatabaseSettings); services.AddScoped((IServiceProvider x) => DataContextHelper.GetDbContext(myDatabaseSettings, x)); - services.AddScoped((IServiceProvider x) - => DataContextHelper.GetDbContext(myDatabaseSettings, x)); + services.AddScoped(sp => + { + return myDatabaseSettings.Default == "FileRepository" ? + new FileRepository(myDatabaseSettings, sp) : + DataContextHelper.GetDbContext(myDatabaseSettings, sp); + }); return services; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 346e42d6..f109126f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -35,7 +35,7 @@ public class ConversationService : IConversationService public async Task GetConversation(string id) { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var query = from sess in db.Conversation where sess.Id == id orderby sess.CreatedTime descending @@ -45,7 +45,7 @@ public class ConversationService : IConversationService public async Task> GetConversations() { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var query = from sess in db.Conversation where sess.UserId == _user.Id orderby sess.CreatedTime descending @@ -55,7 +55,7 @@ public class ConversationService : IConversationService public async Task NewConversation(Conversation sess) { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var record = ConversationRecord.FromConversation(sess); record.Id = sess.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString()); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 7fde67b1..7bd06412 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -9,13 +9,13 @@ namespace BotSharp.Core.Conversations.Services; public class ConversationStateService : IConversationStateService, IDisposable { private ConversationState _state; - private IAgentService _agentService; + private MyDatabaseSettings _dbSettings; private string _conversationId; private string _file; - public ConversationStateService(IAgentService agentService) + public ConversationStateService(MyDatabaseSettings dbSettings) { - _agentService = agentService; + _dbSettings = dbSettings; } public void SetState(string name, string value) @@ -65,8 +65,12 @@ public class ConversationStateService : IConversationStateService, IDisposable private string GetStorageFile(string conversationId) { - var dir = _agentService.GetDataDir(); - return Path.Combine(dir, "conversations", conversationId + ".state"); + var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return Path.Combine(dir, "state.dict"); } public string GetState(string name) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 771c06e5..df7f3703 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -6,9 +6,11 @@ namespace BotSharp.Core.Conversations.Services; public class ConversationStorage : IConversationStorage { private readonly IAgentService _agent; - public ConversationStorage(IAgentService agent) + private readonly MyDatabaseSettings _dbSettings; + public ConversationStorage(IAgentService agent, MyDatabaseSettings dbSettings) { _agent = agent; + _dbSettings = dbSettings; } public void Append(string conversationId, RoleDialogModel dialog) @@ -55,17 +57,11 @@ public class ConversationStorage : IConversationStorage private string GetStorageFile(string conversationId) { - var dir = GetConversationDataDir(); - return Path.Combine(dir, conversationId + ".txt"); - } - - public string GetConversationDataDir() - { - var dir = Path.Combine(_agent.GetDataDir(), "conversations"); + var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } - return dir; + return Path.Combine(dir, "dialogs.txt"); } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 9c449b90..b2b3a411 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,9 +1,60 @@ +using Microsoft.EntityFrameworkCore.Infrastructure; + namespace BotSharp.Core.Repository; -public class BotSharpDbContext : Database +public class BotSharpDbContext : Database, IBotSharpRepository { public IQueryable User => Table(); public IQueryable Agent => Table(); public IQueryable UserAgent => Table(); public IQueryable Conversation => Table(); + + public int Transaction(Action action) + { + DatabaseFacade database = base.GetMaster(typeof(TTableInterface)).Database; + int num = 0; + if (database.CurrentTransaction == null) + { + using (Microsoft.EntityFrameworkCore.Storage.IDbContextTransaction dbContextTransaction = database.BeginTransaction()) + { + try + { + action(); + num = base.SaveChanges(); + dbContextTransaction.Commit(); + return num; + } + catch (Exception ex) + { + dbContextTransaction.Rollback(); + if (ex.Message.Contains("See the inner exception for details")) + { + throw ex.InnerException; + } + + throw ex; + } + } + } + + try + { + action(); + return base.SaveChanges(); + } + catch (Exception ex2) + { + if (database.CurrentTransaction != null) + { + database.CurrentTransaction.Rollback(); + } + + if (ex2.Message.Contains("See the inner exception for details")) + { + throw ex2.InnerException; + } + + throw ex2; + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs new file mode 100644 index 00000000..0dc105c5 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -0,0 +1,173 @@ +using System.IO; +using System.Text.Json; + +namespace BotSharp.Core.Repository; + +public class FileRepository : IBotSharpRepository +{ + private readonly MyDatabaseSettings _dbSettings; + private readonly IServiceProvider _services; + private JsonSerializerOptions _options; + + public FileRepository(MyDatabaseSettings dbSettings, IServiceProvider services) + { + _dbSettings = dbSettings; + _services = services; + + _options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; + } + + private List _users; + public IQueryable User + { + get + { + if (_users != null) + { + return _users.AsQueryable(); + } + + var dir = Path.Combine(_dbSettings.FileRepository, "users"); + _users = new List(); + foreach (var d in Directory.GetDirectories(dir)) + { + var json = File.ReadAllText(Path.Combine(d, "user.json")); + _users.Add(JsonSerializer.Deserialize(json, _options)); + } + return _users.AsQueryable(); + } + } + + private List _agents; + public IQueryable Agent + { + get + { + if (_agents != null) + { + return _agents.AsQueryable(); + } + + var agentSettings = _services.GetService(); + var dir = Path.Combine(_dbSettings.FileRepository, agentSettings.DataDir); + _agents = new List(); + foreach (var d in Directory.GetDirectories(dir)) + { + var json = File.ReadAllText(Path.Combine(d, "agent.json")); + _agents.Add(JsonSerializer.Deserialize(json, _options)); + } + return _agents.AsQueryable(); + } + } + + private List _userAgents; + public IQueryable UserAgent + { + get + { + if (_userAgents != null) + { + return _userAgents.AsQueryable(); + } + + var dir = Path.Combine(_dbSettings.FileRepository, "users"); + _userAgents = new List(); + foreach (var d in Directory.GetDirectories(dir)) + { + var json = File.ReadAllText(Path.Combine(d, "agents.json")); + _userAgents.AddRange(JsonSerializer.Deserialize>(json, _options)); + } + return _userAgents.AsQueryable(); + } + } + + private List _conversations; + public IQueryable Conversation + { + get + { + if (_conversations != null) + { + return _conversations.AsQueryable(); + } + + var convSettings = _services.GetService(); + var dir = Path.Combine(_dbSettings.FileRepository, convSettings.DataDir); + _conversations = new List(); + foreach (var d in Directory.GetDirectories(dir)) + { + var json = File.ReadAllText(Path.Combine(d, "conversation.json")); + _conversations.Add(JsonSerializer.Deserialize(json, _options)); + } + return _conversations.AsQueryable(); + } + } + + public void Add(object entity) + { + _conversations = Conversation.ToList(); + if (entity is ConversationRecord conversation) + { + _conversations.Add(conversation); + _changedTableNames.Add(nameof(ConversationRecord)); + } + else if (entity is AgentRecord agent) + { + _agents.Add(agent); + _changedTableNames.Add(nameof(AgentRecord)); + } + } + + List _changedTableNames = new List(); + public int Transaction(Action action) + { + _changedTableNames.Clear(); + action(); + + // Persist to disk + foreach (var table in _changedTableNames) + { + if (table == nameof(ConversationRecord)) + { + var convSettings = _services.GetService(); + + foreach (var conversation in _conversations) + { + var dir = Path.Combine(_dbSettings.FileRepository, + convSettings.DataDir, + conversation.Id); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + var path = Path.Combine(dir, "conversation.json"); + File.WriteAllText(path, JsonSerializer.Serialize(conversation, _options)); + } + } + else if (table == nameof(AgentRecord)) + { + var agentSettings = _services.GetService(); + + foreach (var agent in _agents) + { + var dir = Path.Combine(_dbSettings.FileRepository, + agentSettings.DataDir, + agent.Id); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + var path = Path.Combine(dir, "agent.json"); + File.WriteAllText(path, JsonSerializer.Serialize(agent, _options)); + } + } + } + + return _changedTableNames.Count; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/IBotSharpRepository.cs new file mode 100644 index 00000000..9490ee73 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/IBotSharpRepository.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Core.Repository; + +public interface IBotSharpRepository +{ + IQueryable User { get; } + IQueryable Agent { get; } + IQueryable UserAgent { get; } + IQueryable Conversation { get; } + int Transaction(Action action); + void Add(object entity); +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/MyDatabaseSettings.cs b/src/Infrastructure/BotSharp.Core/Repository/MyDatabaseSettings.cs index cadf7206..dadec9b5 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/MyDatabaseSettings.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/MyDatabaseSettings.cs @@ -3,6 +3,7 @@ namespace BotSharp.Core.Repository; public class MyDatabaseSettings : DatabaseSettings { public string[] Assemblies { get; set; } + public string FileRepository { get; set; } public DbConnectionSetting MongoDb { get; set; } public DbConnectionSetting BotSharp { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 9d979168..7f79bfcb 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -1,9 +1,5 @@ -using BotSharp.Abstraction.Users; using BotSharp.Abstraction.Users.Models; -using BotSharp.Core.Infrastructures; -using BotSharp.Core.Repository.DbTables; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; using Microsoft.IdentityModel.Tokens; using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; @@ -23,7 +19,7 @@ public class UserService : IUserService public async Task CreateUser(User user) { - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var record = db.User.FirstOrDefault(x => x.Email == user.Email.ToLower()); if (record != null) { @@ -49,7 +45,7 @@ public class UserService : IUserService var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization)); var (userEmail, password) = base64.SplitAsTuple(":"); - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var record = db.User.FirstOrDefault(x => x.Email == userEmail); if (record == null) { @@ -104,7 +100,7 @@ public class UserService : IUserService { var userId = _user.Id; - var db = _services.GetRequiredService(); + var db = _services.GetRequiredService(); var user = (from u in db.User where u.Id == userId select new User