add mongo repository

This commit is contained in:
Jicheng Lu 2023-08-25 23:41:01 -05:00
parent 76b1e6de32
commit a2a3759353
38 changed files with 498 additions and 97 deletions

View file

@ -1,4 +1,7 @@
namespace BotSharp.Core.Repository;
using BotSharp.Abstraction.Repositories.Records;
using System.Linq;
namespace BotSharp.Abstraction.Repositories;
public interface IBotSharpRepository
{

View file

@ -0,0 +1,5 @@
namespace BotSharp.Abstraction.Repositories;
public interface IBotSharpTable
{
}

View file

@ -0,0 +1,31 @@
namespace BotSharp.Abstraction.Repositories;
public class MyDatabaseSettings : DatabaseSettings
{
public string[] Assemblies { get; set; }
public string FileRepository { get; set; }
public string MongoDb { get; set; }
public DbConnectionSetting BotSharp { get; set; }
}
public class DatabaseSettings
{
public string Default { get; set; }
public DbConnectionSetting DefaultConnection { get; set; }
public bool EnableSqlLog { get; set; }
public bool EnableSensitiveDataLogging { get; set; }
public bool EnableRetryOnFailure { get; set; }
public bool UseCamelCase { get; set; }
}
public class DbConnectionSetting
{
public string Master { get; set; }
public string[] Slavers { get; set; }
public DbConnectionSetting()
{
Slavers = new string[0];
}
}

View file

@ -1,11 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace BotSharp.Abstraction.Repositories.Records;
namespace BotSharp.Core.Repository.DbTables;
[Table("Agent")]
public class AgentRecord : DbRecord, IBotSharpTable
public class AgentRecord : RecordBase
{
[Required]
[MaxLength(64)]
@ -15,10 +10,10 @@ public class AgentRecord : DbRecord, IBotSharpTable
public string? Description { get; set; }
[Required]
public DateTime CreatedDateTime { get; set; }
public DateTime CreatedTime { get; set; }
[Required]
public DateTime UpdatedDateTime { get; set; }
public DateTime UpdatedTime { get; set; }
public static AgentRecord FromAgent(Agent agent)
{
@ -37,8 +32,8 @@ public class AgentRecord : DbRecord, IBotSharpTable
Id = Id,
Name = Name,
Description = Description,
CreatedDateTime = CreatedDateTime,
UpdatedDateTime = UpdatedDateTime
CreatedDateTime = CreatedTime,
UpdatedDateTime = UpdatedTime
};
}
}

View file

@ -1,11 +1,8 @@
using BotSharp.Abstraction.Conversations.Models;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace BotSharp.Core.Repository.DbTables;
namespace BotSharp.Abstraction.Repositories.Records;
[Table("Conversation")]
public class ConversationRecord : DbRecord, IBotSharpTable
public class ConversationRecord : RecordBase
{
[Required]
[MaxLength(36)]

View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Abstraction.Repositories.Records
{
public class RecordBase
{
public string? Id { get; set; }
}
}

View file

@ -1,11 +1,8 @@
using BotSharp.Abstraction.Users.Models;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace BotSharp.Core.Repository.DbTables;
namespace BotSharp.Abstraction.Repositories.Records;
[Table("UserAgent")]
public class UserAgentRecord : DbRecord, IBotSharpTable
public class UserAgentRecord : RecordBase
{
[Required]
[StringLength(36)]

View file

@ -1,11 +1,8 @@
using BotSharp.Abstraction.Users.Models;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace BotSharp.Core.Repository.DbTables;
namespace BotSharp.Abstraction.Repositories.Records;
[Table("User")]
public class UserRecord : DbRecord, IBotSharpTable
public class UserRecord : RecordBase
{
[Required]
[MaxLength(64)]

View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Repositories.Records;
namespace BotSharp.Core.Agents.Services;
@ -21,8 +23,8 @@ public partial class AgentService
record = AgentRecord.FromAgent(agent);
record.Id = Guid.NewGuid().ToString();
record.CreatedDateTime = DateTime.UtcNow;
record.UpdatedDateTime = DateTime.UtcNow;
record.CreatedTime = DateTime.UtcNow;
record.UpdatedTime = DateTime.UtcNow;
var userAgentRecord = new UserAgentRecord
{

View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Repositories;
using System.IO;
namespace BotSharp.Core.Agents.Services;

View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Repositories;
using System.IO;
namespace BotSharp.Core.Agents.Services;
@ -20,7 +21,7 @@ public partial class AgentService
record.Name = agent.Name;
record.Description = agent.Description;
record.UpdatedDateTime = DateTime.UtcNow;
record.UpdatedTime = DateTime.UtcNow;
});
// Save instruction to file

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Repositories;
using Microsoft.Extensions.Logging;
using System.IO;

View file

@ -82,4 +82,8 @@
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Repository\Abstraction\" />
</ItemGroup>
</Project>

View file

@ -1,7 +1,9 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Repositories;
using BotSharp.Core.Functions;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Configuration;
using DatabaseSettings = BotSharp.Abstraction.Repositories.DatabaseSettings;
namespace BotSharp.Core;

View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Repositories.Records;
namespace BotSharp.Core.Conversations.Services;

View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories;
using System.IO;
namespace BotSharp.Core.Conversations.Services;

View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories;
using System.IO;
namespace BotSharp.Core.Conversations.Services;

View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Repositories;
using System.IO;
namespace BotSharp.Core.Functions;

View file

@ -1,5 +0,0 @@
namespace BotSharp.Core.Repository.Abstraction;
public interface IBotSharpTable
{
}

View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Repositories.Records;
using Microsoft.EntityFrameworkCore.Infrastructure;
namespace BotSharp.Core.Repository;

View file

@ -1,22 +0,0 @@
using BotSharp.Abstraction.Conversations.Models;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Bson.Serialization.IdGenerators;
namespace BotSharp.Core.Repository.Collections;
public class ConversationCollection : IMongoDbCollection
{
[BsonId(IdGenerator = typeof(ObjectIdGenerator))]
public ObjectId Id { get; set; }
public string SessionId { get; set; }
public string UserId { get; set; }
public string Model { get; set; }
public string Title { get; set; }
public List<RoleDialogModel> Messages { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime UpdatedAt { get; set; }
}

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Repositories;
using Microsoft.Data.SqlClient;
using MySqlConnector;
using System.Data.Common;
@ -16,16 +17,7 @@ public static class DataContextHelper
AppDomain.CurrentDomain.SetData("Assemblies", settings.Assemblies);
var dc = new T();
if (typeof(T) == typeof(MongoDbContext))
{
dc.BindDbContext<IMongoDbCollection, DbContext4MongoDb>(new DatabaseBind
{
ServiceProvider = serviceProvider,
MasterConnection = new MongoDbConnection(settings.MongoDb.Master),
IsRelational = false
});
}
else if (typeof(T) == typeof(BotSharpDbContext))
if (typeof(T) == typeof(BotSharpDbContext))
{
if (typeof(Tdb).Name.StartsWith("DbContext4SqlServer"))
{

View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Repositories.Records;
using System.IO;
using System.Text.Json;

View file

@ -1,8 +0,0 @@
using MongoDB.Bson;
namespace BotSharp.Core.Repository;
public interface IMongoDbCollection
{
ObjectId Id { get; set; }
}

View file

@ -1,10 +0,0 @@
using BotSharp.Core.Repository.Collections;
using MongoDB.Driver;
namespace BotSharp.Core.Repository;
public class MongoDbContext : Database
{
public IMongoCollection<ConversationCollection> Conversations
=> Collection<ConversationCollection>("conversations");
}

View file

@ -1,9 +0,0 @@
namespace BotSharp.Core.Repository;
public class MyDatabaseSettings : DatabaseSettings
{
public string[] Assemblies { get; set; }
public string FileRepository { get; set; }
public DbConnectionSetting MongoDb { get; set; }
public DbConnectionSetting BotSharp { get; set; }
}

View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Repositories.Records;
using BotSharp.Abstraction.Users.Models;
using Microsoft.Extensions.Configuration;
using Microsoft.IdentityModel.Tokens;
@ -27,7 +29,7 @@ public class UserService : IUserService
}
record = UserRecord.FromUser(user);
record.Id = Guid.NewGuid().ToString();
//record.Id = Guid.NewGuid().ToString();
record.Email = user.Email.ToLower();
record.Salt = Guid.NewGuid().ToString("N");
record.Password = Utilities.HashText(user.Password, record.Salt);

View file

@ -14,8 +14,6 @@ global using BotSharp.Abstraction.Knowledges;
global using BotSharp.Abstraction.Users;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Core.Repository;
global using BotSharp.Core.Repository.Abstraction;
global using BotSharp.Core.Repository.DbTables;
global using BotSharp.Core.Agents.Services;
global using BotSharp.Core.Conversations.Services;
global using BotSharp.Core.Infrastructures;

View file

@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
<LangVersion>10.0</LangVersion>
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
<VersionPrefix>0.9.0</VersionPrefix>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="2.19.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,14 @@
namespace BotSharp.Plugin.MongoRepository.Collections
{
public class AgentCollection : MongoBase
{
public string Name { get; set; }
public string Description { get; set; }
public List<string> Functions { get; set; }
public string Instruction { get; set; }
public List<string> Routes { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
}
}

View file

@ -0,0 +1,13 @@
namespace BotSharp.Plugin.MongoRepository.Collections;
public class ConversationCollection : MongoBase
{
public string AgentId { get; set; }
public string UserId { get; set; }
public string Title { get; set; }
public string Dialog { get; set; }
public string State { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
}

View file

@ -0,0 +1,11 @@
namespace BotSharp.Plugin.MongoRepository.Collections
{
public class UserAgentCollection : MongoBase
{
public string UserId { get; set; }
public string AgentId { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
}
}

View file

@ -0,0 +1,15 @@
namespace BotSharp.Plugin.MongoRepository.Collections
{
public class UserCollection : MongoBase
{
public string FirstName { get; set; }
public string LastName { get; set; }
public string Email { get; set; }
public string Salt { get; set; }
public string Password { get; set; }
public string? ExternalId { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
}
}

View file

@ -0,0 +1,11 @@
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Bson.Serialization.IdGenerators;
namespace BotSharp.Plugin.MongoRepository;
[BsonIgnoreExtraElements(Inherited = true)]
public class MongoBase
{
[BsonId(IdGenerator = typeof(StringObjectIdGenerator))]
public string Id { get; set; }
}

View file

@ -0,0 +1,39 @@
using BotSharp.Plugin.MongoRepository.Collections;
namespace BotSharp.Core.Repository;
public class MongoDbContext
{
private readonly MongoClient _mongoClient;
private readonly string _mongoDbDatabaseName;
public MongoDbContext(string mongoDbConnectionString)
{
_mongoClient = new MongoClient(mongoDbConnectionString);
_mongoDbDatabaseName = GetDatabaseName(mongoDbConnectionString);
}
private string GetDatabaseName(string mongoDbConnectionString)
{
var databaseName = mongoDbConnectionString.Substring(mongoDbConnectionString.LastIndexOf("/", StringComparison.InvariantCultureIgnoreCase) + 1);
if (databaseName.Contains("?"))
{
databaseName = databaseName.Substring(0, databaseName.IndexOf("?", StringComparison.InvariantCultureIgnoreCase));
}
return databaseName;
}
private IMongoDatabase Database { get { return _mongoClient.GetDatabase(_mongoDbDatabaseName); } }
public IMongoCollection<AgentCollection> Agents
=> Database.GetCollection<AgentCollection>("OneBrainAgents");
public IMongoCollection<ConversationCollection> Conversations
=> Database.GetCollection<ConversationCollection>("OneBrainConversations");
public IMongoCollection<UserCollection> Users
=> Database.GetCollection<UserCollection>("OneBrainUsers");
public IMongoCollection<UserAgentCollection> UserAgents
=> Database.GetCollection<UserAgentCollection>("OneBrainUserAgents");
}

View file

@ -0,0 +1,16 @@
namespace BotSharp.Core.Repository
{
public class MongoRepositoryPlugin : IBotSharpPlugin
{
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddSingleton((IServiceProvider x) =>
{
var databaseSettings = x.GetRequiredService<MyDatabaseSettings>();
return new MongoDbContext(databaseSettings.MongoDb);
});
services.AddScoped<IBotSharpRepository, MongoRepository>();
}
}
}

View file

@ -0,0 +1,256 @@
using BotSharp.Plugin.MongoRepository.Collections;
namespace BotSharp.Core.Repository
{
public class MongoRepository : IBotSharpRepository
{
private readonly MongoDbContext _dc;
private readonly IServiceProvider _services;
private UpdateOptions _options;
public MongoRepository(MongoDbContext dc, IServiceProvider services)
{
_dc = dc;
_services = services;
_options = new UpdateOptions
{
IsUpsert = true,
};
}
private List<AgentRecord> _agents;
public IQueryable<AgentRecord> Agent
{
get
{
if (_agents != null)
{
return _agents.AsQueryable();
}
var agentDocs = _dc.Agents?.AsQueryable()?.ToList() ?? new List<AgentCollection>();
_agents = agentDocs.Select(x => new AgentRecord
{
Id = x.Id?.ToString(),
Name = x.Name,
Description = x.Description,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
return _agents.AsQueryable();
}
}
private List<UserRecord> _users;
public IQueryable<UserRecord> User
{
get
{
if (_users != null)
{
return _users.AsQueryable();
}
var userDocs = _dc.Users?.AsQueryable()?.ToList() ?? new List<UserCollection>();
_users = userDocs.Select(x => new UserRecord
{
Id = x.Id?.ToString(),
FirstName = x.FirstName,
LastName = x.LastName,
Email = x.Email,
Password = x.Password,
Salt = x.Salt,
ExternalId = x.ExternalId,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
return _users.AsQueryable();
}
}
private List<UserAgentRecord> _userAgents;
public IQueryable<UserAgentRecord> UserAgent
{
get
{
if (_userAgents != null && _userAgents.Count > 0)
{
return _userAgents.AsQueryable();
}
var userDocs = _dc.UserAgents?.AsQueryable()?.ToList() ?? new List<UserAgentCollection>();
_userAgents = userDocs.Select(x => new UserAgentRecord
{
Id = x.Id?.ToString(),
AgentId = x.AgentId,
UserId = x.UserId,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
return _userAgents.AsQueryable();
}
}
private List<ConversationRecord> _conversations;
public IQueryable<ConversationRecord> Conversation
{
get
{
if (_conversations != null)
{
return _conversations.AsQueryable();
}
var conversationDocs = _dc.Conversations?.AsQueryable()?.ToList() ?? new List<ConversationCollection>();
_conversations = conversationDocs.Select(x => new ConversationRecord
{
Id = x.Id?.ToString(),
AgentId = x.AgentId,
UserId = x.UserId,
Title = x.Title,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
return _conversations.AsQueryable();
}
}
List<string> _changedTableNames = new List<string>();
public void Add<TTableInterface>(object entity)
{
if (entity is ConversationRecord conversation)
{
_conversations.Add(conversation);
_changedTableNames.Add(nameof(ConversationRecord));
}
else if (entity is AgentRecord agent)
{
_agents.Add(agent);
_changedTableNames.Add(nameof(AgentRecord));
}
else if (entity is UserRecord user)
{
_users.Add(user);
_changedTableNames.Add(nameof(UserRecord));
}
else if (entity is UserAgentRecord userAgent)
{
_userAgents.Add(userAgent);
_changedTableNames.Add(nameof(UserAgentRecord));
}
}
public int Transaction<TTableInterface>(Action action)
{
_changedTableNames.Clear();
action();
foreach (var table in _changedTableNames)
{
if (table == nameof(ConversationRecord))
{
var conversations = _conversations.Select(x => new ConversationCollection
{
Id = x.Id.IfNullOrEmptyAs(ObjectId.GenerateNewId().ToString()),
AgentId = x.AgentId,
UserId = x.UserId,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
foreach (var conversation in conversations)
{
var filter = Builders<ConversationCollection>.Filter.Eq(x => x.Id, conversation.Id);
var update = Builders<ConversationCollection>.Update
.Set(x => x.AgentId, conversation.AgentId)
.Set(x => x.UserId, conversation.UserId)
.Set(x => x.CreatedTime, conversation.CreatedTime)
.Set(x => x.UpdatedTime, conversation.UpdatedTime);
_dc.Conversations.UpdateOne(filter, update, _options);
}
}
else if (table == nameof(AgentRecord))
{
var agents = _agents.Select(x => new AgentCollection
{
Id = x.Id.IfNullOrEmptyAs(ObjectId.GenerateNewId().ToString()),
Name = x.Name,
Description = x.Description,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
foreach (var agent in agents)
{
var filter = Builders<AgentCollection>.Filter.Eq(x => x.Id, agent.Id);
var update = Builders<AgentCollection>.Update
.Set(x => x.Name, agent.Name)
.Set(x => x.Description, agent.Description)
.Set(x => x.CreatedTime, agent.CreatedTime)
.Set(x => x.UpdatedTime, agent.UpdatedTime);
_dc.Agents.UpdateOne(filter, update, _options);
}
}
else if (table == nameof(UserRecord))
{
var users = _users.Select(x => new UserCollection
{
Id = x.Id.IfNullOrEmptyAs(ObjectId.GenerateNewId().ToString()),
FirstName = x.FirstName,
LastName = x.LastName,
Salt = x.Salt,
Password = x.Password,
Email = x.Email,
ExternalId = x.ExternalId,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
foreach (var user in users)
{
var filter = Builders<UserCollection>.Filter.Eq(x => x.Id, user.Id);
var update = Builders<UserCollection>.Update
.Set(x => x.FirstName, user.FirstName)
.Set(x => x.LastName, user.LastName)
.Set(x => x.Email, user.Email)
.Set(x => x.Salt, user.Salt)
.Set(x => x.Password, user.Password)
.Set(x => x.ExternalId, user.ExternalId)
.Set(x => x.CreatedTime, user.CreatedTime)
.Set(x => x.UpdatedTime, user.UpdatedTime);
_dc.Users.UpdateOne(filter, update, _options);
}
}
else if (table == nameof(UserAgentRecord))
{
var userAgents = _userAgents.Select(x => new UserAgentCollection
{
Id = x.Id.IfNullOrEmptyAs(ObjectId.GenerateNewId().ToString()),
AgentId = x.AgentId,
UserId = x.UserId,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
}).ToList();
foreach (var userAgent in userAgents)
{
var filter = Builders<UserAgentCollection>.Filter.Eq(x => x.Id, userAgent.Id);
var update = Builders<UserAgentCollection>.Update
.Set(x => x.AgentId, userAgent.AgentId)
.Set(x => x.UserId, userAgent.UserId)
.Set(x => x.CreatedTime, userAgent.CreatedTime)
.Set(x => x.UpdatedTime, userAgent.UpdatedTime);
_dc.UserAgents.UpdateOne(filter, update, _options);
}
}
}
return _changedTableNames.Count;
}
}
}

View file

@ -0,0 +1,14 @@
global using System;
global using System.Collections.Generic;
global using System.Text;
global using System.Threading.Tasks;
global using System.Linq;
global using System.Text.Json;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Repositories.Records;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Plugins;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using MongoDB.Bson;
global using MongoDB.Driver;