add mongo

This commit is contained in:
Jicheng Lu 2024-09-09 12:05:11 -05:00
parent b1946256a1
commit 1c0818b11e
10 changed files with 109 additions and 9 deletions

View file

@ -102,7 +102,7 @@ public interface IBotSharpRepository
#endregion
#region Knowledge
bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs);
bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs);
VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName);
#endregion
}

View file

@ -16,6 +16,12 @@ public class VectorCollectionConfig
[JsonPropertyName("text_embedding")]
public KnowledgeEmbeddingConfig TextEmbedding { get; set; }
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;
[JsonPropertyName("create_user_id")]
public string CreateUserId { get; set; } = string.Empty;
}
public class KnowledgeEmbeddingConfig

View file

@ -235,7 +235,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
#endregion
#region Knowledge
public bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs) =>
public bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs) =>
throw new NotImplementedException();
public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) =>

View file

@ -5,7 +5,7 @@ namespace BotSharp.Core.Repository;
public partial class FileRepository
{
public bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs)
public bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs)
{
var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
if (!Directory.Exists(dir))
@ -20,6 +20,13 @@ public partial class FileRepository
public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)) return null;
var file = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE);
if (!File.Exists(file)) return null;
var str = File.ReadAllText(file);
var configs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
return configs.FirstOrDefault(x => x.Name == collectionName);
}
}

View file

@ -5,7 +5,17 @@ public partial class KnowledgeService
public async Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var saved = db.SaveKnowledgeCollectionConfigs(configs.Collections);
var collections = configs.Collections ?? new();
var userService = _services.GetRequiredService<IUserService>();
var user = await userService.GetUser(_user.Id);
foreach (var collection in collections)
{
collection.CreateDate = DateTime.UtcNow;
collection.CreateUserId = user.Id;
}
var saved = db.ResetKnowledgeCollectionConfigs(collections);
return await Task.FromResult(saved);
}
}

View file

@ -3,15 +3,18 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService : IKnowledgeService
{
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
private readonly KnowledgeBaseSettings _settings;
private readonly ILogger<KnowledgeService> _logger;
public KnowledgeService(
IServiceProvider services,
IUserIdentity user,
KnowledgeBaseSettings settings,
ILogger<KnowledgeService> logger)
{
_services = services;
_user = user;
_settings = settings;
_logger = logger;
}

View file

@ -0,0 +1,10 @@
namespace BotSharp.Plugin.MongoStorage.Collections;
public class KnowledgeCollectionConfigDocument : MongoBase
{
public string Name { get; set; }
public string Type { get; set; }
public KnowledgeEmbeddingConfigMongoModel TextEmbedding { get; set; }
public DateTime CreateDate { get; set; }
public string CreateUserId { get; set; }
}

View file

@ -0,0 +1,30 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Models;
public class KnowledgeEmbeddingConfigMongoModel
{
public string Provider { get; set; }
public string Model { get; set; }
public int Dimension { get; set; }
public static KnowledgeEmbeddingConfigMongoModel ToMongoModel(KnowledgeEmbeddingConfig model)
{
return new KnowledgeEmbeddingConfigMongoModel
{
Provider = model.Provider,
Model = model.Model,
Dimension = model.Dimension
};
}
public static KnowledgeEmbeddingConfig ToDomainModel(KnowledgeEmbeddingConfigMongoModel model)
{
return new KnowledgeEmbeddingConfig
{
Provider = model.Provider,
Model = model.Model,
Dimension = model.Dimension
};
}
}

View file

@ -153,4 +153,7 @@ public class MongoDbContext
public IMongoCollection<TranslationMemoryDocument> TranslationMemories
=> Database.GetCollection<TranslationMemoryDocument>($"{_collectionPrefix}_TranslationMemories");
public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
=> Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
}

View file

@ -4,9 +4,40 @@ namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public bool SaveKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs) =>
throw new NotImplementedException();
public bool ResetKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs)
{
var docs = configs?.Select(x => new KnowledgeCollectionConfigDocument
{
Id = Guid.NewGuid().ToString(),
Name = x.Name,
Type = x.Type,
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding),
CreateDate = x.CreateDate,
CreateUserId = x.CreateUserId,
})?.ToList() ?? new List<KnowledgeCollectionConfigDocument>();
public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName) =>
throw new NotImplementedException();
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
_dc.KnowledgeCollectionConfigs.DeleteMany(filter);
_dc.KnowledgeCollectionConfigs.InsertMany(docs);
return true;
}
public VectorCollectionConfig? GetKnowledgeCollectionConfig(string collectionName)
{
if (string.IsNullOrWhiteSpace(collectionName)) return null;
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Name, collectionName);
var config = _dc.KnowledgeCollectionConfigs.Find(filter).FirstOrDefault();
if (config == null) return null;
return new VectorCollectionConfig
{
Name = config.Name,
Type = config.Type,
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(config.TextEmbedding),
CreateDate = config.CreateDate,
CreateUserId = config.CreateUserId
};
}
}