update settings

This commit is contained in:
Jicheng Lu 2024-08-30 19:31:18 -05:00
parent a528a3f6b9
commit f38fdf7497
22 changed files with 46 additions and 31 deletions

View file

@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters;
public interface IPdf2ImageConverter
{
public string Name { get; }
public string Provider { get; }
/// <summary>
/// Convert pdf pages to images, and return a list of image file paths

View file

@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files;
public class FileCoreSettings
{
public string Storage { get; set; } = FileStorageEnum.LocalFileStorage;
public string Pdf2TextConverter { get; set; }
public string Pdf2ImageConverter { get; set; }
public SettingBase Pdf2TextConverter { get; set; }
public SettingBase Pdf2ImageConverter { get; set; }
}
public class SettingBase
{
public string Provider { get; set; }
}

View file

@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph;
public interface IGraphDb
{
public string Name { get; }
public string Provider { get; }
Task<GraphSearchData> Search(string query, GraphSearchOptions options);
}

View file

@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges
{
public interface IPdf2TextConverter
{
public string Name { get; }
public string Provider { get; }
Task<string> ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum);
}
}

View file

@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Knowledges.Settings;
public class KnowledgeBaseSettings
{
public string VectorDb { get; set; }
public string GraphDb { get; set; }
public SettingBase VectorDb { get; set; }
public SettingBase GraphDb { get; set; }
public DefaultKnowledgeBaseSetting Default { get; set; }
public List<VectorCollectionSetting> Collections { get; set; } = new();
@ -23,9 +23,13 @@ public class VectorCollectionSetting
public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; }
}
public class KnowledgeTextEmbeddingSetting
public class KnowledgeTextEmbeddingSetting : SettingBase
{
public string Provider { get; set; }
public string Model { get; set; }
public int Dimension { get; set; }
}
public class SettingBase
{
public string Provider { get; set; }
}

View file

@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.VectorStorage;
public interface IVectorDb
{
string Name { get; }
string Provider { get; }
Task<IEnumerable<string>> GetCollections();
Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);

View file

@ -98,7 +98,7 @@ public partial class FileInstructService
{
var images = new List<string>();
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter);
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
if (converter == null || files.IsNullOrEmpty())
{
return images;

View file

@ -276,7 +276,7 @@ public partial class LocalFileStorageService
private IPdf2ImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter);
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

View file

@ -104,7 +104,7 @@ public class KnowledgeBaseController : ControllerBase
public async Task<IActionResult> UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum)
{
var setttings = _services.GetRequiredService<FileCoreSettings>();
var textConverter = _services.GetServices<IPdf2TextConverter>().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter);
var textConverter = _services.GetServices<IPdf2TextConverter>().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider);
var filePath = Path.GetTempFileName();
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None))

View file

@ -35,7 +35,7 @@ public class GraphDb : IGraphDb
_settings = settings;
}
public string Name => "Remote";
public string Provider => "Remote";
public async Task<GraphSearchData> Search(string query, GraphSearchOptions options)
{

View file

@ -21,7 +21,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback
var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName);
var vector = await embedding.GetVectorAsync(args.Question);
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider);
var knowledges = await vectorDb.Search(collectionName, vector, new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer });
if (!knowledges.IsNullOrEmpty())

View file

@ -25,7 +25,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback
args.Question
});
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider);
await vectorDb.CreateCollection(collectionName, vector[0].Length);
var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0],

View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Settings;
namespace BotSharp.Plugin.KnowledgeBase.Helpers;
public static class KnowledgeSettingHelper

View file

@ -8,7 +8,7 @@ public class MemoryVectorDb : IVectorDb
private readonly Dictionary<string, List<VecRecord>> _vectors = new Dictionary<string, List<VecRecord>>();
public string Name => "MemoryVector";
public string Provider => "MemoryVector";
public async Task<bool> CreateCollection(string collectionName, int dimension)
{

View file

@ -22,13 +22,13 @@ public partial class KnowledgeService : IKnowledgeService
private IVectorDb GetVectorDb()
{
var db = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
var db = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider);
return db;
}
private IGraphDb GetGraphDb()
{
var db = _services.GetServices<IGraphDb>().FirstOrDefault(x => x.Name == _settings.GraphDb);
var db = _services.GetServices<IGraphDb>().FirstOrDefault(x => x.Provider == _settings.GraphDb.Provider);
return db;
}

View file

@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
public class PigPdf2TextConverter : IPdf2TextConverter
{
public string Name => "Pig";
public string Provider => "Pig";
public Task<string> ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum)
{

View file

@ -9,7 +9,7 @@ namespace BotSharp.Plugin.MetaAI.Providers;
public class FaissDb : IVectorDb
{
public string Name => "Faiss";
public string Provider => "Faiss";
public Task<bool> CreateCollection(string collectionName, int dimension)
{

View file

@ -29,7 +29,7 @@ public class Pdf2TextConverter : IPdf2TextConverter
_paddleSharpSettings = paddleSharpSettings;
}
public string Name => "Paddle";
public string Provider => "Paddle";
public async Task<string> ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum)
{

View file

@ -19,7 +19,7 @@ public class QdrantDb : IVectorDb
_services = services;
}
public string Name => "Qdrant";
public string Provider => "Qdrant";
private QdrantClient GetClient()
{

View file

@ -22,7 +22,7 @@ namespace BotSharp.Plugin.SemanticKernel
}
public string Name => "SemanticKernel";
public string Provider => "SemanticKernel";
public async Task<bool> CreateCollection(string collectionName, int dimension)
{

View file

@ -254,7 +254,7 @@ public partial class TencentCosService
private IPdf2ImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter);
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

View file

@ -233,8 +233,12 @@
"FileCore": {
"Storage": "LocalFileStorage",
"Pdf2TextConverter": "",
"Pdf2ImageConverter": ""
"Pdf2TextConverter": {
"Provider": ""
},
"Pdf2ImageConverter": {
"Provider": ""
}
},
"TencentCos": {
@ -262,8 +266,12 @@
},
"KnowledgeBase": {
"VectorDb": "Qdrant",
"GraphDb": "Default",
"VectorDb": {
"Provider": "Qdrant"
},
"GraphDb": {
"Provider": "Remote"
},
"Default": {
"CollectionName": "BotSharp",
"TextEmbedding": {