add knowledge collections

This commit is contained in:
Jicheng Lu 2024-08-13 13:26:57 -05:00
parent 2be4dd3053
commit f2c1777ea9
7 changed files with 34 additions and 5 deletions

View file

@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Knowledges;
public interface IKnowledgeService
{
Task<IEnumerable<string>> GetKnowledgeCollections();
Task<IEnumerable<KnowledgeRetrievalResult>> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options);
Task FeedKnowledge(string collectionName, KnowledgeCreationModel model);
Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter);

View file

@ -31,6 +31,7 @@ public class SettingService : ISettingService
var plugins = pluginService.GetPlugins(_services);
var plugin = plugins.First(x => x.Module.Settings.Name == settingName);
var instance = plugin.Module.GetNewSettingsInstance();
_config.Bind(settingName, instance);
if (mask)
{

View file

@ -16,6 +16,12 @@ public class KnowledgeBaseController : ControllerBase
_services = services;
}
[HttpGet("knowledge/collections")]
public async Task<IEnumerable<string>> GetKnowledgeCollections()
{
return await _knowledgeService.GetKnowledgeCollections();
}
[HttpPost("/knowledge/{collection}/search")]
public async Task<IEnumerable<KnowledgeRetrivalViewModel>> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model)
{

View file

@ -21,7 +21,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback
embedding.SetModelName(_settings.TextEmbedding.Model);
var vector = await embedding.GetVectorAsync(args.Question);
var vectorDb = _services.GetRequiredService<IVectorDb>();
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List<string> { KnowledgePayloadName.Answer });
if (!knowledges.IsNullOrEmpty())

View file

@ -25,7 +25,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback
args.Question
});
var vectorDb = _services.GetRequiredService<IVectorDb>();
var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length);
var id = Guid.NewGuid().ToString();

View file

@ -2,6 +2,20 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task<IEnumerable<string>> GetKnowledgeCollections()
{
try
{
var db = GetVectorDb();
return await db.GetCollections();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting knowledge collections. {ex.Message}\r\n{ex.InnerException}");
return Enumerable.Empty<string>();
}
}
public async Task<StringIdPagedItems<KnowledgeCollectionData>> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter)
{
try

View file

@ -128,10 +128,17 @@ public class QdrantDb : IVectorDb
public async Task<IEnumerable<KnowledgeSearchResult>> Search(string collectionName, float[] vector,
IEnumerable<string> fields, int limit = 5, float confidence = 0.5f, bool withVector = false)
{
var client = GetClient();
var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
var results = new List<KnowledgeSearchResult>();
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
if (!exist)
{
return results;
}
var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
foreach (var point in points)
{
var data = new Dictionary<string, string>();