SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #657 from iceljc/master

Browse source 
add collection exist
This commit is contained in:
iceljc 2024-09-24 15:37:37 -05:00 committed by GitHub
parent 2b6b92c023 689c5d2572
commit ec6db642de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 81 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -6,6 +6,7 @@ namespace BotSharp.Abstraction.Knowledges;
public interface IKnowledgeService
{
#region Vector
Task<bool> ExistVectorCollection(string collectionName);
Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
Task<bool> DeleteVectorCollection(string collectionName);
Task<IEnumerable<string>> GetVectorCollections(string type);

1
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -116,7 +116,6 @@ public interface IBotSharpRepository
bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
bool DeleteKnowledgeCollectionConfig(string collectionName);
IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
/// <summary>
/// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.

3
src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
View file

@ -5,7 +5,8 @@ namespace BotSharp.Abstraction.VectorStorage;
public interface IVectorDb
{
string Provider { get; }
Task<bool> DoesCollectionExist(string collectionName);
Task<IEnumerable<string>> GetCollections();
Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);

2
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs
View file

@ -213,7 +213,7 @@ public partial class FileRepository
return new PagedItems<KnowledgeDocMetaData>
{
Items = records.Skip(filter.Offset).Take(filter.Size),
Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size),
Count = records.Count
};
}

7
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Graph.Models;
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.VectorStorage.Models;
using BotSharp.OpenAPI.ViewModels.Knowledges;
@ -20,6 +19,12 @@ public class KnowledgeBaseController : ControllerBase
}
#region Vector
[HttpGet("knowledge/vector/{collection}/exist")]
public async Task<bool> ExistVectorCollection([FromRoute] string collection)
{
return await _knowledgeService.ExistVectorCollection(collection);
}
[HttpGet("knowledge/vector/collections")]
public async Task<IEnumerable<string>> GetVectorCollections([FromQuery] string type)
{

6
src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
View file

@ -10,6 +10,12 @@ public class MemoryVectorDb : IVectorDb
public string Provider => "MemoryVector";
public async Task<bool> DoesCollectionExist(string collectionName)
{
return false;
}
public async Task<bool> CreateCollection(string collectionName, int dimension)
{
_collections[collectionName] = dimension;

22
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -3,7 +3,6 @@ using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Knowledges.Helpers;
using BotSharp.Abstraction.VectorStorage.Enums;
using System.Collections;
using System.Net.Http;
using System.Net.Mime;
@ -13,13 +12,21 @@ public partial class KnowledgeService
{
public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
{
var res = new UploadKnowledgeResponse
{
Success = [],
Failed = files?.Select(x => x.FileName) ?? new List<string>()
};
if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
{
return new UploadKnowledgeResponse
{
Success = [],
Failed = files?.Select(x => x.FileName) ?? new List<string>()
};
return res;
}
var exist = await ExistVectorCollection(collectionName);
if (!exist)
{
return res;
}
var db = _services.GetRequiredService<IBotSharpRepository>();
@ -103,6 +110,9 @@ public partial class KnowledgeService
try
{
var exist = await ExistVectorCollection(collectionName);
if (!exist) return false;
var db = _services.GetRequiredService<IBotSharpRepository>();
var userId = await GetUserId();
var vectorStoreProvider = _settings.VectorDb.Provider;

17
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
View file

@ -7,6 +7,23 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
#region Collection
public async Task<bool> ExistVectorCollection(string collectionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var vectorDb = GetVectorDb();
var exist = await vectorDb.DoesCollectionExist(collectionName);
if (exist) return true;
var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
{
CollectionNames = [collectionName],
VectorStroageProviders = [_settings.VectorDb.Provider]
});
return !configs.IsNullOrEmpty();
}
public async Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model)
{
try

49
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -39,16 +39,22 @@ public class QdrantDb : IVectorDb
return _client;
}
public async Task<bool> CreateCollection(string collectionName, int dimension)
public async Task<bool> DoesCollectionExist(string collectionName)
{
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
return await client.CollectionExistsAsync(collectionName);
}
public async Task<bool> CreateCollection(string collectionName, int dimension)
{
var exist = await DoesCollectionExist(collectionName);
if (exist) return false;
try
{
// Create a new collection
var client = GetClient();
await client.CreateCollectionAsync(collectionName, new VectorParams()
{
Size = (ulong)dimension,
@ -65,11 +71,11 @@ public class QdrantDb : IVectorDb
public async Task<bool> DeleteCollection(string collectionName)
{
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
var exist = await DoesCollectionExist(collectionName);
if (!exist) return false;
var client = GetClient();
await client.DeleteCollectionAsync(collectionName);
return true;
}
@ -83,8 +89,7 @@ public class QdrantDb : IVectorDb
public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
{
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
var exist = await DoesCollectionExist(collectionName);
if (!exist)
{
return new StringIdPagedItems<VectorCollectionData>();
@ -126,6 +131,7 @@ public class QdrantDb : IVectorDb
};
}
var client = GetClient();
var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size,
offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
@ -152,15 +158,18 @@ public class QdrantDb : IVectorDb
public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
bool withPayload = false, bool withVector = false)
{
if (ids.IsNullOrEmpty()) return Enumerable.Empty<VectorCollectionData>();
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
if (ids.IsNullOrEmpty())
{
return Enumerable.Empty<VectorCollectionData>();
}
var exist = await DoesCollectionExist(collectionName);
if (!exist)
{
return Enumerable.Empty<VectorCollectionData>();
}
var client = GetClient();
var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
return points.Select(x => new VectorCollectionData
@ -209,8 +218,7 @@ public class QdrantDb : IVectorDb
{
var results = new List<VectorCollectionData>();
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
var exist = await DoesCollectionExist(collectionName);
if (!exist)
{
return results;
@ -221,7 +229,8 @@ public class QdrantDb : IVectorDb
{
payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } };
}
var client = GetClient();
var points = await client.SearchAsync(collectionName,
vector,
limit: (ulong)limit,
@ -244,33 +253,27 @@ public class QdrantDb : IVectorDb
{
if (ids.IsNullOrEmpty()) return false;
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
var exist = await DoesCollectionExist(collectionName);
if (!exist)
{
return false;
}
var client = GetClient();
var result = await client.DeleteAsync(collectionName, ids);
return result.Status == UpdateStatus.Completed;
}
public async Task<bool> DeleteCollectionAllData(string collectionName)
{
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
var exist = await DoesCollectionExist(collectionName);
if (!exist)
{
return false;
}
var client = GetClient();
var result = await client.DeleteAsync(collectionName, new Filter());
return result.Status == UpdateStatus.Completed;
}
private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
{
return await client.CollectionExistsAsync(collectionName);
}
}

6
src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
View file

@ -25,6 +25,12 @@ namespace BotSharp.Plugin.SemanticKernel
public string Provider => "SemanticKernel";
public async Task<bool> DoesCollectionExist(string collectionName)
{
return false;
}
public async Task<bool> CreateCollection(string collectionName, int dimension)
{
await _memoryStore.CreateCollectionAsync(collectionName);