diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 688e939e..52657646 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Models; using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Abstraction.Knowledges; @@ -69,6 +70,11 @@ public interface IKnowledgeService Task DeleteVectorCollectionSnapshot(string collectionName, string snapshotName); #endregion + #region Index + Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options); + #endregion + #region Common Task RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/SuccessFailResponse.cs b/src/Infrastructure/BotSharp.Abstraction/Models/SuccessFailResponse.cs new file mode 100644 index 00000000..11985259 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/SuccessFailResponse.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Models; + +public class SuccessFailResponse +{ + public List Success { get; set; } = []; + public List Fail { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 39d80360..b078abb5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -39,4 +39,9 @@ public interface IVectorDb => throw new NotImplementedException(); Task DeleteCollectionShapshot(string collectionName, string snapshotName) => throw new NotImplementedException(); + + Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) + => throw new NotImplementedException(); + Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) + => throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionIndexOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionIndexOptions.cs new file mode 100644 index 00000000..809bbf6b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionIndexOptions.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionIndexOptions +{ + [JsonPropertyName("field_name")] + public string FieldName { get; set; } = null!; +} + +public class CreateVectorCollectionIndexOptions : VectorCollectionIndexOptions +{ + [JsonPropertyName("field_schema_type")] + public string FieldSchemaType { get; set; } = null!; +} + +public class DeleteVectorCollectionIndexOptions : VectorCollectionIndexOptions +{ + +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 07994e85..9752e02c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Graph.Models; using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.OpenAPI.ViewModels.Knowledges; +using BotSharp.OpenAPI.ViewModels.Knowledges.Request; namespace BotSharp.OpenAPI.Controllers; @@ -125,16 +126,16 @@ public class KnowledgeBaseController : ControllerBase return await _knowledgeService.DeleteVectorCollectionAllData(collection); } - [HttpPost("/knowledge/vector/{collection}/payload/index")] - public async Task CreateCollectionPayloadIndex() + [HttpPost("/knowledge/vector/{collection}/payload/indexes")] + public async Task> CreateCollectionPayloadIndexes([FromRoute] string collection, [FromBody] CreateVectorCollectionIndexRequest request) { - return false; + return await _knowledgeService.CreateVectorCollectionPayloadIndexes(collection, request.Options); } - [HttpDelete("/knowledge/vector/{collection}/payload/index")] - public async Task DeleteCollectionPayloadIndex() + [HttpDelete("/knowledge/vector/{collection}/payload/indexes")] + public async Task> DeleteCollectionPayloadIndexes([FromRoute] string collection, [FromBody] DeleteVectorCollectionIndexRequest request) { - return false; + return await _knowledgeService.DeleteVectorCollectionPayloadIndexes(collection, request.Options); } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs new file mode 100644 index 00000000..fada7032 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/VectorCollectionIndexRequest.cs @@ -0,0 +1,13 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges.Request; + +public class CreateVectorCollectionIndexRequest +{ + public IEnumerable Options { get; set; } = []; +} + +public class DeleteVectorCollectionIndexRequest +{ + public IEnumerable Options { get; set; } = []; +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs new file mode 100644 index 00000000..96cd3f25 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs @@ -0,0 +1,74 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty()) + { + return new(); + } + + var response = new SuccessFailResponse(); + var vectorDb = GetVectorDb(); + foreach (var option in options) + { + var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option); + var field = $"{option.FieldName}-{option.FieldSchemaType}"; + if (created) + { + response.Success.Add(field); + } + else + { + _logger.LogError($"Failed to create vector collection payload index ({collectionName}-{field})."); + response.Fail.Add(field); + } + } + return response; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when creating vector collection payload index ({collectionName})."); + return new(); + } + } + + public async Task> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable options) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || options.IsNullOrEmpty()) + { + return new(); + } + + var response = new SuccessFailResponse(); + var vectorDb = GetVectorDb(); + foreach (var option in options) + { + var deleted = await vectorDb.DeleteCollectionPayloadIndex(collectionName, option); + var field = $"{option.FieldName}"; + if (deleted) + { + response.Success.Add(field); + } + else + { + _logger.LogError($"Failed to deleting vector collection payload index ({collectionName}-{field})."); + response.Fail.Add(field); + } + } + return response; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when deleting vector collection payload index ({collectionName})."); + return new(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 7d13620b..40cf6982 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -402,17 +402,32 @@ public class QdrantDb : IVectorDb } - //public async Task CreateCollectionPayloadIndex(string collectionName) - //{ - // var exist = await DoesCollectionExist(collectionName); - // if (!exist) - // { - // return false; - // } + public async Task CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return false; + } - // var client = GetClient(); - // var result = await client.CreatePayloadIndexAsync(collectionName, "text", PayloadSchemaType.Keyword); - //} + var client = GetClient(); + var schemaType = ConvertPayloadSchemaType(options.FieldSchemaType); + var result = await client.CreatePayloadIndexAsync(collectionName, options.FieldName, schemaType); + return result.Status == UpdateStatus.Completed; + } + + public async Task DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options) + { + var exist = await DoesCollectionExist(collectionName); + if (!exist) + { + return false; + } + + var client = GetClient(); + var result = await client.DeletePayloadIndexAsync(collectionName, options.FieldName); + return result.Status == UpdateStatus.Completed; + } #endregion #region Snapshots @@ -559,4 +574,44 @@ public class QdrantDb : IVectorDb } } #endregion + + + #region Private methods + private PayloadSchemaType ConvertPayloadSchemaType(string schemaType) + { + PayloadSchemaType res; + switch (schemaType.ToLower()) + { + case "text": + res = PayloadSchemaType.Text; + break; + case "keyword": + res = PayloadSchemaType.Keyword; + break; + case "integer": + res = PayloadSchemaType.Integer; + break; + case "float": + res = PayloadSchemaType.Float; + break; + case "bool": + res = PayloadSchemaType.Bool; + break; + case "geo": + res = PayloadSchemaType.Geo; + break; + case "datetime": + res = PayloadSchemaType.Datetime; + break; + case "uuid": + res = PayloadSchemaType.Uuid; + break; + default: + res = PayloadSchemaType.UnknownType; + break; + } + + return res; + } + #endregion }