diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 52657646..d9d5eed4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -17,6 +17,7 @@ public interface IKnowledgeService Task DeleteVectorCollectionData(string collectionName, string id); Task DeleteVectorCollectionAllData(string collectionName); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); + Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); Task UpsertVectorCollectionData(string collectionName, VectorUpdateModel update); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index b078abb5..95d6257c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -14,8 +14,7 @@ public interface IVectorDb => throw new NotImplementedException(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter) => throw new NotImplementedException(); - Task> GetCollectionData(string collectionName, IEnumerable ids, - bool withPayload = false, bool withVector = false) + Task> GetCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) => throw new NotImplementedException(); Task CreateCollection(string collectionName, int dimension) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorQueryOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorQueryOptions.cs new file mode 100644 index 00000000..37bad608 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorQueryOptions.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorQueryOptions +{ + public bool WithPayload { get; set; } + public bool WithVector { get; set; } + + public static VectorQueryOptions Default() + { + return new(); + } +} diff --git a/src/Infrastructure/BotSharp.Core.SideCar/Services/BotSharpConversationSideCar.cs b/src/Infrastructure/BotSharp.Core.SideCar/Services/BotSharpConversationSideCar.cs index 3def01ef..79711cb6 100644 --- a/src/Infrastructure/BotSharp.Core.SideCar/Services/BotSharpConversationSideCar.cs +++ b/src/Infrastructure/BotSharp.Core.SideCar/Services/BotSharpConversationSideCar.cs @@ -112,7 +112,7 @@ public class BotSharpConversationSideCar : IConversationSideCar var response = await InnerExecute(agentId, text, postback, states); AfterExecute(); - _logger.LogInformation($"Existing side car conversation..."); + _logger.LogInformation($"Exiting side car conversation..."); return response; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 47067354..48392968 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -99,6 +99,19 @@ public class KnowledgeBaseController : ControllerBase return created; } + [HttpGet("/knowledge/vector/{collection}/points")] + public async Task> GetVectorCollectionData([FromRoute] string collection, [FromQuery] QueryVectorDataRequest request) + { + var options = new VectorQueryOptions + { + WithPayload = request.WithPayload, + WithVector = request.WithVector + }; + + var points = await _knowledgeService.GetVectorCollectionData(collection, request.Ids, options); + return points.Select(x => VectorKnowledgeViewModel.From(x)); + } + [HttpPut("/knowledge/vector/{collection}/update")] public async Task UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request) { @@ -125,7 +138,10 @@ public class KnowledgeBaseController : ControllerBase { return await _knowledgeService.DeleteVectorCollectionAllData(collection); } + #endregion + + #region Index [HttpPost("/knowledge/vector/{collection}/payload/indexes")] public async Task> CreateCollectionPayloadIndexes([FromRoute] string collection, [FromBody] CreateVectorCollectionIndexRequest request) { @@ -277,6 +293,7 @@ public class KnowledgeBaseController : ControllerBase } #endregion + #region Private methods private FileStreamResult BuildFileResult(string fileName, BinaryData fileData) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs new file mode 100644 index 00000000..9aa9566e --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/QueryVectorDataRequest.cs @@ -0,0 +1,8 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges.Request; + +public class QueryVectorDataRequest +{ + public IEnumerable Ids { get; set; } = []; + public bool WithVector { get; set; } + public bool WithPayload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs index bbd7dc3a..61b96a97 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorKnowledgeViewModel.cs @@ -30,4 +30,14 @@ public class VectorKnowledgeViewModel Vector = result.Vector }; } + + public static VectorKnowledgeViewModel From(VectorCollectionData data) + { + return new VectorKnowledgeViewModel + { + Id = data.Id, + Data = data.Data, + Vector = data.Vector + }; + } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index a2327298..27823202 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -39,8 +39,7 @@ public class MemoryVectorDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(string collectionName, IEnumerable ids, - bool withPayload = false, bool withVector = false) + public Task> GetCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 491261d8..d55d5a4a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -170,7 +170,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})"); + _logger.LogError(ex, $"Error when importing doc content to knowledgebase ({collectionName}-{fileName})"); return false; } } @@ -212,7 +212,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when deleting knowledge document " + + _logger.LogError(ex, $"Error when deleting knowledge document " + $"(Collection: {collectionName}, File id: {fileId})"); return false; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs index dbbe8885..7207bace 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs @@ -15,7 +15,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when searching graph knowledge (Query: {query})."); + _logger.LogError(ex, $"Error when searching graph knowledge (Query: {query})."); return new GraphSearchResult(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs index 96cd3f25..640464de 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Index.cs @@ -18,14 +18,14 @@ public partial class KnowledgeService foreach (var option in options) { var created = await vectorDb.CreateCollectionPayloadIndex(collectionName, option); - var field = $"{option.FieldName}-{option.FieldSchemaType}"; + var field = $"{option.FieldName} ({option.FieldSchemaType})"; if (created) { response.Success.Add(field); } else { - _logger.LogError($"Failed to create vector collection payload index ({collectionName}-{field})."); + _logger.LogError($"Failed to create vector collection payload index ({collectionName} => {field})."); response.Fail.Add(field); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index 102300f2..f21fe1de 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.VectorStorage.Enums; -using System; namespace BotSharp.Plugin.KnowledgeBase.Services; @@ -63,7 +62,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when creating a vector collection ({collectionName})."); + _logger.LogError(ex, $"Error when creating a vector collection ({collectionName})."); return false; } } @@ -85,7 +84,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when getting vector db collections."); + _logger.LogError(ex, $"Error when getting vector db collections."); return Enumerable.Empty(); } } @@ -112,7 +111,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when getting vector db collection details."); + _logger.LogError(ex, $"Error when getting vector db collection details."); return null; } } @@ -144,7 +143,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when deleting collection ({collectionName})."); + _logger.LogError(ex, $"Error when deleting collection ({collectionName})."); return false; } } @@ -172,11 +171,33 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when creating vector collection data."); + _logger.LogError(ex, $"Error when creating vector collection data."); return false; } } + public async Task> GetVectorCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || ids.IsNullOrEmpty()) + { + return []; + } + + var db = GetVectorDb(); + var pointIds = ids.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)); + var points = await db.GetCollectionData(collectionName, pointIds, options); + return points; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when querying vector collection {collectionName} points."); + return []; + } + } + + public async Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update) { try @@ -204,7 +225,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when updating vector collection data."); + _logger.LogError(ex, $"Error when updating vector collection data."); return false; } } @@ -221,7 +242,7 @@ public partial class KnowledgeService } var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, [guid], withVector: true, withPayload: true); + var found = await db.GetCollectionData(collectionName, [guid], options: new() { WithVector = true, WithPayload = true }); if (!found.IsNullOrEmpty()) { if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text) @@ -240,7 +261,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when updating vector collection data."); + _logger.LogError(ex, $"Error when updating vector collection data."); return false; } } @@ -259,7 +280,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}-{id})."); + _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName}-{id})."); return false; } } @@ -274,7 +295,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName})."); + _logger.LogError(ex, $"Error when deleting vector collection data ({collectionName})."); return false; } } @@ -294,7 +315,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when getting vector knowledge collection data ({collectionName})."); + _logger.LogError(ex, $"Error when getting vector knowledge collection data ({collectionName})."); return new StringIdPagedItems(); } } @@ -315,7 +336,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when searching vector knowledge ({collectionName})."); + _logger.LogError(ex, $"Error when searching vector knowledge ({collectionName})."); return Enumerable.Empty(); } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index de582b7c..4bca24a2 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -185,8 +185,7 @@ public class QdrantDb : IVectorDb } - public async Task> GetCollectionData(string collectionName, IEnumerable ids, - bool withPayload = false, bool withVector = false) + public async Task> GetCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) { if (ids.IsNullOrEmpty()) { @@ -201,7 +200,7 @@ public class QdrantDb : IVectorDb var client = GetClient(); var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList(); - var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector); + var points = await client.RetrieveAsync(collectionName, pointIds, options?.WithPayload ?? false, options?.WithVector ?? false); return points.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 2189e8a2..e87ea423 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -48,8 +48,7 @@ namespace BotSharp.Plugin.SemanticKernel throw new NotImplementedException(); } - public Task> GetCollectionData(string collectionName, IEnumerable ids, - bool withPayload = false, bool withVector = false) + public Task> GetCollectionData(string collectionName, IEnumerable ids, VectorQueryOptions? options = null) { throw new NotImplementedException(); }