diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f2eb82fe..f145aa6f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,5 +14,7 @@ public interface IKnowledgeService #region List Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); + Task> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter); + Task DeleteKnowledgeCollectionData(string collectionName, string id); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs new file mode 100644 index 00000000..131bacc0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeRetrievalResult +{ + public string Id { get; set; } + public string Text { get; set; } + public float Score { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs index 296e1ed6..18e55634 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.Abstraction.Knowledges.Models; public class RetrievedResult diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 50788756..413e998a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,9 +4,10 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - Task> GetCollections(); + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index cd01ec05..debec995 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; -using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -95,7 +94,7 @@ public class KnowledgeBaseController : ControllerBase [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) - {; + { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); @@ -107,4 +106,20 @@ public class KnowledgeBaseController : ControllerBase Items = items }; } + + [HttpPost("/knowledge/{collection}/similar")] + public async Task> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + { + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + .ToList() ?? new List(); + + return + } + + [HttpDelete("/knowledge/{collection}/data/{id}")] + public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) + { + return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 16ebadda..5ce43369 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -9,9 +9,11 @@ public class KnowledgeCollectionDataViewModel public string Id { get; set; } [JsonPropertyName("question")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Question { get; set; } [JsonPropertyName("answer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Answer { get; set; } [JsonPropertyName("vector")] diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index 61598a71..f7c9276d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -7,13 +7,14 @@ public class MemVectorDatabase : IVectorDb { private readonly Dictionary _collections = new Dictionary(); private readonly Dictionary> _vectors = new Dictionary>(); + public async Task CreateCollection(string collectionName, int dim) { _collections[collectionName] = dim; _vectors[collectionName] = new List(); } - public async Task> GetCollections() + public async Task> GetCollections() { return _collections.Select(x => x.Key).ToList(); } @@ -23,7 +24,7 @@ public class MemVectorDatabase : IVectorDb throw new NotImplementedException(); } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) { @@ -54,6 +55,12 @@ public class MemVectorDatabase : IVectorDb return true; } + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } + + #region Private methods private float[] CalEuclideanDistance(float[] vec, List records) { var a = np.zeros((records.Count, vec.Length), np.float32); @@ -69,7 +76,7 @@ public class MemVectorDatabase : IVectorDb return c.ToArray(); } - public NDArray CalCosineSimilarity(float[] vec, List records) + private NDArray CalCosineSimilarity(float[] vec, List records) { var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); @@ -113,7 +120,7 @@ public class MemVectorDatabase : IVectorDb return resIndex.ToArray(); } - public (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) { var squaredX = np.sum(np.multiply(x, x), axis: 1); var normX = np.sqrt(squaredX); @@ -128,4 +135,5 @@ public class MemVectorDatabase : IVectorDb return (x / normX, normX); } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 2f8547f4..d3b5ed84 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -11,8 +11,22 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); return new StringIdPagedItems(); } } + + public async Task DeleteKnowledgeCollectionData(string collectionName, string id) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionData(collectionName, id); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 5983d49a..7e884632 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,12 +19,12 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollections() + public Task> GetCollections() { throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) { throw new NotImplementedException(); } @@ -33,4 +33,9 @@ public class FaissDb : IVectorDb { throw new NotImplementedException(); } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index d331e28e..f7bb25c5 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -32,7 +32,7 @@ public class QdrantDb : IVectorDb return _client; } - public async Task> GetCollections() + public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); @@ -100,7 +100,6 @@ public class QdrantDb : IVectorDb Uuid = id }, Vectors = vector, - Payload = { { KnowledgePayloadName.Text, text } @@ -125,13 +124,19 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { var client = GetClient(); - var points = await client.SearchAsync(collectionName, vector, - limit: (ulong)limit, - scoreThreshold: confidence); + var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); } + + public async Task DeleteCollectionData(string collectionName, string id) + { + var client = GetClient(); + var guid = Guid.Parse(id); + var result = await client.DeleteAsync(collectionName, guid); + return result.Status == UpdateStatus.Completed; + } } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index c831d15b..b52ab71e 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; +using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -29,7 +30,7 @@ namespace BotSharp.Plugin.SemanticKernel throw new System.NotImplementedException(); } - public async Task> GetCollections() + public async Task> GetCollections() { var result = new List(); await foreach (var collection in _memoryStore.GetCollectionsAsync()) @@ -39,7 +40,7 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); @@ -60,5 +61,10 @@ namespace BotSharp.Plugin.SemanticKernel #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. return true; } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } }