From 47d7f0156296fd1cd717ea7ae3c78ff8854e6e5b Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 22 Aug 2024 22:13:40 -0500 Subject: [PATCH] add vector knowledge update --- .../Knowledges/IKnowledgeService.cs | 1 + .../VectorStorage/Models/VectorUpdateModel.cs | 8 ++++++ .../Controllers/KnowledgeBaseController.cs | 15 ++++++++++ .../VectorKnowledgeUpdateRequest.cs | 15 ++++++++++ .../Services/KnowledgeService.Delete.cs | 2 +- .../Services/KnowledgeService.Get.cs | 14 +++++----- .../Services/KnowledgeService.Update.cs | 28 +++++++++++++++++++ 7 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 5bf7aced..579ced84 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -10,6 +10,7 @@ public interface IKnowledgeService Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); Task> GetVectorCollectionData(string collectionName, VectorFilter filter); Task DeleteVectorCollectionData(string collectionName, string id); + Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs new file mode 100644 index 00000000..074bb7e8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorUpdateModel.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorUpdateModel +{ + public string Id { get; set; } + public string Text { get; set; } + public Dictionary? Payload { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 90773dfd..9ae1128a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Graph.Models; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.OpenAPI.ViewModels.Knowledges; +using System.Reflection.Metadata.Ecma335; namespace BotSharp.OpenAPI.Controllers; @@ -54,6 +55,20 @@ public class KnowledgeBaseController : ControllerBase }; } + [HttpPut("/knowledge/vector/{collection}/update")] + public async Task UpdateVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUpdateRequest request) + { + var update = new VectorUpdateModel + { + Id = request.Id, + Text = request.Text, + Payload = request.Payload + }; + + var updated = await _knowledgeService.UpdateVectorCollectionData(collection, update); + return updated; + } + [HttpDelete("/knowledge/vector/{collection}/data/{id}")] public async Task DeleteVectorCollectionData([FromRoute] string collection, [FromRoute] string id) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs new file mode 100644 index 00000000..4a440ab1 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUpdateRequest.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorKnowledgeUpdateRequest +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("text")] + public string Text { get; set; } + + [JsonPropertyName("payload")] + public Dictionary? Payload { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index 3cc72ae2..e78990a7 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -16,7 +16,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); return false; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 947d3f66..1843a6bc 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -9,12 +9,12 @@ public partial class KnowledgeService { try { - var collections = _settings.Collections.Select(x => x.Name).ToList(); - return collections; + var db = GetVectorDb(); + return await db.GetCollections(); } catch (Exception ex) { - _logger.LogWarning($"Error when getting knowledge collections. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}"); return Enumerable.Empty(); } } @@ -34,7 +34,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); return new StringIdPagedItems(); } } @@ -55,7 +55,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); return new List(); } } @@ -73,7 +73,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when searching graph {query}. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}"); return new GraphSearchResult(); } } @@ -99,7 +99,7 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when searching knowledge (vector collection: {collectionName}) {query}. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when searching knowledge (Vector collection: {collectionName}) (Query: {query}). {ex.Message}\r\n{ex.InnerException}"); return new KnowledgeSearchResult(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs new file mode 100644 index 00000000..2175e9d2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs @@ -0,0 +1,28 @@ +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid)) + { + return false; + } + + var textEmbedding = GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(update.Text); + + var db = GetVectorDb(); + return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } +}