From e760bd1968904ae0853f79ab74857cee979e7878 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 20 Sep 2024 13:42:08 -0500 Subject: [PATCH] refine knowledge doc filter --- .../Knowledges/IKnowledgeService.cs | 28 ++++++++ .../Knowledges/Models/KnowledgeFileFilter.cs | 9 +++ .../FileRepository.KnowledgeBase.cs | 10 +++ .../Controllers/KnowledgeBaseController.cs | 13 ++-- .../Services/KnowledgeService.Document.cs | 67 ++++++++++++++++--- .../MongoRepository.KnowledgeBase.cs | 10 +++ 6 files changed, 123 insertions(+), 14 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 921d650b..447f80b1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -22,9 +22,37 @@ public interface IKnowledgeService #endregion #region Document + /// + /// Save documents and their contents to knowledgebase + /// + /// + /// + /// Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files); + /// + /// Save document content to knowledgebase without saving the document + /// + /// + /// + /// + /// + /// + /// Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, DocMetaRefData? refData = null); + /// + /// Delete one document and its related knowledge in the collection + /// + /// + /// + /// Task DeleteKnowledgeDocument(string collectionName, Guid fileId); + /// + /// Delete all documents and their related knowledge in the collection + /// + /// + /// + /// + Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs index 933b0505..d275cf34 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs @@ -4,5 +4,14 @@ public class KnowledgeFileFilter : Pagination { public IEnumerable? FileIds { get; set; } + public IEnumerable? FileNames { get; set; } + + public IEnumerable? ContentTypes { get; set; } + public IEnumerable? FileSources { get; set; } + + public KnowledgeFileFilter() + { + + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index f60d5507..53927629 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -189,10 +189,20 @@ public partial class FileRepository matched = matched && filter.FileIds.Contains(metaData.FileId); } + if (!filter.FileNames.IsNullOrEmpty()) + { + matched = matched && filter.FileNames.Contains(metaData.FileName); + } + if (!filter.FileSources.IsNullOrEmpty()) { matched = matched & filter.FileSources.Contains(metaData.FileSource); } + + if (!filter.ContentTypes.IsNullOrEmpty()) + { + matched = matched && filter.ContentTypes.Contains(metaData.ContentType); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index f59e0d01..af01a685 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -149,14 +149,17 @@ public class KnowledgeBaseController : ControllerBase return response; } + [HttpDelete("/knowledge/document/{collection}/delete")] + public async Task DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) + { + var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request); + return response; + } + [HttpPost("/knowledge/document/{collection}/page")] public async Task> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) { - var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter - { - Page = request.Page, - Size = request.Size - }); + var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request); return new PagedItems { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 744cd587..bb804673 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Knowledges.Helpers; using BotSharp.Abstraction.VectorStorage.Enums; +using System.Collections; using System.Net.Http; using System.Net.Mime; @@ -151,8 +152,8 @@ public partial class KnowledgeService // Get doc meta data var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { - FileIds = [ fileId ], - Size = 1 + Size = 1, + FileIds = [ fileId ] }); // Delete doc @@ -177,6 +178,57 @@ public partial class KnowledgeService } } + public async Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + { + if (string.IsNullOrWhiteSpace(collectionName)) return false; + + + var pageSize = filter.Size; + var innerFilter = new KnowledgeFileFilter + { + Page = 1, + Size = pageSize, + FileIds = filter.FileIds, + FileNames = filter.FileNames, + FileSources = filter.FileSources, + ContentTypes = filter.ContentTypes + }; + + var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + + var total = pageData.Count; + if (total == 0) return false; + + var page = 1; + var totalPages = total % pageSize == 0 ? total / pageSize : total / pageSize + 1; + + while (page <= totalPages) + { + if (page > 1) + { + pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + } + + var fileIds = pageData.Items.Select(x => x.FileId).ToList(); + foreach (var fileId in fileIds) + { + try + { + await DeleteKnowledgeDocument(collectionName, fileId); + } + catch + { + continue; + } + } + + page++; + } + + return true; + } + + public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) @@ -189,11 +241,7 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter - { - Page = filter.Page, - Size = filter.Size - }); + var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter); var files = pagedData.Items?.Select(x => new KnowledgeFileModel { @@ -222,8 +270,8 @@ public partial class KnowledgeService // Get doc binary data var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { - FileIds = [ fileId ], - Size = 1 + Size = 1, + FileIds = [ fileId ] }); var metaData = pageData?.Items?.FirstOrDefault(); @@ -247,6 +295,7 @@ public partial class KnowledgeService } + #region Private methods /// /// Get file content type and file bytes diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index 85e8b297..1ceffb44 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -191,10 +191,20 @@ public partial class MongoRepository docFilters.Add(builder.In(x => x.FileId, filter.FileIds)); } + if (!filter.FileNames.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.FileName, filter.FileNames)); + } + if (!filter.FileSources.IsNullOrEmpty()) { docFilters.Add(builder.In(x => x.FileSource, filter.FileSources)); } + + if (!filter.ContentTypes.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.ContentType, filter.ContentTypes)); + } } var filterDef = builder.And(docFilters);