diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs index 36f300a2..c3ca76fc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs @@ -5,6 +5,7 @@ public class KnowledgeFileModel public Guid FileId { get; set; } public string FileName { get; set; } public string FileExtension { get; set; } + public string FileSource { get; set; } public string ContentType { get; set; } public string FileUrl { get; set; } public DocMetaRefData? RefData { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs index 8414f002..af3372db 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs @@ -11,4 +11,5 @@ public static class KnowledgePayloadName public static string FileId = "fileId"; public static string FileName = "fileName"; public static string FileSource = "fileSource"; + public static string FileUrl = "fileUrl"; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Helpers/TextChopper.cs similarity index 85% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Helpers/TextChopper.cs index 8421e537..a3f88a56 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Helpers/TextChopper.cs @@ -1,6 +1,6 @@ using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +namespace BotSharp.Abstraction.Knowledges.Helpers; public static class TextChopper { @@ -14,18 +14,22 @@ public static class TextChopper private static List ChopByWord(string content, ChunkOption option) { var chunks = new List(); - var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); + var words = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); var chunk = string.Empty; for (int i = 0; i < words.Count; i++) { - chunk += words[i] + " "; + chunk += words[i]; if (chunk.Length > option.Size) { chunks.Add(chunk.Trim()); chunk = string.Empty; i -= option.Conjunction; } + else + { + chunk += " "; + } } if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk)) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 307cb57d..447f80b1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -22,8 +22,37 @@ public interface IKnowledgeService #endregion #region Document - Task UploadKnowledgeDocuments(string collectionName, IEnumerable files); + /// + /// Save documents and their contents to knowledgebase + /// + /// + /// + /// + Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files); + /// + /// Save document content to knowledgebase without saving the document + /// + /// + /// + /// + /// + /// + /// + Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable contents, DocMetaRefData? refData = null); + /// + /// Delete one document and its related knowledge in the collection + /// + /// + /// + /// Task DeleteKnowledgeDocument(string collectionName, Guid fileId); + /// + /// Delete all documents and their related knowledge in the collection + /// + /// + /// + /// + Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter); Task GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs index e111bb5b..a1139c90 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs @@ -42,10 +42,13 @@ public class DocMetaRefData [JsonPropertyName("name")] public string Name { get; set; } + [JsonPropertyName("type")] + public string Type { get; set; } + [JsonPropertyName("url")] public string Url { get; set; } - [JsonPropertyName("json_content")] + [JsonPropertyName("data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? JsonContent { get; set; } + public IDictionary? Data { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs index 8dd17461..d275cf34 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs @@ -3,4 +3,15 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFileFilter : Pagination { public IEnumerable? FileIds { get; set; } + + public IEnumerable? FileNames { get; set; } + + public IEnumerable? ContentTypes { get; set; } + + public IEnumerable? FileSources { get; set; } + + public KnowledgeFileFilter() + { + + } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index 3ef93da4..53927629 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -182,10 +182,29 @@ public partial class FileRepository var matched = true; // Apply filter - if (filter != null && !filter.FileIds.IsNullOrEmpty()) + if (filter != null) { - matched = matched && filter.FileIds.Contains(metaData.FileId); + if (!filter.FileIds.IsNullOrEmpty()) + { + matched = matched && filter.FileIds.Contains(metaData.FileId); + } + + if (!filter.FileNames.IsNullOrEmpty()) + { + matched = matched && filter.FileNames.Contains(metaData.FileName); + } + + if (!filter.FileSources.IsNullOrEmpty()) + { + matched = matched & filter.FileSources.Contains(metaData.FileSource); + } + + if (!filter.ContentTypes.IsNullOrEmpty()) + { + matched = matched && filter.ContentTypes.Contains(metaData.ContentType); + } } + if (!matched) continue; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 0acfff2e..af01a685 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -115,7 +115,7 @@ public class KnowledgeBaseController : ControllerBase [HttpPost("/knowledge/document/{collection}/upload")] public async Task UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) { - var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files); + var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, request.Files); return response; } @@ -138,7 +138,7 @@ public class KnowledgeBaseController : ControllerBase }); } - var response = await _knowledgeService.UploadKnowledgeDocuments(collection, docs); + var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, docs); return response; } @@ -149,14 +149,17 @@ public class KnowledgeBaseController : ControllerBase return response; } - [HttpPost("/knowledge/document/{collection}/list")] + [HttpDelete("/knowledge/document/{collection}/delete")] + public async Task DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) + { + var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request); + return response; + } + + [HttpPost("/knowledge/document/{collection}/page")] public async Task> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request) { - var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter - { - Page = request.Page, - Size = request.Size - }); + var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request); return new PagedItems { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs index 1d1ce27e..c9ea7ac3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs @@ -10,6 +10,9 @@ public class KnowledgeFileViewModel [JsonPropertyName("file_name")] public string FileName { get; set; } + [JsonPropertyName("file_source")] + public string FileSource { get; set; } + [JsonPropertyName("file_extension")] public string FileExtension { get; set; } @@ -29,6 +32,7 @@ public class KnowledgeFileViewModel { FileId = model.FileId, FileName = model.FileName, + FileSource = model.FileSource, FileExtension = model.FileExtension, ContentType = model.ContentType, FileUrl = model.FileUrl, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 811df72a..bb804673 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -1,7 +1,9 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Knowledges.Helpers; using BotSharp.Abstraction.VectorStorage.Enums; +using System.Collections; using System.Net.Http; using System.Net.Mime; @@ -9,7 +11,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task UploadKnowledgeDocuments(string collectionName, IEnumerable files) + public async Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files) { if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty()) { @@ -89,6 +91,50 @@ public partial class KnowledgeService } + public async Task ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, + IEnumerable contents, DocMetaRefData? refData = null) + { + if (string.IsNullOrWhiteSpace(collectionName) + || string.IsNullOrWhiteSpace(fileName) + || contents.IsNullOrEmpty()) + { + return false; + } + + try + { + var db = _services.GetRequiredService(); + var userId = await GetUserId(); + var vectorStoreProvider = _settings.VectorDb.Provider; + var fileId = Guid.NewGuid(); + var contentType = FileUtility.GetFileContentType(fileName); + + var dataIds = await SaveToVectorDb(collectionName, fileId, fileName, contents, fileSource, fileUrl: refData?.Url); + db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData + { + Collection = collectionName, + FileId = fileId, + FileName = fileName, + FileSource = fileSource, + ContentType = contentType, + VectorStoreProvider = vectorStoreProvider, + VectorDataIds = dataIds, + RefData = refData, + CreateDate = DateTime.UtcNow, + CreateUserId = userId + }); + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when importing doc content to knowledgebase ({collectionName}-{fileName})" + + $"\r\n{ex.Message}" + + $"\r\n{ex.InnerException}"); + return false; + } + } + + public async Task DeleteKnowledgeDocument(string collectionName, Guid fileId) { if (string.IsNullOrWhiteSpace(collectionName)) @@ -106,8 +152,8 @@ public partial class KnowledgeService // Get doc meta data var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { - FileIds = [ fileId ], - Size = 1 + Size = 1, + FileIds = [ fileId ] }); // Delete doc @@ -132,6 +178,57 @@ public partial class KnowledgeService } } + public async Task DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) + { + if (string.IsNullOrWhiteSpace(collectionName)) return false; + + + var pageSize = filter.Size; + var innerFilter = new KnowledgeFileFilter + { + Page = 1, + Size = pageSize, + FileIds = filter.FileIds, + FileNames = filter.FileNames, + FileSources = filter.FileSources, + ContentTypes = filter.ContentTypes + }; + + var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + + var total = pageData.Count; + if (total == 0) return false; + + var page = 1; + var totalPages = total % pageSize == 0 ? total / pageSize : total / pageSize + 1; + + while (page <= totalPages) + { + if (page > 1) + { + pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter); + } + + var fileIds = pageData.Items.Select(x => x.FileId).ToList(); + foreach (var fileId in fileIds) + { + try + { + await DeleteKnowledgeDocument(collectionName, fileId); + } + catch + { + continue; + } + } + + page++; + } + + return true; + } + + public async Task> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter) { if (string.IsNullOrWhiteSpace(collectionName)) @@ -144,16 +241,13 @@ public partial class KnowledgeService var vectorStoreProvider = _settings.VectorDb.Provider; // Get doc meta data - var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter - { - Page = filter.Page, - Size = filter.Size - }); + var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter); var files = pagedData.Items?.Select(x => new KnowledgeFileModel { FileId = x.FileId, FileName = x.FileName, + FileSource = x.FileSource, FileExtension = Path.GetExtension(x.FileName), ContentType = x.ContentType, FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName), @@ -176,8 +270,8 @@ public partial class KnowledgeService // Get doc binary data var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter { - FileIds = [ fileId ], - Size = 1 + Size = 1, + FileIds = [ fileId ] }); var metaData = pageData?.Items?.FirstOrDefault(); @@ -201,6 +295,7 @@ public partial class KnowledgeService } + #region Private methods /// /// Get file content type and file bytes @@ -259,7 +354,7 @@ public partial class KnowledgeService var lines = TextChopper.Chop(content, new ChunkOption { Size = 1024, - Conjunction = 32, + Conjunction = 12, SplitByWord = true, }); return lines; @@ -282,7 +377,7 @@ public partial class KnowledgeService private async Task> SaveToVectorDb( string collectionName, Guid fileId, string fileName, IEnumerable contents, - string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File) + string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File, string? fileUrl = null) { if (contents.IsNullOrEmpty()) { @@ -293,19 +388,25 @@ public partial class KnowledgeService var vectorDb = GetVectorDb(); var textEmbedding = GetTextEmbedding(collectionName); + var payload = new Dictionary + { + { KnowledgePayloadName.DataSource, vectorDataSource }, + { KnowledgePayloadName.FileId, fileId.ToString() }, + { KnowledgePayloadName.FileName, fileName }, + { KnowledgePayloadName.FileSource, fileSource } + }; + + if (!string.IsNullOrWhiteSpace(fileUrl)) + { + payload[KnowledgePayloadName.FileUrl] = fileUrl; + } + for (int i = 0; i < contents.Count(); i++) { var content = contents.ElementAt(i); var vector = await textEmbedding.GetVectorAsync(content); var dataId = Guid.NewGuid(); - var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary - { - { KnowledgePayloadName.DataSource, vectorDataSource }, - { KnowledgePayloadName.FileId, fileId.ToString() }, - { KnowledgePayloadName.FileName, fileName }, - { KnowledgePayloadName.FileSource, fileSource }, - { "textNumber", $"{i + 1}" } - }); + var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload); if (!saved) continue; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs index 059fbe3e..bc213dea 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs @@ -6,8 +6,9 @@ public class KnowledgeFileMetaRefMongoModel { public string Id { get; set; } public string Name { get; set; } + public string Type { get; set; } public string Url { get; set; } - public string? JsonContent { get; set; } + public IDictionary? Data { get; set; } public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model) { @@ -17,8 +18,9 @@ public class KnowledgeFileMetaRefMongoModel { Id = model.Id, Name = model.Name, + Type = model.Type, Url = model.Url, - JsonContent = model.JsonContent + Data = model.Data }; } @@ -30,8 +32,9 @@ public class KnowledgeFileMetaRefMongoModel { Id = model.Id, Name = model.Name, + Type = model.Type, Url = model.Url, - JsonContent = model.JsonContent + Data = model.Data }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs index 43d3b410..1ceffb44 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs @@ -184,9 +184,27 @@ public partial class MongoRepository }; // Apply filters - if (filter != null && !filter.FileIds.IsNullOrEmpty()) + if (filter != null) { - docFilters.Add(builder.In(x => x.FileId, filter.FileIds)); + if (!filter.FileIds.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.FileId, filter.FileIds)); + } + + if (!filter.FileNames.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.FileName, filter.FileNames)); + } + + if (!filter.FileSources.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.FileSource, filter.FileSources)); + } + + if (!filter.ContentTypes.IsNullOrEmpty()) + { + docFilters.Add(builder.In(x => x.ContentType, filter.ContentTypes)); + } } var filterDef = builder.And(docFilters);