From 16c50075ee1c45b28858992b0ce858a3e6c742af Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 9 Sep 2024 17:42:09 -0500 Subject: [PATCH] add upload vector knowledge files --- .../Files/IFileStorageService.cs | 4 + .../Knowledges/IKnowledgeService.cs | 1 + .../Models/UploadKnowledgeResponse.cs | 10 +++ .../Utilities/StringExtensions.cs | 7 ++ .../LocalFileStorageService.Knowledge.cs | 34 ++++++++ .../Storage/LocalFileStorageService.cs | 2 + .../FileRepository/FileRepository.cs | 2 +- .../Controllers/ConversationController.cs | 3 +- .../Controllers/KnowledgeBaseController.cs | 51 ++++++----- .../VectorKnowledgeUploadRequest.cs | 6 ++ .../Provider/NativeWhisperProvider.cs | 2 +- .../Services/KnowledgeService.Document.cs | 85 +++++++++++++++++++ src/WebStarter/appsettings.json | 3 +- 13 files changed, 182 insertions(+), 28 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 373466c4..d16d5012 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -61,4 +61,8 @@ public interface IFileStorageService bool SaveSpeechFile(string conversationId, string fileName, BinaryData data); BinaryData GetSpeechFile(string conversationId, string fileName); #endregion + + #region Knowledge + bool SaveKnowledgeFiles(string collectionName, string fileId, string fileName, Stream stream); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 64f6fdc0..3e4aecd8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -22,6 +22,7 @@ public interface IKnowledgeService #endregion #region Document + Task UploadVectorKnowledge(string collectionName, IEnumerable files); #endregion #region Common diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs new file mode 100644 index 00000000..3f5df77a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class UploadKnowledgeResponse +{ + [JsonPropertyName("success")] + public IEnumerable Success { get; set; } = new List(); + + [JsonPropertyName("failed")] + public IEnumerable Failed { get; set; } = new List(); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index 625a1f4b..e5009698 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -44,6 +44,13 @@ public static class StringExtensions return str1.Equals(str2, option); } + public static string RemoveWhiteSpaces(this string? str) + { + if (string.IsNullOrWhiteSpace(str)) return string.Empty; + + return str.Replace(" ", "").Replace("\t", "").Replace("\n", "").Replace("\r", ""); + } + public static string JsonContent(this string text) { var m = Regex.Match(text, @"\{(?:[^{}]|(?\{)|(?<-open>\}))+(?(open)(?!))\}"); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs new file mode 100644 index 00000000..f417654e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs @@ -0,0 +1,34 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService +{ + public bool SaveKnowledgeFiles(string collectionName, string fileId, string fileName, Stream stream) + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId)) + { + return false; + } + + try + { + var dir = Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, collectionName, fileId); + if (ExistDirectory(dir)) + { + Directory.Delete(dir); + } + Directory.CreateDirectory(dir); + + var filePath = Path.Combine(dir, fileName); + using var fs = File.Create(filePath); + stream.CopyTo(fs); + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving knowledge file (Collection: {collectionName}, File name: {fileName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 750803c4..17738109 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -19,6 +19,8 @@ public partial class LocalFileStorageService : IFileStorageService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; private const string TEXT_TO_SPEECH_FOLDER = "speeches"; + private const string KNOWLEDGE_FOLDER = "knowledgebase"; + private const string KNOWLEDGE_DOC_FOLDER = "document"; public LocalFileStorageService( BotSharpDatabaseSettings dbSettings, diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 9e220d60..713f1d8b 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -40,7 +40,7 @@ public partial class FileRepository : IBotSharpRepository private const string AGENT_RESPONSES_FOLDER = "responses"; private const string AGENT_TASKS_FOLDER = "tasks"; private const string USERS_FOLDER = "users"; - private const string KNOWLEDGE_FOLDER = "knowledge"; + private const string KNOWLEDGE_FOLDER = "knowledgebase"; private const string VECTOR_FOLDER = "vector"; private const string COLLECTION_CONFIG_FILE = "collection-config.json"; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 565677f7..b4aa32fb 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -366,8 +366,7 @@ public class ConversationController : ControllerBase #region Files and attachments [HttpPost("/conversation/{conversationId}/attachments")] - public IActionResult UploadAttachments([FromRoute] string conversationId, - IFormFile[] files) + public IActionResult UploadAttachments([FromRoute] string conversationId, IFormFile[] files) { if (files != null && files.Length > 0) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 51ddfa0b..b5da8213 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -99,29 +99,6 @@ public class KnowledgeBaseController : ControllerBase { return await _knowledgeService.DeleteVectorCollectionData(collection, id); } - - [HttpPost("/knowledge/vector/{collection}/upload")] - public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); - - var filePath = Path.GetTempFileName(); - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - { - await file.CopyToAsync(stream); - await stream.FlushAsync(); - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel - { - Content = content - }); - - System.IO.File.Delete(filePath); - return Ok(new { count = 1, file.Length }); - } #endregion @@ -144,7 +121,35 @@ public class KnowledgeBaseController : ControllerBase #region Document + //[HttpPost("/knowledge/vector/{collection}/upload")] + //public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) + //{ + // var setttings = _services.GetRequiredService(); + // var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); + // var filePath = Path.GetTempFileName(); + // using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) + // { + // await file.CopyToAsync(stream); + // await stream.FlushAsync(); + // } + + // var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); + // await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel + // { + // Content = content + // }); + + // System.IO.File.Delete(filePath); + // return Ok(new { count = 1, file.Length }); + //} + + [HttpPost("/knowledge/vector/{collection}/upload")] + public async Task UploadVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request) + { + var response = await _knowledgeService.UploadVectorKnowledge(collection, request.Files); + return response; + } #endregion #region Common diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs new file mode 100644 index 00000000..c49bdec1 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs @@ -0,0 +1,6 @@ +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorKnowledgeUploadRequest +{ + public IEnumerable Files { get; set; } = new List(); +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index e6ac230e..1947ebc4 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -14,7 +14,7 @@ public class NativeWhisperProvider : IAudioCompletion private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; - public string Provider => "native"; + public string Provider => "native-whisper"; public NativeWhisperProvider( BotSharpDatabaseSettings dbSettings, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index 0e4710bc..ddbca650 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -1,7 +1,89 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files.Utilities; + namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { + public async Task UploadVectorKnowledge(string collectionName, IEnumerable files) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return new UploadKnowledgeResponse + { + Success = [], + Failed = files.Select(x => x.FileName) + }; + } + + var fileStoreage = _services.GetRequiredService(); + var cleanCollectionName = collectionName.RemoveWhiteSpaces(); + var successFiles = new List(); + var failedFiles = new List(); + + foreach (var file in files) + { + if (string.IsNullOrWhiteSpace(file.FileData) || string.IsNullOrWhiteSpace(file.FileName)) + { + continue; + } + + var dataIds = new List(); + + try + { + // Chop text + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + using var stream = new MemoryStream(bytes); + using var reader = new StreamReader(stream); + var content = await reader.ReadToEndAsync(); + + // Save file + var fileId = Guid.NewGuid().ToString(); + var saved = fileStoreage.SaveKnowledgeFiles(cleanCollectionName, fileId, file.FileName, stream); + reader.Close(); + stream.Close(); + + if (!saved) + { + failedFiles.Add(file.FileName); + continue; + } + + // Text embedding + var vectorDb = GetVectorDb(); + var textEmbedding = GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(content); + + // Save to vector db + var dataId = Guid.NewGuid(); + await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary + { + { "fileName", file.FileName }, + { "fileId", fileId }, + { "page", "0" } + }); + + dataIds.Add(dataId.ToString()); + successFiles.Add(file.FileName); + } + catch (Exception ex) + { + _logger.LogError($"Error when processing knowledge file ({file.FileName}). {ex.Message}\r\n{ex.InnerException}"); + failedFiles.Add(file.FileName); + continue; + } + } + + return new UploadKnowledgeResponse + { + Success = successFiles, + Failed = failedFiles + }; + } + + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; @@ -24,4 +106,7 @@ public partial class KnowledgeService Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); } } + + #region Private methods + #endregion } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 4fcc745c..a755c118 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -141,7 +141,8 @@ "Enable": true, "BatchSize": 50, "MessageLimit": 2, - "BufferHours": 12 + "BufferHours": 12, + "ExcludeAgentIds": [] }, "RateLimit": { "MaxConversationPerDay": 100,