diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs index 521507a0..1cb804a7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs @@ -1,8 +1,4 @@ -using AspectInjector.Broker; -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Knowledges.Models; -using StackExchange.Redis; using System.IO; namespace BotSharp.Core.Files.Services; @@ -29,8 +25,10 @@ public partial class LocalFileStorageService Directory.CreateDirectory(dir); var filePath = Path.Combine(dir, fileName); - using var fs = File.Create(filePath); + using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write); stream.CopyTo(fs); + fs.Flush(); + fs.Close(); return true; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs index 9e9b5f9d..8421e537 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs @@ -14,40 +14,48 @@ public static class TextChopper private static List ChopByWord(string content, ChunkOption option) { var chunks = new List(); + var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList(); - var words = content.Split(' ') - .Where(x => !string.IsNullOrWhiteSpace(x)) - .ToList(); - - var chunk = ""; + var chunk = string.Empty; for (int i = 0; i < words.Count; i++) { chunk += words[i] + " "; if (chunk.Length > option.Size) { chunks.Add(chunk.Trim()); - chunk = ""; + chunk = string.Empty; i -= option.Conjunction; } } + if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk)) + { + chunks.Add(chunk); + } + return chunks; } private static List ChopByChar(string content, ChunkOption option) { var chunks = new List(); + var chunk = string.Empty; var currentPos = 0; + while (currentPos < content.Length) { - var len = content.Length - currentPos > option.Size ? - option.Size : - content.Length - currentPos; - var chunk = content.Substring(currentPos, len); + var len = content.Length - currentPos > option.Size ? option.Size : content.Length - currentPos; + chunk = content.Substring(currentPos, len); chunks.Add(chunk); // move backward currentPos += option.Size - option.Conjunction; } + + if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk)) + { + chunks.Add(chunk); + } + return chunks; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index bc0fd5bf..27a2e590 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using System.Net.Http; +using System.Net.Mime; namespace BotSharp.Plugin.KnowledgeBase.Services; @@ -34,43 +35,23 @@ public partial class KnowledgeService try { - var dataIds = new List(); - - // Chop text (to do) + // Get document info var (contentType, bytes) = await GetFileInfo(file); - using var stream = new MemoryStream(bytes); - using var reader = new StreamReader(stream); - var content = await reader.ReadToEndAsync(); - - // Save file + var contents = await GetFileContent(contentType, bytes); + + // Save document var fileId = Guid.NewGuid().ToString(); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, file.FileName, stream); - reader.Close(); - stream.Close(); - + var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes); if (!saved) { failedFiles.Add(file.FileName); continue; } - // Text embedding - var vectorDb = GetVectorDb(); - var textEmbedding = GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(content); - // Save to vector db - var dataId = Guid.NewGuid(); - saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary + var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents); + if (!dataIds.IsNullOrEmpty()) { - { "fileName", file.FileName }, - { "fileId", fileId }, - { "page", "0" } - }); - - if (saved) - { - dataIds.Add(dataId.ToString()); fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData { Collection = collectionName, @@ -117,8 +98,8 @@ public partial class KnowledgeService var vectorDb = GetVectorDb(); var vectorStoreProvider = _settings.VectorDb.Provider; - fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); + fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId); if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty()) { @@ -212,5 +193,76 @@ public partial class KnowledgeService return (string.Empty, new byte[0]); } + + private async Task> GetFileContent(string contentType, byte[] bytes) + { + var results = new List(); + + if (contentType.IsEqualTo(MediaTypeNames.Text.Plain)) + { + using var stream = new MemoryStream(bytes); + using var reader = new StreamReader(stream); + var content = await reader.ReadToEndAsync(); + + var lines = TextChopper.Chop(content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + reader.Close(); + stream.Close(); + results.AddRange(lines); + } + else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf)) + { + // to do + } + + return results; + } + + private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes) + { + var fileStoreage = _services.GetRequiredService(); + using var stream = new MemoryStream(bytes); + stream.Position = 0; + + var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, stream); + stream.Close(); + return saved; + } + + private async Task> SaveToVectorDb(string collectionName, string fileId, string fileName, IEnumerable contents) + { + if (contents.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + var dataIds = new List(); + var vectorDb = GetVectorDb(); + var textEmbedding = GetTextEmbedding(collectionName); + + for (int i = 0; i < contents.Count(); i++) + { + var content = contents.ElementAt(i); + var vector = await textEmbedding.GetVectorAsync(content); + var dataId = Guid.NewGuid(); + var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary + { + { "fileName", fileName }, + { "fileId", fileId }, + { "textNumber", $"{i + 1}" } + }); + + if (!saved) continue; + + dataIds.Add(dataId.ToString()); + } + + return dataIds; + } #endregion }