Merge pull request #636 from iceljc/features/add-knowledge-docs

fix read knowledge doc

src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs

@@ -3,5 +3,11 @@ namespace BotSharp.Abstraction.Files.Models;
 public class ExternalFileModel : FileDataModel
 {
     [JsonPropertyName("file_url")]
-    public string FileUrl { get; set; } = string.Empty;
+    public string? FileUrl { get; set; }
+
+    /// <summary>
+    /// File data => format: "data:image/png;base64,aaaaaaaa"
+    /// </summary>
+    [JsonPropertyName("file_data")]
+    public new string? FileData { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBinaryDataModel.cs

@@ -3,5 +3,6 @@ namespace BotSharp.Abstraction.Files.Models;
 public class FileBinaryDataModel
 {
     public string FileName { get; set; }
+    public string ContentType { get; set; }
     public BinaryData FileBinaryData { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs

@@ -15,13 +15,13 @@ public class VectorCollectionConfig
     public string Name { get; set; }
 
     /// <summary>
-    /// Collection type, e.g., question-answer, document
+    /// Collection type, e.g., question-answer, document, etc.
     /// </summary>
     [JsonPropertyName("type")]
     public string Type { get; set; }
 
-    [JsonPropertyName("vector_storage")]
-    public VectorStorageConfig VectorStorage { get; set; }
+    [JsonPropertyName("vector_store")]
+    public VectorStoreConfig VectorStore { get; set; }
 
     [JsonPropertyName("text_embedding")]
     public KnowledgeEmbeddingConfig TextEmbedding { get; set; }
@@ -39,7 +39,7 @@ public class KnowledgeEmbeddingConfig
     public int Dimension { get; set; }
 }
 
-public class VectorStorageConfig
+public class VectorStoreConfig
 {
     [JsonPropertyName("provider")]
     public string Provider { get; set; }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs

@@ -32,7 +32,7 @@ public partial class LocalFileStorageService
     public BinaryData GetSpeechFile(string conversationId, string fileName)
     {
         var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER,  fileName);
-        using var file = new FileStream(path, FileMode.Open, FileAccess.Read);
-        return BinaryData.FromStream(file);
+        using var fs = new FileStream(path, FileMode.Open, FileAccess.Read);
+        return BinaryData.FromStream(fs);
     }
 }

src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs

@@ -127,24 +127,33 @@ public partial class LocalFileStorageService
         var files = new List<KnowledgeFileModel>();
         foreach (var folder in Directory.GetDirectories(docDir))
         {
-            var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE);
-            if (!File.Exists(metaFile)) continue;
-
-            var content = File.ReadAllText(metaFile);
-            var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
-            if (metaData == null) continue;
-
-            var fileName = Path.GetFileNameWithoutExtension(metaData.FileName);
-            var fileExtension = Path.GetExtension(metaData.FileName);
-
-            files.Add(new KnowledgeFileModel
+            try
             {
-                FileId = metaData.FileId,
-                FileName = metaData.FileName,
-                FileExtension = fileExtension.Substring(1),
-                ContentType = FileUtility.GetFileContentType(metaData.FileName),
-                FileUrl = BuildKnowledgeFileUrl(collectionName, metaData.FileId)
-            });
+                var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE);
+                if (!File.Exists(metaFile)) continue;
+
+                var content = File.ReadAllText(metaFile);
+                var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
+                if (metaData == null) continue;
+
+                var fileName = Path.GetFileNameWithoutExtension(metaData.FileName);
+                var fileExtension = Path.GetExtension(metaData.FileName);
+
+                files.Add(new KnowledgeFileModel
+                {
+                    FileId = metaData.FileId,
+                    FileName = metaData.FileName,
+                    FileExtension = fileExtension.Substring(1),
+                    ContentType = FileUtility.GetFileContentType(metaData.FileName),
+                    FileUrl = BuildKnowledgeFileUrl(collectionName, metaData.FileId)
+                });
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning($"Error when getting knowledgebase file. ({folder})" +
+                    $"\r\n{ex.Message}\r\n{ex.InnerException}");
+                continue;
+            }
         }
 
         return files;
@@ -166,12 +175,14 @@ public partial class LocalFileStorageService
         var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE);
         var content = File.ReadAllText(metaFile);
         var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
-        using var stream = new FileStream(fileDir, FileMode.Open, FileAccess.Read);
+        var file = Path.Combine(fileDir, metaData.FileName);
+        using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
         stream.Position = 0;
 
         return new FileBinaryDataModel
         {
             FileName = metaData.FileName,
+            ContentType = metaData.ContentType,
             FileBinaryData = BinaryData.FromStream(stream)
         };
     }
@@ -185,7 +196,7 @@ public partial class LocalFileStorageService
 
     private string BuildKnowledgeFileUrl(string collectionName, string fileId)
     {
-        return $"/knowledge/file/{collectionName}/file/{fileId}";
+        return $"/knowledge/document/{collectionName}/file/{fileId}";
     }
     #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -97,7 +97,7 @@ public partial class FileRepository
 
         if (!filter.VectorStroageProviders.IsNullOrEmpty())
         {
-            configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStorage?.Provider)).ToList();
+            configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStore?.Provider)).ToList();
         }
 
         return configs;

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -1,8 +1,11 @@
+using Azure.Core;
 using BotSharp.Abstraction.Files.Constants;
+using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Graph.Models;
 using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
+using System.IO;
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -129,6 +132,29 @@ public class KnowledgeBaseController : ControllerBase
         return response;
     }
 
+    [HttpPost("/knowledge/document/{collection}/form-upload")]
+    public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromForm] IEnumerable<IFormFile> files)
+    {
+        if (files.IsNullOrEmpty())
+        {
+            return new UploadKnowledgeResponse();
+        }
+
+        var docs = new List<ExternalFileModel>();
+        foreach (var file in files)
+        {
+            var data = FileUtility.BuildFileDataFromFile(file);
+            docs.Add(new ExternalFileModel
+            {
+                FileName = file.FileName,
+                FileData = data
+            });
+        }
+
+        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, docs);
+        return response;
+    }
+
     [HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
     public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
     {
@@ -147,7 +173,10 @@ public class KnowledgeBaseController : ControllerBase
     public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
     {
         var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId);
-        return BuildFileResult(file);
+        var stream = file.FileBinaryData.ToStream();
+        stream.Position = 0;
+
+        return new FileStreamResult(stream, file.ContentType) { FileDownloadName = file.FileName };
     }
     #endregion
 
@@ -160,17 +189,4 @@ public class KnowledgeBaseController : ControllerBase
         return saved ? "Success" : "Fail";
     }
     #endregion
-
-
-    #region Private methods
-    private FileContentResult BuildFileResult(FileBinaryDataModel? file)
-    {
-        if (file == null)
-        {
-            return File(new byte[0], "application/octet-stream", "error.txt");
-        }
-
-        return File(file.FileBinaryData.ToArray(), "application/octet-stream", file.FileName);
-    }
-    #endregion
 }

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -177,34 +177,47 @@ public partial class KnowledgeService
         return (string.Empty, new byte[0]);
     }
 
+    #region Read doc content
     private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes)
     {
-        var results = new List<string>();
+        IEnumerable<string> results = new List<string>();
 
         if (contentType.IsEqualTo(MediaTypeNames.Text.Plain))
         {
-            using var stream = new MemoryStream(bytes);
-            using var reader = new StreamReader(stream);
-            var content = await reader.ReadToEndAsync();
-            reader.Close();
-            stream.Close();
-
-            var lines = TextChopper.Chop(content, new ChunkOption
-            {
-                Size = 1024,
-                Conjunction = 32,
-                SplitByWord = true,
-            });
-            results.AddRange(lines);
+            results = await ReadTxt(bytes);
         }
         else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))
         {
-            // to do
+            results = await ReadPdf(bytes);
         }
         
         return results;
     }
 
+    private async Task<IEnumerable<string>> ReadTxt(byte[] bytes)
+    {
+        using var stream = new MemoryStream(bytes);
+        using var reader = new StreamReader(stream);
+        var content = await reader.ReadToEndAsync();
+        reader.Close();
+        stream.Close();
+
+        var lines = TextChopper.Chop(content, new ChunkOption
+        {
+            Size = 1024,
+            Conjunction = 32,
+            SplitByWord = true,
+        });
+        return lines;
+    }
+
+    private async Task<IEnumerable<string>> ReadPdf(byte[] bytes)
+    {
+        return Enumerable.Empty<string>();
+    }
+    #endregion
+
+
     private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
     {
         var fileStoreage = _services.GetRequiredService<IFileStorageService>();

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -27,7 +27,7 @@ public partial class KnowledgeService
                     {
                         Name = collectionName,
                         Type = collectionType,
-                        VectorStorage = new VectorStorageConfig
+                        VectorStore = new VectorStoreConfig
                         {
                             Provider = _settings.VectorDb.Provider
                         },

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs

@@ -4,6 +4,6 @@ public class KnowledgeCollectionConfigDocument : MongoBase
 {
     public string Name { get; set; }
     public string Type { get; set; }
-    public KnowledgeVectorStorageConfigMongoModel VectorStorage { get; set; }
+    public KnowledgeVectorStoreConfigMongoModel VectorStore { get; set; }
     public KnowledgeEmbeddingConfigMongoModel TextEmbedding { get; set; }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs

@@ -1,24 +0,0 @@
-using BotSharp.Abstraction.VectorStorage.Models;
-
-namespace BotSharp.Plugin.MongoStorage.Models;
-
-public class KnowledgeVectorStorageConfigMongoModel
-{
-    public string Provider { get; set; }
-
-    public static KnowledgeVectorStorageConfigMongoModel ToMongoModel(VectorStorageConfig model)
-    {
-        return new KnowledgeVectorStorageConfigMongoModel
-        {
-            Provider = model.Provider
-        };
-    }
-
-    public static VectorStorageConfig ToDomainModel(KnowledgeVectorStorageConfigMongoModel model)
-    {
-        return new VectorStorageConfig
-        {
-            Provider = model.Provider
-        };
-    }
-}

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs

@@ -0,0 +1,24 @@
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class KnowledgeVectorStoreConfigMongoModel
+{
+    public string Provider { get; set; }
+
+    public static KnowledgeVectorStoreConfigMongoModel ToMongoModel(VectorStoreConfig model)
+    {
+        return new KnowledgeVectorStoreConfigMongoModel
+        {
+            Provider = model.Provider
+        };
+    }
+
+    public static VectorStoreConfig ToDomainModel(KnowledgeVectorStoreConfigMongoModel model)
+    {
+        return new VectorStoreConfig
+        {
+            Provider = model.Provider
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -13,7 +13,7 @@ public partial class MongoRepository
                 Id = Guid.NewGuid().ToString(),
                 Name = x.Name,
                 Type = x.Type,
-                VectorStorage = KnowledgeVectorStorageConfigMongoModel.ToMongoModel(x.VectorStorage),
+                VectorStore = KnowledgeVectorStoreConfigMongoModel.ToMongoModel(x.VectorStore),
                 TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding)
             })?.ToList() ?? new List<KnowledgeCollectionConfigDocument>();
 
@@ -38,7 +38,7 @@ public partial class MongoRepository
             if (found != null)
             {
                 found.Type = doc.Type;
-                found.VectorStorage = doc.VectorStorage;
+                found.VectorStore = doc.VectorStore;
                 found.TextEmbedding = doc.TextEmbedding;
                 updateDocs.Add(found);
             }
@@ -97,7 +97,7 @@ public partial class MongoRepository
 
         if (!filter.VectorStroageProviders.IsNullOrEmpty())
         {
-            filters.Add(builder.In(x => x.VectorStorage.Provider, filter.VectorStroageProviders));
+            filters.Add(builder.In(x => x.VectorStore.Provider, filter.VectorStroageProviders));
         }
 
         // Get data
@@ -107,7 +107,7 @@ public partial class MongoRepository
         {
             Name = x.Name,
             Type = x.Type,
-            VectorStorage = KnowledgeVectorStorageConfigMongoModel.ToDomainModel(x.VectorStorage),
+            VectorStore = KnowledgeVectorStoreConfigMongoModel.ToDomainModel(x.VectorStore),
             TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding)
         });
     }

src/WebStarter/appsettings.json

@@ -281,17 +281,7 @@
         "Model": "text-embedding-3-small",
         "Dimension": 1536
       }
-    },
-    "Collections": [
-      {
-        "Name": "BotSharp",
-        "TextEmbedding": {
-          "Provider": "openai",
-          "Model": "text-embedding-3-small",
-          "Dimension": 1536
-        }
-      }
-    ]
+    }
   },
 
   "SparkDesk": {