From 4232d162438257578dd2c169fd0914f14e991115 Mon Sep 17 00:00:00 2001
From: Jicheng Lu <103353@smsassist.com>
Date: Wed, 11 Sep 2024 16:12:48 -0500
Subject: [PATCH] add comments

---
 .../Helpers/KnowledgeSettingHelper.cs         |  3 ++
 .../Services/KnowledgeService.Document.cs     | 38 +++++--------------
 2 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs
index 08ad2c49..48742f0a 100644
--- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs
+++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs
@@ -6,6 +6,8 @@ public static class KnowledgeSettingHelper
     {
         var settings = services.GetRequiredService<KnowledgeBaseSettings>();
         var db = services.GetRequiredService<IBotSharpRepository>();
+
+        // Get collection config from db
         var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
         {
             CollectionNames = [collectionName],
@@ -24,6 +26,7 @@ public static class KnowledgeSettingHelper
             dimension = settings.Default.TextEmbedding.Dimension;
         }
 
+        // Set up text embedding
         var embedding = services.GetServices<ITextEmbedding>().FirstOrDefault(x => x.Provider == provider);
 
         if (dimension <= 0)
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
index 27a2e590..2e614a68 100644
--- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
+++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
@@ -10,12 +10,12 @@ public partial class KnowledgeService
 {
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files)
     {
-        if (string.IsNullOrWhiteSpace(collectionName))
+        if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
         {
             return new UploadKnowledgeResponse
             {
                 Success = [],
-                Failed = files.Select(x => x.FileName)
+                Failed = files?.Select(x => x.FileName) ?? new List<string>()
             };
         }
 
@@ -98,7 +98,9 @@ public partial class KnowledgeService
             var vectorDb = GetVectorDb();
             var vectorStoreProvider = _settings.VectorDb.Provider;
 
+            // Get doc meta data
             var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
+            // Delete doc
             fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
 
             if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty())
@@ -128,6 +130,8 @@ public partial class KnowledgeService
 
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var vectorStoreProvider = _settings.VectorDb.Provider;
+
+        // Get doc meta data
         var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
         return files;
     }
@@ -136,34 +140,13 @@ public partial class KnowledgeService
     {
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var vectorStoreProvider = _settings.VectorDb.Provider;
+
+        // Get doc binary data
         var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
         return file;
     }
 
 
-    public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
-    {
-        var index = 0;
-        var lines = TextChopper.Chop(knowledge.Content, new ChunkOption
-        {
-            Size = 1024,
-            Conjunction = 32,
-            SplitByWord = true,
-        });
-
-        var db = GetVectorDb();
-        var textEmbedding = GetTextEmbedding(collectionName);
-
-        await db.CreateCollection(collectionName, textEmbedding.GetDimension());
-        foreach (var line in lines)
-        {
-            var vec = await textEmbedding.GetVectorAsync(line);
-            await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
-            index++;
-            Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
-        }
-    }
-
     #region Private methods
     /// <summary>
     /// Get file content type and file bytes
@@ -203,6 +186,8 @@ public partial class KnowledgeService
             using var stream = new MemoryStream(bytes);
             using var reader = new StreamReader(stream);
             var content = await reader.ReadToEndAsync();
+            reader.Close();
+            stream.Close();
 
             var lines = TextChopper.Chop(content, new ChunkOption
             {
@@ -210,9 +195,6 @@ public partial class KnowledgeService
                 Conjunction = 32,
                 SplitByWord = true,
             });
-
-            reader.Close();
-            stream.Close();
             results.AddRange(lines);
         }
         else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))