Merge branch 'SciSharp:master' into master

src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs

@@ -5,6 +5,7 @@ public class KnowledgeFileModel
     public Guid FileId { get; set; }
     public string FileName { get; set; }
     public string FileExtension { get; set; }
+    public string FileSource { get; set; }
     public string ContentType { get; set; }
     public string FileUrl { get; set; }
     public DocMetaRefData? RefData { get; set; }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs

@@ -11,4 +11,5 @@ public static class KnowledgePayloadName
     public static string FileId = "fileId";
     public static string FileName = "fileName";
     public static string FileSource = "fileSource";
+    public static string FileUrl = "fileUrl";
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Helpers/TextChopper.cs

@@ -1,6 +1,6 @@
 using System.Text.RegularExpressions;
 
-namespace BotSharp.Plugin.KnowledgeBase.Helpers;
+namespace BotSharp.Abstraction.Knowledges.Helpers;
 
 public static class TextChopper
 {
@@ -14,18 +14,22 @@ public static class TextChopper
     private static List<string> ChopByWord(string content, ChunkOption option)
     {
         var chunks = new List<string>();
-        var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
+        var words = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
 
         var chunk = string.Empty;
         for (int i = 0; i < words.Count; i++)
         {
-            chunk += words[i] + " ";
+            chunk += words[i];
             if (chunk.Length > option.Size)
             {
                 chunks.Add(chunk.Trim());
                 chunk = string.Empty;
                 i -= option.Conjunction;
             }
+            else
+            {
+                chunk += " ";
+            }
         }
 
         if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -22,8 +22,37 @@ public interface IKnowledgeService
     #endregion
 
     #region Document
-    Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
+    /// <summary>
+    /// Save documents and their contents to knowledgebase
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="files"></param>
+    /// <returns></returns>
+    Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files);
+    /// <summary>
+    /// Save document content to knowledgebase without saving the document
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileName"></param>
+    /// <param name="fileSource"></param>
+    /// <param name="contents"></param>
+    /// <param name="refData"></param>
+    /// <returns></returns>
+    Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource, IEnumerable<string> contents, DocMetaRefData? refData = null);
+    /// <summary>
+    /// Delete one document and its related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="fileId"></param>
+    /// <returns></returns>
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
+    /// <summary>
+    /// Delete all documents and their related knowledge in the collection
+    /// </summary>
+    /// <param name="collectionName"></param>
+    /// <param name="filter"></param>
+    /// <returns></returns>
+    Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
     Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs

@@ -42,10 +42,13 @@ public class DocMetaRefData
     [JsonPropertyName("name")]
     public string Name { get; set; }
 
+    [JsonPropertyName("type")]
+    public string Type { get; set; }
+
     [JsonPropertyName("url")]
     public string Url { get; set; }
 
-    [JsonPropertyName("json_content")]
+    [JsonPropertyName("data")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? JsonContent { get; set; }
+    public IDictionary<string, string>? Data { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs

@@ -3,4 +3,15 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 public class KnowledgeFileFilter : Pagination
 {
     public IEnumerable<Guid>? FileIds { get; set; }
+
+    public IEnumerable<string>? FileNames { get; set; }
+
+    public IEnumerable<string>? ContentTypes { get; set; }
+
+    public IEnumerable<string>? FileSources { get; set; }
+
+    public KnowledgeFileFilter()
+    {
+        
+    }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -182,10 +182,29 @@ public partial class FileRepository
             var matched = true;
 
             // Apply filter
-            if (filter != null && !filter.FileIds.IsNullOrEmpty())
+            if (filter != null)
             {
-                matched = matched && filter.FileIds.Contains(metaData.FileId);
+                if (!filter.FileIds.IsNullOrEmpty())
+                {
+                    matched = matched && filter.FileIds.Contains(metaData.FileId);
+                }
+
+                if (!filter.FileNames.IsNullOrEmpty())
+                {
+                    matched = matched && filter.FileNames.Contains(metaData.FileName);
+                }
+
+                if (!filter.FileSources.IsNullOrEmpty())
+                {
+                    matched = matched & filter.FileSources.Contains(metaData.FileSource);
+                }
+
+                if (!filter.ContentTypes.IsNullOrEmpty())
+                {
+                    matched = matched && filter.ContentTypes.Contains(metaData.ContentType);
+                }
             }
+            
 
             if (!matched) continue;
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -115,7 +115,7 @@ public class KnowledgeBaseController : ControllerBase
     [HttpPost("/knowledge/document/{collection}/upload")]
     public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
     {
-        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files);
+        var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, request.Files);
         return response;
     }
 
@@ -138,7 +138,7 @@ public class KnowledgeBaseController : ControllerBase
             });
         }
 
-        var response = await _knowledgeService.UploadKnowledgeDocuments(collection, docs);
+        var response = await _knowledgeService.UploadDocumentsToKnowledge(collection, docs);
         return response;
     }
 
@@ -149,14 +149,17 @@ public class KnowledgeBaseController : ControllerBase
         return response;
     }
 
-    [HttpPost("/knowledge/document/{collection}/list")]
+    [HttpDelete("/knowledge/document/{collection}/delete")]
+    public async Task<bool> DeleteKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
+    {
+        var response = await _knowledgeService.DeleteKnowledgeDocuments(collection, request);
+        return response;
+    }
+
+    [HttpPost("/knowledge/document/{collection}/page")]
     public async Task<PagedItems<KnowledgeFileViewModel>> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
     {
-        var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter
-        {
-            Page = request.Page,
-            Size = request.Size
-        });
+        var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, request);
 
         return new PagedItems<KnowledgeFileViewModel>
         {

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs

@@ -10,6 +10,9 @@ public class KnowledgeFileViewModel
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }
 
+    [JsonPropertyName("file_source")]
+    public string FileSource { get; set; }
+
     [JsonPropertyName("file_extension")]
     public string FileExtension { get; set; }
 
@@ -29,6 +32,7 @@ public class KnowledgeFileViewModel
         {
             FileId = model.FileId,
             FileName = model.FileName,
+            FileSource = model.FileSource,
             FileExtension = model.FileExtension,
             ContentType = model.ContentType,
             FileUrl = model.FileUrl,

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -1,7 +1,9 @@
 using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Files.Models;
 using BotSharp.Abstraction.Files.Utilities;
+using BotSharp.Abstraction.Knowledges.Helpers;
 using BotSharp.Abstraction.VectorStorage.Enums;
+using System.Collections;
 using System.Net.Http;
 using System.Net.Mime;
 
@@ -9,7 +11,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 
 public partial class KnowledgeService
 {
-    public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files)
+    public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
     {
         if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
         {
@@ -89,6 +91,50 @@ public partial class KnowledgeService
     }
 
 
+    public async Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileName, string fileSource,
+        IEnumerable<string> contents, DocMetaRefData? refData = null)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)
+            || string.IsNullOrWhiteSpace(fileName)
+            || contents.IsNullOrEmpty())
+        {
+            return false;
+        }
+
+        try
+        {
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            var userId = await GetUserId();
+            var vectorStoreProvider = _settings.VectorDb.Provider;
+            var fileId = Guid.NewGuid();
+            var contentType = FileUtility.GetFileContentType(fileName);
+
+            var dataIds = await SaveToVectorDb(collectionName, fileId, fileName, contents, fileSource, fileUrl: refData?.Url);
+            db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData
+            {
+                Collection = collectionName,
+                FileId = fileId,
+                FileName = fileName,
+                FileSource = fileSource,
+                ContentType = contentType,
+                VectorStoreProvider = vectorStoreProvider,
+                VectorDataIds = dataIds,
+                RefData = refData,
+                CreateDate = DateTime.UtcNow,
+                CreateUserId = userId
+            });
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when importing doc content to knowledgebase ({collectionName}-{fileName})" +
+                $"\r\n{ex.Message}" +
+                $"\r\n{ex.InnerException}");
+            return false;
+        }
+    }
+
+
     public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId)
     {
         if (string.IsNullOrWhiteSpace(collectionName))
@@ -106,8 +152,8 @@ public partial class KnowledgeService
             // Get doc meta data
             var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
             {
-                FileIds = [ fileId ],
-                Size = 1
+                Size = 1,
+                FileIds = [ fileId ]
             });
 
             // Delete doc
@@ -132,6 +178,57 @@ public partial class KnowledgeService
         }
     }
 
+    public async Task<bool> DeleteKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter)
+    {
+        if (string.IsNullOrWhiteSpace(collectionName)) return false;
+
+        
+        var pageSize = filter.Size;
+        var innerFilter = new KnowledgeFileFilter
+        {
+            Page = 1,
+            Size = pageSize,
+            FileIds = filter.FileIds,
+            FileNames = filter.FileNames,
+            FileSources = filter.FileSources,
+            ContentTypes = filter.ContentTypes
+        };
+
+        var pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter);
+
+        var total = pageData.Count;
+        if (total == 0) return false;
+
+        var page = 1;
+        var totalPages = total % pageSize == 0 ? total / pageSize : total / pageSize + 1;
+
+        while (page <= totalPages)
+        {
+            if (page > 1)
+            {
+                pageData = await GetPagedKnowledgeDocuments(collectionName, innerFilter);
+            }
+
+            var fileIds = pageData.Items.Select(x => x.FileId).ToList();
+            foreach (var fileId in fileIds)
+            {
+                try
+                {
+                    await DeleteKnowledgeDocument(collectionName, fileId);
+                }
+                catch
+                {
+                    continue;
+                }
+            }
+
+            page++;
+        }
+
+        return true;
+    }
+
+
     public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter)
     {
         if (string.IsNullOrWhiteSpace(collectionName))
@@ -144,16 +241,13 @@ public partial class KnowledgeService
         var vectorStoreProvider = _settings.VectorDb.Provider;
 
         // Get doc meta data
-        var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
-        {
-            Page = filter.Page,
-            Size = filter.Size
-        });
+        var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, filter);
 
         var files = pagedData.Items?.Select(x => new KnowledgeFileModel
         {
             FileId = x.FileId,
             FileName = x.FileName,
+            FileSource = x.FileSource,
             FileExtension = Path.GetExtension(x.FileName),
             ContentType = x.ContentType,
             FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
@@ -176,8 +270,8 @@ public partial class KnowledgeService
         // Get doc binary data
         var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
         {
-            FileIds = [ fileId ],
-            Size = 1
+            Size = 1,
+            FileIds = [ fileId ]
         });
 
         var metaData = pageData?.Items?.FirstOrDefault();
@@ -201,6 +295,7 @@ public partial class KnowledgeService
     }
 
 
+
     #region Private methods
     /// <summary>
     /// Get file content type and file bytes
@@ -259,7 +354,7 @@ public partial class KnowledgeService
         var lines = TextChopper.Chop(content, new ChunkOption
         {
             Size = 1024,
-            Conjunction = 32,
+            Conjunction = 12,
             SplitByWord = true,
         });
         return lines;
@@ -282,7 +377,7 @@ public partial class KnowledgeService
 
     private async Task<IEnumerable<string>> SaveToVectorDb(
         string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
-        string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
+        string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File, string? fileUrl = null)
     {
         if (contents.IsNullOrEmpty())
         {
@@ -293,19 +388,25 @@ public partial class KnowledgeService
         var vectorDb = GetVectorDb();
         var textEmbedding = GetTextEmbedding(collectionName);
 
+        var payload = new Dictionary<string, string>
+        {
+            { KnowledgePayloadName.DataSource, vectorDataSource },
+            { KnowledgePayloadName.FileId, fileId.ToString() },
+            { KnowledgePayloadName.FileName, fileName },
+            { KnowledgePayloadName.FileSource, fileSource }
+        };
+
+        if (!string.IsNullOrWhiteSpace(fileUrl))
+        {
+            payload[KnowledgePayloadName.FileUrl] = fileUrl;
+        }
+
         for (int i = 0; i < contents.Count(); i++)
         {
             var content = contents.ElementAt(i);
             var vector = await textEmbedding.GetVectorAsync(content);
             var dataId = Guid.NewGuid();
-            var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
-            {
-                { KnowledgePayloadName.DataSource, vectorDataSource },
-                { KnowledgePayloadName.FileId, fileId.ToString() },
-                { KnowledgePayloadName.FileName, fileName },
-                { KnowledgePayloadName.FileSource, fileSource },
-                { "textNumber", $"{i + 1}" }
-            });
+            var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload);
 
             if (!saved) continue;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs

@@ -6,8 +6,9 @@ public class KnowledgeFileMetaRefMongoModel
 {
     public string Id { get; set; }
     public string Name { get; set; }
+    public string Type { get; set; }
     public string Url { get; set; }
-    public string? JsonContent { get; set; }
+    public IDictionary<string, string>? Data { get; set; }
 
     public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model)
     {
@@ -17,8 +18,9 @@ public class KnowledgeFileMetaRefMongoModel
         {
             Id = model.Id,
             Name = model.Name,
+            Type = model.Type,
             Url = model.Url,
-            JsonContent = model.JsonContent
+            Data = model.Data
         };
     }
 
@@ -30,8 +32,9 @@ public class KnowledgeFileMetaRefMongoModel
         {
             Id = model.Id,
             Name = model.Name,
+            Type = model.Type,
             Url = model.Url,
-            JsonContent = model.JsonContent
+            Data = model.Data
         };
     }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs

@@ -184,9 +184,27 @@ public partial class MongoRepository
         };
         
         // Apply filters
-        if (filter != null && !filter.FileIds.IsNullOrEmpty())
+        if (filter != null)
         {
-            docFilters.Add(builder.In(x => x.FileId, filter.FileIds));
+            if (!filter.FileIds.IsNullOrEmpty())
+            {
+                docFilters.Add(builder.In(x => x.FileId, filter.FileIds));
+            }
+
+            if (!filter.FileNames.IsNullOrEmpty())
+            {
+                docFilters.Add(builder.In(x => x.FileName, filter.FileNames));
+            }
+
+            if (!filter.FileSources.IsNullOrEmpty())
+            {
+                docFilters.Add(builder.In(x => x.FileSource, filter.FileSources));
+            }
+
+            if (!filter.ContentTypes.IsNullOrEmpty())
+            {
+                docFilters.Add(builder.In(x => x.ContentType, filter.ContentTypes));
+            }
         }
 
         var filterDef = builder.And(docFilters);