SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine knowledge doc

Browse source 
1. move doc meta to db.
2. add doc source and vector data source
This commit is contained in:
Jicheng Lu 2024-09-13 17:23:05 -05:00
parent b5f7062ce6
commit 113340e073
24 changed files with 293 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -72,13 +72,7 @@ public interface IFileStorageService
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData);
KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId);
IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider);
string GetKnowledgeBaseFileUrl(string collectionName, string fileId);
FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId);
#endregion
}

8
src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Knowledges.Enums;
namespace BotSharp.Abstraction.Files.Models;
public class ExternalFileModel : FileDataModel
@ -10,4 +12,10 @@ public class ExternalFileModel : FileDataModel
/// </summary>
[JsonPropertyName("file_data")]
public new string? FileData { get; set; }
/// <summary>
/// The file source, e.g., api, user upload, external web, etc.
/// </summary>
[JsonPropertyName("file_source")]
public string FileSource { get; set; } = KnowledgeDocSource.Api;
}

3
src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs
View file

@ -8,4 +8,7 @@ public static class KnowledgePayloadName
public static string Request = "request";
public static string Response = "response";
public static string DataSource = "dataSource";
public static string FileId = "fileId";
public static string FileName = "fileName";
public static string FileSource = "fileSource";
}

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -23,7 +23,7 @@ public interface IKnowledgeService
#region Document
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName);
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId);
#endregion

13
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeDocMetaData
@ -13,12 +11,23 @@ public class KnowledgeDocMetaData
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_source")]
public string FileSource { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
[JsonPropertyName("vector_store_provider")]
public string VectorStoreProvider { get; set; }
[JsonPropertyName("vector_data_ids")]
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
[JsonPropertyName("web_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? WebUrl { get; set; }
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;

6
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs Normal file
View file

@ -0,0 +1,6 @@
namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeFileFilter : Pagination
{
public IEnumerable<string>? FileIds { get; set; }
}

3
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -111,5 +111,8 @@ public interface IBotSharpRepository
bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
bool DeleteKnowledgeCollectionConfig(string collectionName);
IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
#endregion
}

94
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
View file

@ -67,96 +67,15 @@ public partial class LocalFileStorageService
return true;
}
public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData)
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(fileId))
{
return false;
return string.Empty;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId);
if (!ExistDirectory(dir))
{
Directory.CreateDirectory(dir);
}
var metaFile = Path.Combine(dir, KNOWLEDGE_DOC_META_FILE);
var content = JsonSerializer.Serialize(metaData, _jsonOptions);
File.WriteAllText(metaFile, content);
return true;
}
public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
{
return null;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var metaFile = Path.Combine(docDir, fileId, KNOWLEDGE_DOC_META_FILE);
if (!File.Exists(metaFile))
{
return null;
}
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
return metaData;
}
public IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return Enumerable.Empty<KnowledgeFileModel>();
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(docDir))
{
return Enumerable.Empty<KnowledgeFileModel>();
}
var files = new List<KnowledgeFileModel>();
foreach (var folder in Directory.GetDirectories(docDir))
{
try
{
var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE);
if (!File.Exists(metaFile)) continue;
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
if (metaData == null) continue;
var fileName = Path.GetFileNameWithoutExtension(metaData.FileName);
var fileExtension = Path.GetExtension(metaData.FileName);
files.Add(new KnowledgeFileModel
{
FileId = metaData.FileId,
FileName = metaData.FileName,
FileExtension = fileExtension.Substring(1),
ContentType = FileUtility.GetFileContentType(metaData.FileName),
FileUrl = BuildKnowledgeFileUrl(collectionName, metaData.FileId)
});
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting knowledgebase file. ({folder})" +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
continue;
}
}
return files;
return $"/knowledge/document/{collectionName}/file/{fileId}";
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
@ -193,10 +112,5 @@ public partial class LocalFileStorageService
{
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
}
private string BuildKnowledgeFileUrl(string collectionName, string fileId)
{
return $"/knowledge/document/{collectionName}/file/{fileId}";
}
#endregion
}

6
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -243,5 +243,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository
public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter) =>
throw new NotImplementedException();
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) =>
throw new NotImplementedException();
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) =>
throw new NotImplementedException();
#endregion
}

1
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
View file

@ -347,6 +347,7 @@ namespace BotSharp.Core.Repository
}
if (!matched) continue;
records.Add(record);
}

79
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs
View file

@ -5,6 +5,7 @@ namespace BotSharp.Core.Repository;
public partial class FileRepository
{
#region Configs
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
{
var vectorDir = BuildKnowledgeCollectionConfigDir();
@ -46,7 +47,6 @@ public partial class FileRepository
}
File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options));
return true;
}
@ -102,11 +102,88 @@ public partial class FileRepository
return configs;
}
#endregion
#region Documents
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData)
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
{
return false;
}
var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr());
var docDir = Path.Combine(dir, metaData.FileId);
if (!Directory.Exists(docDir))
{
Directory.CreateDirectory(docDir);
}
var metaFile = Path.Combine(docDir, KNOWLEDGE_DOC_META_FILE);
var content = JsonSerializer.Serialize(metaData, _options);
File.WriteAllText(metaFile, content);
return true;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return new PagedItems<KnowledgeDocMetaData>();
}
var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir))
{
return new PagedItems<KnowledgeDocMetaData>();
}
var records = new List<KnowledgeDocMetaData>();
foreach (var folder in Directory.GetDirectories(dir))
{
var metaFile = Path.Combine(folder, KNOWLEDGE_DOC_META_FILE);
if (!File.Exists(metaFile)) continue;
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _options);
if (metaData == null) continue;
var matched = true;
// Apply filter
if (filter != null && !filter.FileIds.IsNullOrEmpty())
{
matched = matched && filter.FileIds.Contains(metaData.FileId);
}
if (!matched) continue;
records.Add(metaData);
}
return new PagedItems<KnowledgeDocMetaData>
{
Items = records.Skip(filter.Offset).Take(filter.Size),
Count = records.Count
};
}
#endregion
#region Private methods
private string BuildKnowledgeCollectionConfigDir()
{
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
}
private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
}
#endregion
}

2
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
View file

@ -43,6 +43,8 @@ public partial class FileRepository : IBotSharpRepository
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string VECTOR_FOLDER = "vector";
private const string COLLECTION_CONFIG_FILE = "collection-config.json";
private const string KNOWLEDGE_DOC_FOLDER = "document";
private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
public FileRepository(
IServiceProvider services,

1
src/Infrastructure/BotSharp.Core/Using.cs
View file

@ -32,6 +32,7 @@ global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Translation.Attributes;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.Core.Repository;
global using BotSharp.Core.Routing;
global using BotSharp.Core.Agents.Services;

17
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -143,11 +143,20 @@ public class KnowledgeBaseController : ControllerBase
return response;
}
[HttpGet("/knowledge/document/{collection}/list")]
public async Task<IEnumerable<KnowledgeFileViewModel>> GetKnowledgeDocuments([FromRoute] string collection)
[HttpPost("/knowledge/document/{collection}/list")]
public async Task<PagedItems<KnowledgeFileViewModel>> GetPagedKnowledgeDocuments([FromRoute] string collection, [FromBody] GetKnowledgeDocsRequest request)
{
var files = await _knowledgeService.GetKnowledgeDocuments(collection);
return files.Select(x => KnowledgeFileViewModel.From(x));
var data = await _knowledgeService.GetPagedKnowledgeDocuments(collection, new KnowledgeFileFilter
{
Page = request.Page,
Size = request.Size
});
return new PagedItems<KnowledgeFileViewModel>
{
Items = data.Items.Select(x => KnowledgeFileViewModel.From(x)),
Count = data.Count
};
}
[HttpGet("/knowledge/document/{collection}/file/{fileId}")]

1
src/Infrastructure/BotSharp.OpenAPI/Using.cs
View file

@ -27,6 +27,7 @@ global using BotSharp.Abstraction.Repositories.Filters;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.VectorStorage.Enums;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.OpenAPI.ViewModels.Conversations;
global using BotSharp.OpenAPI.ViewModels.Users;
global using BotSharp.OpenAPI.ViewModels.Agents;

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GetKnowledgeDocsRequest.cs Normal file
View file

@ -0,0 +1,5 @@
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class GetKnowledgeDocsRequest : KnowledgeFileFilter
{
}

3
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs
View file

@ -1,6 +1,9 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class VectorKnowledgeUploadRequest
{
[JsonPropertyName("files")]
public IEnumerable<ExternalFileModel> Files { get; set; } = new List<ExternalFileModel>();
}

58
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.VectorStorage.Enums;
using System.Net.Http;
using System.Net.Mime;
@ -19,6 +20,7 @@ public partial class KnowledgeService
};
}
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
var userId = await GetUserId();
var vectorStoreProvider = _settings.VectorDb.Provider;
@ -49,15 +51,17 @@ public partial class KnowledgeService
}
// Save to vector db
var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents);
var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents, file.FileSource);
if (!dataIds.IsNullOrEmpty())
{
fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData
db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData
{
Collection = collectionName,
FileId = fileId,
FileName = file.FileName,
FileSource = file.FileSource,
ContentType = contentType,
VectorStoreProvider = vectorStoreProvider,
VectorDataIds = dataIds,
CreateDate = DateTime.UtcNow,
CreateUserId = userId
@ -94,18 +98,24 @@ public partial class KnowledgeService
try
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorDb = GetVectorDb();
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc meta data
var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = new[] { fileId }
});
// Delete doc
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty())
var found = pagedData?.Items?.FirstOrDefault();
if (found != null && !found.VectorDataIds.IsNullOrEmpty())
{
var guids = metaData.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
await vectorDb.DeleteCollectionData(collectionName, guids);
}
@ -120,20 +130,38 @@ public partial class KnowledgeService
}
}
public async Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName)
public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return Enumerable.Empty<KnowledgeFileModel>();
return new PagedItems<KnowledgeFileModel>();
}
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc meta data
var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
return files;
var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
Page = filter.Page,
Size = filter.Size
});
var files = pagedData.Items?.Select(x => new KnowledgeFileModel
{
FileId = x.FileId,
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId)
})?.ToList() ?? new List<KnowledgeFileModel>();
return new PagedItems<KnowledgeFileModel>
{
Items = files,
Count = pagedData.Count
};
}
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId)
@ -226,7 +254,9 @@ public partial class KnowledgeService
return saved;
}
private async Task<IEnumerable<string>> SaveToVectorDb(string collectionName, string fileId, string fileName, IEnumerable<string> contents)
private async Task<IEnumerable<string>> SaveToVectorDb(
string collectionName, string fileId, string fileName, IEnumerable<string> contents,
string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
{
if (contents.IsNullOrEmpty())
{
@ -244,8 +274,10 @@ public partial class KnowledgeService
var dataId = Guid.NewGuid();
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
{
{ "fileName", fileName },
{ "fileId", fileId },
{ KnowledgePayloadName.DataSource, vectorDataSource },
{ KnowledgePayloadName.FileId, fileId },
{ KnowledgePayloadName.FileName, fileName },
{ KnowledgePayloadName.FileSource, fileSource },
{ "textNumber", $"{i + 1}" }
});

15
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs Normal file
View file

@ -0,0 +1,15 @@
namespace BotSharp.Plugin.MongoStorage.Collections;
public class KnowledgeCollectionFileDocument : MongoBase
{
public string Collection { get; set; }
public string FileId { get; set; }
public string FileName { get; set; }
public string FileSource { get; set; }
public string ContentType { get; set; }
public string VectorStoreProvider { get; set; }
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
public string? WebUrl { get; set; }
public DateTime CreateDate { get; set; }
public string CreateUserId { get; set; }
}

3
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -156,4 +156,7 @@ public class MongoDbContext
public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
=> Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
public IMongoCollection<KnowledgeCollectionFileDocument> KnowledgeCollectionFiles
=> Database.GetCollection<KnowledgeCollectionFileDocument>($"{_collectionPrefix}_KnowledgeCollectionFiles");
}

81
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
View file

@ -1,9 +1,11 @@
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
#region Configs
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
{
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
@ -111,4 +113,83 @@ public partial class MongoRepository
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding)
});
}
#endregion
#region Documents
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData)
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
{
return false;
}
var doc = new KnowledgeCollectionFileDocument
{
Collection = metaData.Collection,
FileId = metaData.FileId,
FileName = metaData.FileName,
FileSource = metaData.FileSource,
ContentType = metaData.ContentType,
VectorStoreProvider = metaData.VectorStoreProvider,
VectorDataIds = metaData.VectorDataIds,
WebUrl = metaData.WebUrl,
CreateDate = metaData.CreateDate,
CreateUserId = metaData.CreateUserId
};
_dc.KnowledgeCollectionFiles.InsertOne(doc);
return true;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return new PagedItems<KnowledgeDocMetaData>();
}
var builder = Builders<KnowledgeCollectionFileDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
};
// Apply filters
if (filter != null && !filter.FileIds.IsNullOrEmpty())
{
docFilters.Add(builder.In(x => x.FileId, filter.FileIds));
}
var filterDef = builder.And(docFilters);
var sortDef = Builders<KnowledgeCollectionFileDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef);
var files = docs?.Select(x => new KnowledgeDocMetaData
{
Collection = x.Collection,
FileId = x.FileId,
FileName = x.FileName,
FileSource = x.FileSource,
ContentType = x.ContentType,
VectorStoreProvider = x.VectorStoreProvider,
VectorDataIds = x.VectorDataIds,
WebUrl = x.WebUrl,
CreateDate = x.CreateDate,
CreateUserId = x.CreateUserId
})?.ToList() ?? new List<KnowledgeDocMetaData>();
return new PagedItems<KnowledgeDocMetaData>
{
Items = files,
Count = (int)count
};
}
#endregion
}

2
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -192,7 +192,7 @@ public class QdrantDb : IVectorDb
{
foreach (var item in payload)
{
point.Payload.Add(item.Key, item.Value);
point.Payload[item.Key] = item.Value;
}
}

14
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Knowledges.Models;
namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
@ -14,17 +12,7 @@ public partial class TencentCosService
throw new NotImplementedException();
}
public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData)
{
throw new NotImplementedException();
}
public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
{
throw new NotImplementedException();
}
public IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider)
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
{
throw new NotImplementedException();
}

1
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.Users;
using BotSharp.Plugin.TencentCos.Settings;
using System.Net.Mime;