SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

sync file and vector

Browse source
This commit is contained in:
Jicheng Lu 2024-09-10 14:02:25 -05:00
parent 16c50075ee
commit 4cdb1981ab
34 changed files with 531 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -37,7 +37,7 @@ public interface IFileStorageService
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files);
/// <summary>
/// Delete files under messages
@ -54,7 +54,7 @@ public interface IFileStorageService
#region User
string GetUserAvatar();
bool SaveUserAvatar(InputFileModel file);
bool SaveUserAvatar(FileDataModel file);
#endregion
#region Speech
@ -63,6 +63,18 @@ public interface IFileStorageService
#endregion
#region Knowledge
bool SaveKnowledgeFiles(string collectionName, string fileId, string fileName, Stream stream);
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream);
/// <summary>
/// Delete files in a knowledge collection. If fileId is null, remove all files in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData);
KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId);
#endregion
}

7
src/Infrastructure/BotSharp.Abstraction/Files/Models/ExternalFileModel.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Abstraction.Files.Models;
public class ExternalFileModel : FileDataModel
{
[JsonPropertyName("file_url")]
public string FileUrl { get; set; } = string.Empty;
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs → src/Infrastructure/BotSharp.Abstraction/Files/Models/FileDataModel.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Files.Models;
public class InputFileModel : FileBase
public class FileDataModel : FileBase
{
/// <summary>
/// File name with extension

4
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -10,7 +10,6 @@ public interface IKnowledgeService
Task<bool> DeleteVectorCollection(string collectionName);
Task<IEnumerable<string>> GetVectorCollections(string type);
Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model);
Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
Task<bool> DeleteVectorCollectionData(string collectionName, string id);
Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
@ -22,7 +21,8 @@ public interface IKnowledgeService
#endregion
#region Document
Task<UploadKnowledgeResponse> UploadVectorKnowledge(string collectionName, IEnumerable<InputFileModel> files);
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
#endregion
#region Common

24
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs Normal file
View file

@ -0,0 +1,24 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeDocMetaData
{
[JsonPropertyName("collection")]
public string Collection { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
[JsonPropertyName("vector_data_ids")]
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;
[JsonPropertyName("create_user_id")]
public string CreateUserId { get; set; }
}

8
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/UploadKnowledgeResponse.cs
View file

@ -7,4 +7,12 @@ public class UploadKnowledgeResponse
[JsonPropertyName("failed")]
public IEnumerable<string> Failed { get; set; } = new List<string>();
[JsonPropertyName("is_success")]
public bool IsSuccess {
get
{
return !Success.IsNullOrEmpty() && Failed.IsNullOrEmpty();
}
}
}

2
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -101,7 +101,7 @@ public interface IBotSharpRepository
#endregion
#region Knowledge
#region KnowledgeBase
/// <summary>
/// Save knowledge collection configs. If reset is true, it will remove everything and then save the new configs.
/// </summary>

2
src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs
View file

@ -44,7 +44,7 @@ public static class StringExtensions
return str1.Equals(str2, option);
}
public static string RemoveWhiteSpaces(this string? str)
public static string CleanStr(this string? str)
{
if (string.IsNullOrWhiteSpace(str)) return string.Empty;

2
src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
View file

@ -13,5 +13,5 @@ public interface IVectorDb
Task<bool> DeleteCollection(string collectionName);
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
Task<bool> DeleteCollectionData(string collectionName, Guid id);
Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
}

1
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigFilter.cs
View file

@ -4,6 +4,7 @@ public class VectorCollectionConfigFilter
{
public IEnumerable<string>? CollectionNames { get; set; }
public IEnumerable<string>? CollectionTypes { get; set; }
public IEnumerable<string>? VectorStroageProviders { get; set; }
public static VectorCollectionConfigFilter Empty()
{

15
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionConfigModel.cs
View file

@ -20,14 +20,11 @@ public class VectorCollectionConfig
[JsonPropertyName("type")]
public string Type { get; set; }
[JsonPropertyName("vector_storage")]
public VectorStorageConfig VectorStorage { get; set; }
[JsonPropertyName("text_embedding")]
public KnowledgeEmbeddingConfig TextEmbedding { get; set; }
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;
[JsonPropertyName("create_user_id")]
public string CreateUserId { get; set; } = string.Empty;
}
public class KnowledgeEmbeddingConfig
@ -40,4 +37,10 @@ public class KnowledgeEmbeddingConfig
[JsonPropertyName("dimension")]
public int Dimension { get; set; }
}
public class VectorStorageConfig
{
[JsonPropertyName("provider")]
public string Provider { get; set; }
}

2
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -118,7 +118,7 @@ public partial class LocalFileStorageService
return foundMsgs;
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files)
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files)
{
if (files.IsNullOrEmpty()) return false;

34
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Knowledge.cs
View file

@ -1,34 +0,0 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public bool SaveKnowledgeFiles(string collectionName, string fileId, string fileName, Stream stream)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
{
return false;
}
try
{
var dir = Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, collectionName, fileId);
if (ExistDirectory(dir))
{
Directory.Delete(dir);
}
Directory.CreateDirectory(dir);
var filePath = Path.Combine(dir, fileName);
using var fs = File.Create(filePath);
stream.CopyTo(fs);
return true;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving knowledge file (Collection: {collectionName}, File name: {fileName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
}

113
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs Normal file
View file

@ -0,0 +1,113 @@
using BotSharp.Abstraction.Knowledges.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, Stream stream)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId);
if (ExistDirectory(dir))
{
Directory.Delete(dir);
}
Directory.CreateDirectory(dir);
var filePath = Path.Combine(dir, fileName);
using var fs = File.Create(filePath);
stream.CopyTo(fs);
return true;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving knowledge file " +
$"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (string.IsNullOrEmpty(fileId))
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId);
if (ExistDirectory(fileDir))
{
Directory.Delete(fileDir, true);
}
}
return true;
}
public bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId, KnowledgeDocMetaData metaData)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
{
return false;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId);
if (!ExistDirectory(dir))
{
Directory.CreateDirectory(dir);
}
var metaFile = Path.Combine(dir, KNOWLEDGE_DOC_META_FILE);
var content = JsonSerializer.Serialize(metaData, _jsonOptions);
File.WriteAllText(metaFile, content);
return true;
}
public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
{
return null;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var metaFile = Path.Combine(docDir, fileId, KNOWLEDGE_DOC_META_FILE);
if (!File.Exists(metaFile))
{
return null;
}
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
return metaData;
}
private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
}
}

2
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs
View file

@ -16,7 +16,7 @@ public partial class LocalFileStorageService
return found;
}
public bool SaveUserAvatar(InputFileModel file)
public bool SaveUserAvatar(FileDataModel file)
{
if (file == null || string.IsNullOrEmpty(file.FileData)) return false;

9
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -21,6 +21,15 @@ public partial class LocalFileStorageService : IFileStorageService
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true,
AllowTrailingCommas = true
};
public LocalFileStorageService(
BotSharpDatabaseSettings dbSettings,

53
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Knowledge.cs → src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs
View file

@ -7,13 +7,13 @@ public partial class FileRepository
{
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
{
var dir = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
if (!Directory.Exists(dir))
var vectorDir = BuildKnowledgeCollectionConfigDir();
if (!Directory.Exists(vectorDir))
{
Directory.CreateDirectory(dir);
Directory.CreateDirectory(vectorDir);
}
var configFile = Path.Combine(dir, COLLECTION_CONFIG_FILE);
var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE);
if (reset)
{
File.WriteAllText(configFile, JsonSerializer.Serialize(configs ?? new(), _options));
@ -27,7 +27,24 @@ public partial class FileRepository
var str = File.ReadAllText(configFile);
var savedConfigs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
savedConfigs.AddRange(configs);
// Update if collection already exists, otherwise insert
foreach (var config in configs)
{
if (string.IsNullOrWhiteSpace(config.Name)) continue;
var found = savedConfigs.FirstOrDefault(x => x.Name == config.Name);
if (found != null)
{
found.TextEmbedding = config.TextEmbedding;
found.Type = config.Type;
}
else
{
savedConfigs.Add(config);
}
}
File.WriteAllText(configFile, JsonSerializer.Serialize(savedConfigs ?? new(), _options));
return true;
@ -37,7 +54,8 @@ public partial class FileRepository
{
if (string.IsNullOrWhiteSpace(collectionName)) return false;
var configFile = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE);
var vectorDir = BuildKnowledgeCollectionConfigDir();
var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE);
if (!File.Exists(configFile)) return false;
var str = File.ReadAllText(configFile);
@ -55,15 +73,18 @@ public partial class FileRepository
return Enumerable.Empty<VectorCollectionConfig>();
}
var file = Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER, COLLECTION_CONFIG_FILE);
if (!File.Exists(file))
var vectorDir = BuildKnowledgeCollectionConfigDir();
var configFile = Path.Combine(vectorDir, COLLECTION_CONFIG_FILE);
if (!File.Exists(configFile))
{
return Enumerable.Empty<VectorCollectionConfig>();
}
var str = File.ReadAllText(file);
var configs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(str, _options) ?? new();
// Get data
var content = File.ReadAllText(configFile);
var configs = JsonSerializer.Deserialize<List<VectorCollectionConfig>>(content, _options) ?? new();
// Apply filters
if (!filter.CollectionNames.IsNullOrEmpty())
{
configs = configs.Where(x => filter.CollectionNames.Contains(x.Name)).ToList();
@ -74,6 +95,18 @@ public partial class FileRepository
configs = configs.Where(x => filter.CollectionTypes.Contains(x.Type)).ToList();
}
if (!filter.VectorStroageProviders.IsNullOrEmpty())
{
configs = configs.Where(x => filter.VectorStroageProviders.Contains(x.VectorStorage?.Provider)).ToList();
}
return configs;
}
#region Private methods
private string BuildKnowledgeCollectionConfigDir()
{
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
}
#endregion
}

38
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -32,7 +32,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpDelete("knowledge/vector/{collection}/delete-collection")]
public async Task<bool> DeleteVectorCollections([FromRoute] string collection)
public async Task<bool> DeleteVectorCollection([FromRoute] string collection)
{
return await _knowledgeService.DeleteVectorCollection(collection);
}
@ -121,33 +121,17 @@ public class KnowledgeBaseController : ControllerBase
#region Document
//[HttpPost("/knowledge/vector/{collection}/upload")]
//public async Task<IActionResult> UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum)
//{
// var setttings = _services.GetRequiredService<FileCoreSettings>();
// var textConverter = _services.GetServices<IPdf2TextConverter>().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider);
// var filePath = Path.GetTempFileName();
// using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None))
// {
// await file.CopyToAsync(stream);
// await stream.FlushAsync();
// }
// var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum);
// await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel
// {
// Content = content
// });
// System.IO.File.Delete(filePath);
// return Ok(new { count = 1, file.Length });
//}
[HttpPost("/knowledge/vector/{collection}/upload")]
public async Task<UploadKnowledgeResponse> UploadVectorKnowledge([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
[HttpPost("/knowledge/document/{collection}/upload")]
public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments([FromRoute] string collection, [FromBody] VectorKnowledgeUploadRequest request)
{
var response = await _knowledgeService.UploadVectorKnowledge(collection, request.Files);
var response = await _knowledgeService.UploadKnowledgeDocuments(collection, request.Files);
return response;
}
[HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
{
var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
return response;
}
#endregion

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -138,7 +138,7 @@ public class UserController : ControllerBase
public bool UploadUserAvatar([FromBody] UserAvatarModel input)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var file = new InputFileModel
var file = new FileDataModel
{
FileName = input.FileName,
FileData = input.FileData,

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs
View file

@ -3,5 +3,5 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations;
public class InputMessageFiles
{
public List<MessageState> States { get; set; } = new();
public List<InputFileModel> Files { get; set; } = new();
public List<FileDataModel> Files { get; set; } = new();
}

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeUploadRequest.cs
View file

@ -2,5 +2,5 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class VectorKnowledgeUploadRequest
{
public IEnumerable<InputFileModel> Files { get; set; } = new List<InputFileModel>();
public IEnumerable<ExternalFileModel> Files { get; set; } = new List<ExternalFileModel>();
}

4
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback
{
if (image == null) return;
var files = new List<InputFileModel>()
var files = new List<FileDataModel>()
{
new InputFileModel
new FileDataModel
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -77,7 +77,7 @@ public class GenerateImageFn : IFunctionCallback
{
if (images.IsNullOrEmpty()) return;
var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel
var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}"

2
src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
View file

@ -74,7 +74,7 @@ public class MemoryVectorDb : IVectorDb
return true;
}
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
{
return await Task.FromResult(false);
}

8
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Common.cs
View file

@ -6,14 +6,6 @@ public partial class KnowledgeService
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var collections = configs.Collections ?? new();
var userId = await GetUserId();
foreach (var collection in collections)
{
collection.CreateDate = DateTime.UtcNow;
collection.CreateUserId = userId;
}
var saved = db.AddKnowledgeCollectionConfigs(collections, reset: true);
return await Task.FromResult(saved);
}

102
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -1,12 +1,13 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using System.Net.Http;
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task<UploadKnowledgeResponse> UploadVectorKnowledge(string collectionName, IEnumerable<InputFileModel> files)
public async Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files)
{
if (string.IsNullOrWhiteSpace(collectionName))
{
@ -18,30 +19,32 @@ public partial class KnowledgeService
}
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
var cleanCollectionName = collectionName.RemoveWhiteSpaces();
var userId = await GetUserId();
var vectorStoreProvider = _settings.VectorDb.Provider;
var successFiles = new List<string>();
var failedFiles = new List<string>();
foreach (var file in files)
{
if (string.IsNullOrWhiteSpace(file.FileData) || string.IsNullOrWhiteSpace(file.FileName))
if (string.IsNullOrWhiteSpace(file.FileData)
&& string.IsNullOrWhiteSpace(file.FileUrl))
{
continue;
}
var dataIds = new List<string>();
try
{
// Chop text
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var dataIds = new List<string>();
// Chop text (to do)
var (contentType, bytes) = await GetFileInfo(file);
using var stream = new MemoryStream(bytes);
using var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync();
// Save file
var fileId = Guid.NewGuid().ToString();
var saved = fileStoreage.SaveKnowledgeFiles(cleanCollectionName, fileId, file.FileName, stream);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, file.FileName, stream);
reader.Close();
stream.Close();
@ -58,15 +61,31 @@ public partial class KnowledgeService
// Save to vector db
var dataId = Guid.NewGuid();
await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
{
{ "fileName", file.FileName },
{ "fileId", fileId },
{ "page", "0" }
});
dataIds.Add(dataId.ToString());
successFiles.Add(file.FileName);
if (saved)
{
dataIds.Add(dataId.ToString());
fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData
{
Collection = collectionName,
FileName = file.FileName,
ContentType = contentType,
VectorDataIds = dataIds,
CreateDate = DateTime.UtcNow,
CreateUserId = userId
});
successFiles.Add(file.FileName);
}
else
{
failedFiles.Add(file.FileName);
}
}
catch (Exception ex)
{
@ -84,6 +103,39 @@ public partial class KnowledgeService
}
public async Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
{
return false;
}
try
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorDb = GetVectorDb();
var vectorStoreProvider = _settings.VectorDb.Provider;
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty())
{
var guids = metaData.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
await vectorDb.DeleteCollectionData(collectionName, guids);
}
return true;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting knowledge document " +
$"(Collection: {collectionName}, File id: {fileId})" +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
{
var index = 0;
@ -108,5 +160,33 @@ public partial class KnowledgeService
}
#region Private methods
/// <summary>
/// Get file content type and file bytes
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
private async Task<(string, byte[])> GetFileInfo(ExternalFileModel file)
{
if (file == null)
{
return (string.Empty, new byte[0]);
}
if (!string.IsNullOrWhiteSpace(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
var contentType = FileUtility.GetFileContentType(file.FileName);
using var client = http.CreateClient();
var bytes = await client.GetByteArrayAsync(file.FileUrl);
return (contentType, bytes);
}
else if (!string.IsNullOrWhiteSpace(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
return (contentType, bytes);
}
return (string.Empty, new byte[0]);
}
#endregion
}

21
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Files;
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
@ -25,14 +27,16 @@ public partial class KnowledgeService
{
Name = collectionName,
Type = collectionType,
VectorStorage = new VectorStorageConfig
{
Provider = _settings.VectorDb.Provider
},
TextEmbedding = new KnowledgeEmbeddingConfig
{
Provider = provider,
Model = model,
Dimension = dimension
},
CreateDate = DateTime.UtcNow,
CreateUserId = userId
}
}
});
}
@ -53,7 +57,8 @@ public partial class KnowledgeService
var db = _services.GetRequiredService<IBotSharpRepository>();
var collectionNames = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
{
CollectionTypes = new[] { type }
CollectionTypes = new[] { type },
VectorStroageProviders = new[] { _settings.VectorDb.Provider }
}).Select(x => x.Name).ToList();
var vectorDb = GetVectorDb();
@ -82,7 +87,11 @@ public partial class KnowledgeService
if (deleted)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorStoreProvider = _settings.VectorDb.Provider;
db.DeleteKnowledgeCollectionConfig(collectionName);
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
}
return deleted;
@ -156,7 +165,7 @@ public partial class KnowledgeService
}
var db = GetVectorDb();
return await db.DeleteCollectionData(collectionName, guid);
return await db.DeleteCollectionData(collectionName, new List<Guid> { guid });
}
catch (Exception ex)
{
@ -202,7 +211,7 @@ public partial class KnowledgeService
catch (Exception ex)
{
_logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return new List<VectorSearchResult>();
return Enumerable.Empty<VectorSearchResult>();
}
}
#endregion

2
src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs
View file

@ -48,7 +48,7 @@ public class FaissDb : IVectorDb
throw new NotImplementedException();
}
public Task<bool> DeleteCollectionData(string collectionName, Guid id)
public Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
{
throw new NotImplementedException();
}

3
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionConfigDocument.cs
View file

@ -4,7 +4,6 @@ public class KnowledgeCollectionConfigDocument : MongoBase
{
public string Name { get; set; }
public string Type { get; set; }
public KnowledgeVectorStorageConfigMongoModel VectorStorage { get; set; }
public KnowledgeEmbeddingConfigMongoModel TextEmbedding { get; set; }
public DateTime CreateDate { get; set; }
public string CreateUserId { get; set; }
}

24
src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStorageConfigMongoModel.cs Normal file
View file

@ -0,0 +1,24 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Models;
public class KnowledgeVectorStorageConfigMongoModel
{
public string Provider { get; set; }
public static KnowledgeVectorStorageConfigMongoModel ToMongoModel(VectorStorageConfig model)
{
return new KnowledgeVectorStorageConfigMongoModel
{
Provider = model.Provider
};
}
public static VectorStorageConfig ToDomainModel(KnowledgeVectorStorageConfigMongoModel model)
{
return new VectorStorageConfig
{
Provider = model.Provider
};
}
}

60
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Knowledge.cs
View file

@ -1,60 +0,0 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
{
var docs = configs?.Select(x => new KnowledgeCollectionConfigDocument
{
Id = Guid.NewGuid().ToString(),
Name = x.Name,
Type = x.Type,
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding),
CreateDate = x.CreateDate,
CreateUserId = x.CreateUserId,
})?.ToList() ?? new List<KnowledgeCollectionConfigDocument>();
if (reset)
{
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
_dc.KnowledgeCollectionConfigs.DeleteMany(filter);
}
_dc.KnowledgeCollectionConfigs.InsertMany(docs);
return true;
}
public bool DeleteKnowledgeCollectionConfig(string collectionName)
{
if (string.IsNullOrWhiteSpace(collectionName)) return false;
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Name, collectionName);
var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
return deleted.DeletedCount > 0;
}
public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter)
{
if (filter == null)
{
return Enumerable.Empty<VectorCollectionConfig>();
}
var builder = Builders<KnowledgeCollectionConfigDocument>.Filter;
var filters = new List<FilterDefinition<KnowledgeCollectionConfigDocument>> { builder.Empty };
var configs = _dc.KnowledgeCollectionConfigs.Find(Builders<KnowledgeCollectionConfigDocument>.Filter.And(filters)).ToList();
return configs.Select(x => new VectorCollectionConfig
{
Name = x.Name,
Type = x.Type,
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding),
CreateDate = x.CreateDate,
CreateUserId= x.CreateUserId
});
}
}

113
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs Normal file
View file

@ -0,0 +1,113 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false)
{
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Empty;
var docs = configs?.Where(x => !string.IsNullOrWhiteSpace(x.Name))
.Select(x => new KnowledgeCollectionConfigDocument
{
Id = Guid.NewGuid().ToString(),
Name = x.Name,
Type = x.Type,
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToMongoModel(x.TextEmbedding)
})?.ToList() ?? new List<KnowledgeCollectionConfigDocument>();
if (reset)
{
_dc.KnowledgeCollectionConfigs.DeleteMany(filter);
_dc.KnowledgeCollectionConfigs.InsertMany(docs);
return true;
}
// Update if collection already exists, otherwise insert.
var insertDocs = new List<KnowledgeCollectionConfigDocument>();
var updateDocs = new List<KnowledgeCollectionConfigDocument>();
var names = docs.Select(x => x.Name).ToList();
filter = Builders<KnowledgeCollectionConfigDocument>.Filter.In(x => x.Name, names);
var savedConfigs = _dc.KnowledgeCollectionConfigs.Find(filter).ToList();
foreach (var doc in docs)
{
var found = savedConfigs.FirstOrDefault(x => x.Name == doc.Name);
if (found != null)
{
found.Type = doc.Type;
found.VectorStorage = doc.VectorStorage;
found.TextEmbedding = doc.TextEmbedding;
updateDocs.Add(found);
}
else
{
insertDocs.Add(doc);
}
}
if (!insertDocs.IsNullOrEmpty())
{
_dc.KnowledgeCollectionConfigs.InsertMany(docs);
}
if (!updateDocs.IsNullOrEmpty())
{
foreach (var doc in updateDocs)
{
filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Id, doc.Id);
_dc.KnowledgeCollectionConfigs.ReplaceOne(filter, doc);
}
}
return true;
}
public bool DeleteKnowledgeCollectionConfig(string collectionName)
{
if (string.IsNullOrWhiteSpace(collectionName)) return false;
var filter = Builders<KnowledgeCollectionConfigDocument>.Filter.Eq(x => x.Name, collectionName);
var deleted = _dc.KnowledgeCollectionConfigs.DeleteMany(filter);
return deleted.DeletedCount > 0;
}
public IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter)
{
if (filter == null)
{
return Enumerable.Empty<VectorCollectionConfig>();
}
var builder = Builders<KnowledgeCollectionConfigDocument>.Filter;
var filters = new List<FilterDefinition<KnowledgeCollectionConfigDocument>> { builder.Empty };
// Apply filters
if (!filter.CollectionNames.IsNullOrEmpty())
{
filters.Add(builder.In(x => x.Name, filter.CollectionNames));
}
if (!filter.CollectionTypes.IsNullOrEmpty())
{
filters.Add(builder.In(x => x.Type, filter.CollectionTypes));
}
if (!filter.VectorStroageProviders.IsNullOrEmpty())
{
filters.Add(builder.In(x => x.VectorStorage.Provider, filter.VectorStroageProviders));
}
// Get data
var configs = _dc.KnowledgeCollectionConfigs.Find(Builders<KnowledgeCollectionConfigDocument>.Filter.And(filters)).ToList();
return configs.Select(x => new VectorCollectionConfig
{
Name = x.Name,
Type = x.Type,
VectorStorage = KnowledgeVectorStorageConfigMongoModel.ToDomainModel(x.VectorStorage),
TextEmbedding = KnowledgeEmbeddingConfigMongoModel.ToDomainModel(x.TextEmbedding)
});
}
}

35
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Abstraction.VectorStorage.Models;
using Microsoft.Extensions.Logging;
using Qdrant.Client;
using Qdrant.Client.Grpc;
@ -10,12 +11,15 @@ public class QdrantDb : IVectorDb
private QdrantClient _client;
private readonly QdrantSetting _setting;
private readonly IServiceProvider _services;
private readonly ILogger<QdrantDb> _logger;
public QdrantDb(
QdrantSetting setting,
ILogger<QdrantDb> logger,
IServiceProvider services)
{
_setting = setting;
_logger = logger;
_services = services;
}
@ -35,21 +39,28 @@ public class QdrantDb : IVectorDb
return _client;
}
public async Task<bool> CreateCollection(string collectionName, int dim)
public async Task<bool> CreateCollection(string collectionName, int dimension)
{
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
if (exist) return false;
// Create a new collection
await client.CreateCollectionAsync(collectionName, new VectorParams()
try
{
Size = (ulong)dim,
Distance = Distance.Cosine
});
return true;
// Create a new collection
await client.CreateCollectionAsync(collectionName, new VectorParams()
{
Size = (ulong)dimension,
Distance = Distance.Cosine
});
return true;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when create collection (Name: {collectionName}, Dimension: {dimension}).");
return false;
}
}
public async Task<bool> DeleteCollection(string collectionName)
@ -145,8 +156,6 @@ public class QdrantDb : IVectorDb
});
}
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
// Insert vectors
@ -216,10 +225,12 @@ public class QdrantDb : IVectorDb
return results;
}
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
{
if (ids.IsNullOrEmpty()) return false;
var client = GetClient();
var result = await client.DeleteAsync(collectionName, id);
var result = await client.DeleteAsync(collectionName, ids);
return result.Status == UpdateStatus.Completed;
}

16
src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
View file

@ -4,6 +4,7 @@ using BotSharp.Abstraction.VectorStorage.Models;
using Microsoft.SemanticKernel.Memory;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace BotSharp.Plugin.SemanticKernel
@ -84,16 +85,15 @@ namespace BotSharp.Plugin.SemanticKernel
return true;
}
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
{
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
if (ids.IsNullOrEmpty()) return false;
if (exist)
{
await _memoryStore.RemoveAsync(collectionName, id.ToString());
return true;
}
return false;
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
if (!exist) return false;
await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString()));
return true;
}
}
}