SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #644 from iceljc/features/add-knowledge-docs

Browse source 
refine knowledge doc
This commit is contained in:
iceljc 2024-09-17 12:51:02 -05:00 committed by GitHub
parent e775ebb6ef 16904e092c
commit cd96a4d05b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 240 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -63,16 +63,17 @@ public interface IFileStorageService
#endregion
#region Knowledge
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData);
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData);
/// <summary>
/// Delete files in a knowledge collection. If fileId is null, remove all files in the collection.
/// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="vectorStoreProvider"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string fileId);
FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId);
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
#endregion
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
View file

@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Models;
public class KnowledgeFileModel
{
public string FileId { get; set; }
public Guid FileId { get; set; }
public string FileName { get; set; }
public string FileExtension { get; set; }
public string ContentType { get; set; }

4
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -22,9 +22,9 @@ public interface IKnowledgeService
#region Document
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
#endregion
#region Common

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
View file

@ -6,7 +6,7 @@ public class KnowledgeDocMetaData
public string Collection { get; set; }
[JsonPropertyName("file_id")]
public string FileId { get; set; }
public Guid FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs
View file

@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeFileFilter : Pagination
{
public IEnumerable<string>? FileIds { get; set; }
public IEnumerable<Guid>? FileIds { get; set; }
}

12
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -112,7 +112,15 @@ public interface IBotSharpRepository
bool DeleteKnowledgeCollectionConfig(string collectionName);
IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
/// <summary>
/// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="vectorStoreProvider"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null);
PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
#endregion
}

46
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
View file

@ -1,23 +1,21 @@
using BotSharp.Abstraction.Knowledges.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId);
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId.ToString());
if (ExistDirectory(dir))
{
Directory.Delete(dir);
@ -40,7 +38,7 @@ public partial class LocalFileStorageService
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
@ -48,16 +46,16 @@ public partial class LocalFileStorageService
return false;
}
var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (string.IsNullOrEmpty(fileId))
if (fileId == null)
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId);
var fileDir = Path.Combine(dir, fileId.ToString());
if (ExistDirectory(fileDir))
{
Directory.Delete(fileDir, true);
@ -67,10 +65,10 @@ public partial class LocalFileStorageService
return true;
}
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return string.Empty;
}
@ -78,39 +76,31 @@ public partial class LocalFileStorageService
return $"/knowledge/document/{collectionName}/file/{fileId}";
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var fileDir = Path.Combine(docDir, fileId);
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = Path.Combine(docDir, fileId.ToString());
if (!ExistDirectory(fileDir)) return null;
var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE);
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
var file = Path.Combine(fileDir, metaData.FileName);
var file = Path.Combine(fileDir, fileName);
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
stream.Position = 0;
return new FileBinaryDataModel
{
FileName = metaData.FileName,
ContentType = metaData.ContentType,
FileBinaryData = BinaryData.FromStream(stream)
};
return BinaryData.FromStream(stream);
}
#region Private methods
private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
}
#endregion
}

1
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -21,7 +21,6 @@ public partial class LocalFileStorageService : IFileStorageService
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
{

5
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -234,7 +234,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
#endregion
#region Knowledge
#region KnowledgeBase
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
throw new NotImplementedException();
@ -247,6 +247,9 @@ public class BotSharpDbContext : Database, IBotSharpRepository
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) =>
throw new NotImplementedException();
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) =>
throw new NotImplementedException();
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) =>
throw new NotImplementedException();
#endregion

40
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs
View file

@ -110,14 +110,13 @@ public partial class FileRepository
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr());
var docDir = Path.Combine(dir, metaData.FileId);
var dir = BuildKnowledgeCollectionFileDir(metaData.Collection, metaData.VectorStoreProvider);
var docDir = Path.Combine(dir, metaData.FileId.ToString());
if (!Directory.Exists(docDir))
{
Directory.CreateDirectory(docDir);
@ -129,6 +128,33 @@ public partial class FileRepository
return true;
}
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir)) return false;
if (fileId == null)
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId.ToString());
if (Directory.Exists(fileDir))
{
Directory.Delete(fileDir, true);
}
}
return true;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
@ -137,7 +163,7 @@ public partial class FileRepository
return new PagedItems<KnowledgeDocMetaData>();
}
var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider);
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir))
{
return new PagedItems<KnowledgeDocMetaData>();
@ -181,9 +207,9 @@ public partial class FileRepository
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
}
private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider)
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
}
#endregion
}

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -137,7 +137,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
{
var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
return response;
@ -160,7 +160,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpGet("/knowledge/document/{collection}/file/{fileId}")]
public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
{
var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId);
var stream = file.FileBinaryData.ToStream();

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class KnowledgeFileViewModel
{
[JsonPropertyName("file_id")]
public string FileId { get; set; }
public Guid FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }

6
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs
View file

@ -44,11 +44,7 @@ public class WelcomeHook : ConversationHookBase
foreach (var message in messages)
{
var richContent = new RichContent<IRichMessage>(message)
{
Editor = message.RichType == RichTypeEnum.QuickReply ? EditorTypeEnum.None : EditorTypeEnum.Text,
};
var richContent = new RichContent<IRichMessage>(message);
var json = JsonSerializer.Serialize(new ChatResponseModel()
{
ConversationId = conversation.Id,

51
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -42,7 +42,7 @@ public partial class KnowledgeService
var contents = await GetFileContent(contentType, bytes);
// Save document
var fileId = Guid.NewGuid().ToString();
var fileId = Guid.NewGuid();
var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
if (!saved)
{
@ -89,9 +89,9 @@ public partial class KnowledgeService
}
public async Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId)
public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
@ -104,21 +104,23 @@ public partial class KnowledgeService
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc meta data
var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = new[] { fileId }
FileIds = [ fileId ],
Size = 1
});
// Delete doc
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
var found = pagedData?.Items?.FirstOrDefault();
fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId);
var found = pageData?.Items?.FirstOrDefault();
if (found != null && !found.VectorDataIds.IsNullOrEmpty())
{
var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
await vectorDb.DeleteCollectionData(collectionName, guids);
}
db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId);
return true;
}
catch (Exception ex)
@ -154,7 +156,7 @@ public partial class KnowledgeService
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId)
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName)
})?.ToList() ?? new List<KnowledgeFileModel>();
return new PagedItems<KnowledgeFileModel>
@ -164,14 +166,29 @@ public partial class KnowledgeService
};
}
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId)
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc binary data
var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
return file;
var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = [ fileId ],
Size = 1
});
var metaData = pageData?.Items?.FirstOrDefault();
if (metaData == null) return null;
var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
return new FileBinaryDataModel
{
FileName = metaData.FileName,
ContentType = metaData.ContentType,
FileBinaryData = binaryData
};
}
@ -246,16 +263,16 @@ public partial class KnowledgeService
#endregion
private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes)
{
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
var data = BinaryData.FromBytes(bytes);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, data);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data);
return saved;
}
private async Task<IEnumerable<string>> SaveToVectorDb(
string collectionName, string fileId, string fileName, IEnumerable<string> contents,
string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
{
if (contents.IsNullOrEmpty())
@ -275,7 +292,7 @@ public partial class KnowledgeService
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
{
{ KnowledgePayloadName.DataSource, vectorDataSource },
{ KnowledgePayloadName.FileId, fileId },
{ KnowledgePayloadName.FileId, fileId.ToString() },
{ KnowledgePayloadName.FileName, fileName },
{ KnowledgePayloadName.FileSource, fileSource },
{ "textNumber", $"{i + 1}" }

3
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
View file

@ -92,7 +92,8 @@ public partial class KnowledgeService
var vectorStoreProvider = _settings.VectorDb.Provider;
db.DeleteKnowledgeCollectionConfig(collectionName);
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider);
db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider);
}
return deleted;

4
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs → src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
View file

@ -1,9 +1,9 @@
namespace BotSharp.Plugin.MongoStorage.Collections;
public class KnowledgeCollectionFileDocument : MongoBase
public class KnowledgeCollectionFileMetaDocument : MongoBase
{
public string Collection { get; set; }
public string FileId { get; set; }
public Guid FileId { get; set; }
public string FileName { get; set; }
public string FileSource { get; set; }
public string ContentType { get; set; }

4
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -157,6 +157,6 @@ public class MongoDbContext
public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
=> Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
public IMongoCollection<KnowledgeCollectionFileDocument> KnowledgeCollectionFiles
=> Database.GetCollection<KnowledgeCollectionFileDocument>($"{_collectionPrefix}_KnowledgeCollectionFiles");
public IMongoCollection<KnowledgeCollectionFileMetaDocument> KnowledgeCollectionFileMeta
=> Database.GetCollection<KnowledgeCollectionFileMetaDocument>($"{_collectionPrefix}_KnowledgeCollectionFileMeta");
}

42
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
View file

@ -120,13 +120,12 @@ public partial class MongoRepository
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
{
return false;
}
var doc = new KnowledgeCollectionFileDocument
var doc = new KnowledgeCollectionFileMetaDocument
{
Id = Guid.NewGuid().ToString(),
Collection = metaData.Collection,
@ -140,10 +139,34 @@ public partial class MongoRepository
CreateUserId = metaData.CreateUserId
};
_dc.KnowledgeCollectionFiles.InsertOne(doc);
_dc.KnowledgeCollectionFileMeta.InsertOne(doc);
return true;
}
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
var filters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
};
if (fileId != null)
{
filters.Add(builder.Eq(x => x.FileId, fileId));
}
var res = _dc.KnowledgeCollectionFileMeta.DeleteMany(builder.And(filters));
return res.DeletedCount > 0;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
@ -152,9 +175,8 @@ public partial class MongoRepository
return new PagedItems<KnowledgeDocMetaData>();
}
var builder = Builders<KnowledgeCollectionFileDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileDocument>>()
var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
@ -167,9 +189,9 @@ public partial class MongoRepository
}
var filterDef = builder.And(docFilters);
var sortDef = Builders<KnowledgeCollectionFileDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef);
var sortDef = Builders<KnowledgeCollectionFileMetaDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef);
var files = docs?.Select(x => new KnowledgeDocMetaData
{

94
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
View file

@ -2,23 +2,101 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var dir = $"{docDir}/{fileId}";
if (ExistDirectory(dir))
{
_cosClient.BucketClient.DeleteDir(dir);
}
var file = $"{dir}/{fileName}";
var res = _cosClient.BucketClient.UploadBytes(file, fileData.ToArray());
return res;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving knowledge file " +
$"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (fileId == null)
{
_cosClient.BucketClient.DeleteDir(dir);
}
else
{
var fileDir = $"{dir}/{fileId}";
if (ExistDirectory(fileDir))
{
_cosClient.BucketClient.DeleteDir(fileDir);
}
}
return true;
}
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return string.Empty;
}
var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
}
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir)) return null;
var file = $"{fileDir}/{fileName}";
var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
if (bytes == null) return null;
return BinaryData.FromBytes(bytes);
}
#region Private methods
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return $"{KNOWLEDGE_FOLDER}/{KNOWLEDGE_DOC_FOLDER}/{vectorStoreProvider.CleanStr()}/{collectionName.CleanStr()}";
}
#endregion
}

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -28,6 +28,8 @@ public partial class TencentCosService : IFileStorageService
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
public TencentCosService(
TencentCosSettings settings,