add knowledge file list

This commit is contained in:
Jicheng Lu 2024-09-10 14:33:42 -05:00
parent 1dbae69018
commit 2a3bea2cea
8 changed files with 139 additions and 1 deletions

View file

@ -76,5 +76,7 @@ public interface IFileStorageService
bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData);
KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId);
IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider);
#endregion
}

View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Files.Models;
public class KnowledgeFileModel
{
public string FileId { get; set; }
public string FileName { get; set; }
public string FileExtension { get; set; }
public string ContentType { get; set; }
public string FileUrl { get; set; }
}

View file

@ -23,6 +23,7 @@ public interface IKnowledgeService
#region Document
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName);
#endregion
#region Common

View file

@ -7,6 +7,9 @@ public class KnowledgeDocMetaData
[JsonPropertyName("collection")]
public string Collection { get; set; }
[JsonPropertyName("file_id")]
public string FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }

View file

@ -1,3 +1,6 @@
using AspectInjector.Broker;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Knowledges.Models;
using System.IO;
@ -89,7 +92,9 @@ public partial class LocalFileStorageService
public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
{
return null;
}
@ -106,8 +111,56 @@ public partial class LocalFileStorageService
return metaData;
}
public IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return Enumerable.Empty<KnowledgeFileModel>();
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(docDir))
{
return Enumerable.Empty<KnowledgeFileModel>();
}
var files = new List<KnowledgeFileModel>();
foreach (var folder in Directory.GetDirectories(docDir))
{
var metaFile = Path.Combine(docDir, folder, KNOWLEDGE_DOC_META_FILE);
if (!File.Exists(metaFile)) continue;
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
if (metaData == null) continue;
var fileName = Path.GetFileNameWithoutExtension(metaData.FileName);
var fileExtension = Path.GetExtension(metaData.FileName);
files.Add(new KnowledgeFileModel
{
FileId = metaData.FileId,
FileName = metaData.FileName,
FileExtension = fileExtension.Substring(1),
ContentType = FileUtility.GetFileContentType(metaData.FileName),
FileUrl = BuildKnowledgeFileUrl(collectionName, vectorStoreProvider, metaData.FileId)
});
}
return files;
}
#region Private methods
private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
}
private string BuildKnowledgeFileUrl(string collectionName, string vectorProvider, string fileId)
{
return $"/knowledge/file/{vectorProvider}/{collectionName}/{fileId}";
}
#endregion
}

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Files.Constants;
using BotSharp.Abstraction.Graph.Models;
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.VectorStorage.Models;
@ -134,8 +135,16 @@ public class KnowledgeBaseController : ControllerBase
var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
return response;
}
[HttpGet("/knowledge/document/{collection}/list")]
public async Task<IEnumerable<KnowledgeFileViewModel>> GetKnowledgeDocuments([FromRoute] string collection)
{
var files = await _knowledgeService.GetKnowledgeDocuments(collection);
return files.Select(x => KnowledgeFileViewModel.From(x));
}
#endregion
#region Common
[HttpPost("/knowledge/vector/refresh-configs")]
public async Task<string> RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request)
@ -144,4 +153,15 @@ public class KnowledgeBaseController : ControllerBase
return saved ? "Success" : "Fail";
}
#endregion
#region Private methods
private FileContentResult BuildFileResult(string file)
{
using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read);
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
return File(bytes, "application/octet-stream", Path.GetFileName(file));
}
#endregion
}

View file

@ -0,0 +1,33 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class KnowledgeFileViewModel
{
[JsonPropertyName("file_id")]
public string FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_extension")]
public string FileExtension { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
public static KnowledgeFileViewModel From(KnowledgeFileModel model)
{
return new KnowledgeFileViewModel
{
FileId = model.FileId,
FileName = model.FileName,
FileExtension = model.FileExtension,
ContentType = model.ContentType,
FileUrl = model.FileUrl
};
}
}

View file

@ -74,6 +74,7 @@ public partial class KnowledgeService
fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData
{
Collection = collectionName,
FileId = fileId,
FileName = file.FileName,
ContentType = contentType,
VectorDataIds = dataIds,
@ -136,6 +137,21 @@ public partial class KnowledgeService
}
}
public async Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName)
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return Enumerable.Empty<KnowledgeFileModel>();
}
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorStoreProvider = _settings.VectorDb.Provider;
var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
return files;
}
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
{
var index = 0;