add knowledge file list
This commit is contained in:
parent
1dbae69018
commit
2a3bea2cea
|
|
@ -76,5 +76,7 @@ public interface IFileStorageService
|
|||
bool SaveKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider,string fileId, KnowledgeDocMetaData metaData);
|
||||
|
||||
KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId);
|
||||
|
||||
IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider);
|
||||
#endregion
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
namespace BotSharp.Abstraction.Files.Models;
|
||||
|
||||
public class KnowledgeFileModel
|
||||
{
|
||||
public string FileId { get; set; }
|
||||
public string FileName { get; set; }
|
||||
public string FileExtension { get; set; }
|
||||
public string ContentType { get; set; }
|
||||
public string FileUrl { get; set; }
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ public interface IKnowledgeService
|
|||
#region Document
|
||||
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
|
||||
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
|
||||
Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName);
|
||||
#endregion
|
||||
|
||||
#region Common
|
||||
|
|
|
|||
|
|
@ -7,6 +7,9 @@ public class KnowledgeDocMetaData
|
|||
[JsonPropertyName("collection")]
|
||||
public string Collection { get; set; }
|
||||
|
||||
[JsonPropertyName("file_id")]
|
||||
public string FileId { get; set; }
|
||||
|
||||
[JsonPropertyName("file_name")]
|
||||
public string FileName { get; set; }
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
using AspectInjector.Broker;
|
||||
using BotSharp.Abstraction.Conversations.Models;
|
||||
using BotSharp.Abstraction.Files.Models;
|
||||
using BotSharp.Abstraction.Knowledges.Models;
|
||||
using System.IO;
|
||||
|
||||
|
|
@ -89,7 +92,9 @@ public partial class LocalFileStorageService
|
|||
|
||||
public KnowledgeDocMetaData? GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, string fileId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
|
||||
if (string.IsNullOrWhiteSpace(collectionName)
|
||||
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|
||||
|| string.IsNullOrWhiteSpace(fileId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
|
@ -106,8 +111,56 @@ public partial class LocalFileStorageService
|
|||
return metaData;
|
||||
}
|
||||
|
||||
public IEnumerable<KnowledgeFileModel> GetKnowledgeBaseFiles(string collectionName, string vectorStoreProvider)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(collectionName)
|
||||
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
|
||||
{
|
||||
return Enumerable.Empty<KnowledgeFileModel>();
|
||||
}
|
||||
|
||||
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
|
||||
if (!ExistDirectory(docDir))
|
||||
{
|
||||
return Enumerable.Empty<KnowledgeFileModel>();
|
||||
}
|
||||
|
||||
var files = new List<KnowledgeFileModel>();
|
||||
foreach (var folder in Directory.GetDirectories(docDir))
|
||||
{
|
||||
var metaFile = Path.Combine(docDir, folder, KNOWLEDGE_DOC_META_FILE);
|
||||
if (!File.Exists(metaFile)) continue;
|
||||
|
||||
var content = File.ReadAllText(metaFile);
|
||||
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
|
||||
if (metaData == null) continue;
|
||||
|
||||
var fileName = Path.GetFileNameWithoutExtension(metaData.FileName);
|
||||
var fileExtension = Path.GetExtension(metaData.FileName);
|
||||
|
||||
files.Add(new KnowledgeFileModel
|
||||
{
|
||||
FileId = metaData.FileId,
|
||||
FileName = metaData.FileName,
|
||||
FileExtension = fileExtension.Substring(1),
|
||||
ContentType = FileUtility.GetFileContentType(metaData.FileName),
|
||||
FileUrl = BuildKnowledgeFileUrl(collectionName, vectorStoreProvider, metaData.FileId)
|
||||
});
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
|
||||
#region Private methods
|
||||
private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
|
||||
{
|
||||
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
|
||||
}
|
||||
|
||||
private string BuildKnowledgeFileUrl(string collectionName, string vectorProvider, string fileId)
|
||||
{
|
||||
return $"/knowledge/file/{vectorProvider}/{collectionName}/{fileId}";
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Files.Constants;
|
||||
using BotSharp.Abstraction.Graph.Models;
|
||||
using BotSharp.Abstraction.Knowledges.Models;
|
||||
using BotSharp.Abstraction.VectorStorage.Models;
|
||||
|
|
@ -134,8 +135,16 @@ public class KnowledgeBaseController : ControllerBase
|
|||
var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
|
||||
return response;
|
||||
}
|
||||
|
||||
[HttpGet("/knowledge/document/{collection}/list")]
|
||||
public async Task<IEnumerable<KnowledgeFileViewModel>> GetKnowledgeDocuments([FromRoute] string collection)
|
||||
{
|
||||
var files = await _knowledgeService.GetKnowledgeDocuments(collection);
|
||||
return files.Select(x => KnowledgeFileViewModel.From(x));
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
||||
#region Common
|
||||
[HttpPost("/knowledge/vector/refresh-configs")]
|
||||
public async Task<string> RefreshVectorCollectionConfigs([FromBody] VectorCollectionConfigsModel request)
|
||||
|
|
@ -144,4 +153,15 @@ public class KnowledgeBaseController : ControllerBase
|
|||
return saved ? "Success" : "Fail";
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
||||
#region Private methods
|
||||
private FileContentResult BuildFileResult(string file)
|
||||
{
|
||||
using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
var bytes = new byte[stream.Length];
|
||||
stream.Read(bytes, 0, (int)stream.Length);
|
||||
return File(bytes, "application/octet-stream", Path.GetFileName(file));
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
|
||||
|
||||
public class KnowledgeFileViewModel
|
||||
{
|
||||
[JsonPropertyName("file_id")]
|
||||
public string FileId { get; set; }
|
||||
|
||||
[JsonPropertyName("file_name")]
|
||||
public string FileName { get; set; }
|
||||
|
||||
[JsonPropertyName("file_extension")]
|
||||
public string FileExtension { get; set; }
|
||||
|
||||
[JsonPropertyName("content_type")]
|
||||
public string ContentType { get; set; }
|
||||
|
||||
[JsonPropertyName("file_url")]
|
||||
public string FileUrl { get; set; }
|
||||
|
||||
public static KnowledgeFileViewModel From(KnowledgeFileModel model)
|
||||
{
|
||||
return new KnowledgeFileViewModel
|
||||
{
|
||||
FileId = model.FileId,
|
||||
FileName = model.FileName,
|
||||
FileExtension = model.FileExtension,
|
||||
ContentType = model.ContentType,
|
||||
FileUrl = model.FileUrl
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -74,6 +74,7 @@ public partial class KnowledgeService
|
|||
fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData
|
||||
{
|
||||
Collection = collectionName,
|
||||
FileId = fileId,
|
||||
FileName = file.FileName,
|
||||
ContentType = contentType,
|
||||
VectorDataIds = dataIds,
|
||||
|
|
@ -136,6 +137,21 @@ public partial class KnowledgeService
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public async Task<IEnumerable<KnowledgeFileModel>> GetKnowledgeDocuments(string collectionName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(collectionName))
|
||||
{
|
||||
return Enumerable.Empty<KnowledgeFileModel>();
|
||||
}
|
||||
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
var vectorStoreProvider = _settings.VectorDb.Provider;
|
||||
var files = fileStorage.GetKnowledgeBaseFiles(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
|
||||
return files;
|
||||
}
|
||||
|
||||
|
||||
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
|
||||
{
|
||||
var index = 0;
|
||||
|
|
|
|||
Loading…
Reference in a new issue