SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #581 from iceljc/features/refine-file-select

Browse source 
Features/refine file select
This commit is contained in:
iceljc 2024-08-08 16:00:22 -05:00 committed by GitHub
parent 0af2085c33 6f62ca37c0
commit 6f13ec2d73
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 255 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -4,21 +4,27 @@ namespace BotSharp.Abstraction.Files;
public interface IFileStorageService
{
#region Common
string GetDirectory(string conversationId);
byte[] GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);
void DeleteDirectory(string dir);
string BuildDirectory(params string[] segments);
#endregion
#region Conversation
/// <summary>
/// Get the files that have been uploaded in the chat.
/// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
/// Get the message file screenshots for specific content types, e.g., pdf
/// </summary>
/// <param name="conversationId"></param>
/// <param name="source"></param>
/// <param name="dialogs"></param>
/// <param name="contentTypes"></param>
/// <param name="includeScreenShot"></param>
/// <param name="offset"></param>
/// <param name="messageIds"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes,
bool includeScreenShot = false, int? offset = null);
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds);
/// <summary>
/// Get the files that have been uploaded in the chat. No screenshot images are included.
@ -45,20 +51,9 @@ public interface IFileStorageService
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
#endregion
#region User
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
#endregion
#region Common
string GetDirectory(string conversationId);
byte[] GetFileBytes(string filePath);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);
void DeleteDirectory(string dir);
string BuildDirectory(params string[] segments);
#endregion
}

4
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs
View file

@ -40,7 +40,7 @@ public class FileBase
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_type")]
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileType { get; set; } = string.Empty;
public string? FileExtension { get; set; } = string.Empty;
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -15,6 +15,6 @@ public class MessageFileModel : FileBase
public override string ToString()
{
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}";
return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}";
}
}

35
src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs
View file

@ -2,13 +2,48 @@ namespace BotSharp.Abstraction.Files.Models;
public class SelectFileOptions
{
/// <summary>
/// Llm provider
/// </summary>
public string? Provider { get; set; }
/// <summary>
/// Llm model id
/// </summary>
public string? ModelId { get; set; }
/// <summary>
/// Agent id
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Template (prompt) name
/// </summary>
public string? Template { get; set; }
/// <summary>
/// Description that user provides to select files
/// </summary>
public string? Description { get; set; }
/// <summary>
/// Whether include bot generated files
/// </summary>
public bool IncludeBotFile { get; set; }
/// <summary>
/// Conversation breakpoint
/// </summary>
public bool FromBreakpoint { get; set; }
/// <summary>
/// Message offset from last
/// </summary>
public int? Offset { get; set; }
/// <summary>
/// File content types. If null, all types of files will be retrived
/// </summary>
public IEnumerable<string>? ContentTypes { get; set; }
}

6
src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs
View file

@ -1,10 +1,4 @@
using BotSharp.Abstraction.Repositories.Enums;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.Extensions.DependencyInjection;
using System;
using System.IO;
using System.Net.Http;
using System.Net.Mime;
namespace BotSharp.Abstraction.Files.Utilities;

2
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

22
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs
View file

@ -20,7 +20,7 @@ public partial class FileInstructService
if (options.IncludeBotFile)
{
var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes);
files = files.Concat(botFiles);
files = MergeMessageFiles(messageIds, files, botFiles);
}
if (files.IsNullOrEmpty())
@ -31,6 +31,24 @@ public partial class FileInstructService
return await SelectFiles(files, dialogs, options);
}
private IEnumerable<MessageFileModel> MergeMessageFiles(IEnumerable<string> messageIds, IEnumerable<MessageFileModel> userFiles, IEnumerable<MessageFileModel> botFiles)
{
var files = new List<MessageFileModel>();
if (messageIds.IsNullOrEmpty()) return files;
foreach (var messageId in messageIds)
{
var users = userFiles.Where(x => x.MessageId == messageId).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).ToList();
if (!users.IsNullOrEmpty()) files.AddRange(users);
if (!bots.IsNullOrEmpty()) files.AddRange(bots);
}
return files;
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, IEnumerable<RoleDialogModel> dialogs, SelectFileOptions options)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
@ -43,7 +61,7 @@ public partial class FileInstructService
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}";
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}";
}).ToList();
var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant;

4
src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs
View file

@ -14,9 +14,9 @@ public partial class LocalFileStorageService
return dir;
}
public byte[] GetFileBytes(string filePath)
public byte[] GetFileBytes(string fileStorageUrl)
{
using var stream = File.OpenRead(filePath);
using var stream = File.OpenRead(fileStorageUrl);
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
return bytes;

164
src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -6,22 +6,20 @@ namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
{
return files;
}
var messageIds = GetMessageIds(dialogs, offset);
var source = FileSourceType.User;
var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
foreach (var messageId in messageIds)
{
var dir = Path.Combine(pathPrefix, messageId, source);
var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User);
if (!ExistDirectory(dir)) continue;
foreach (var subDir in Directory.GetDirectories(dir))
@ -29,22 +27,16 @@ public partial class LocalFileStorageService
var file = Directory.GetFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var screenshots = await GetScreenshots(file, subDir, messageId, source);
if (screenshots.IsNullOrEmpty()) continue;
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
files.AddRange(foundFiles);
files.AddRange(screenshots);
}
}
return files;
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds,
string source, IEnumerable<string>? contentTypes = null)
{
@ -72,14 +64,14 @@ public partial class LocalFileStorageService
}
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var fileExtension = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}",
FileStorageUrl = file,
FileName = fileName,
FileType = fileType,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
};
@ -268,85 +260,6 @@ public partial class LocalFileStorageService
return messageIds;
}
private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, string fileDir, string contentType,
string messageId, string source, bool includeScreenShot)
{
var files = new List<MessageFileModel>();
try
{
if (!_imageTypes.Contains(contentType) && includeScreenShot)
{
var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty())
{
foreach (var screenShot in Directory.GetFiles(screenShotDir))
{
contentType = FileUtility.GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = screenShot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = FileUtility.GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
}
else
{
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = file,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
@ -365,5 +278,62 @@ public partial class LocalFileStorageService
var converters = _services.GetServices<IPdf2ImageConverter>();
return converters.FirstOrDefault();
}
private async Task<IEnumerable<MessageFileModel>> GetScreenshots(string file, string parentDir, string messageId, string source)
{
var files = new List<MessageFileModel>();
try
{
var contentType = FileUtility.GetFileContentType(file);
var screenshotDir = Path.Combine(parentDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty())
{
foreach (var screenshot in Directory.GetFiles(screenshotDir))
{
var fileName = Path.GetFileNameWithoutExtension(screenshot);
var fileExtension = Path.GetExtension(screenshot).Substring(1);
var screenshotContentType = FileUtility.GetFileContentType(screenshot);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileStorageUrl = screenshot,
ContentType = screenshotContentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenshotDir);
foreach (var image in images)
{
var fileName = Path.GetFileNameWithoutExtension(image);
var fileExtension = Path.GetExtension(image).Substring(1);
var screenshotContentType = FileUtility.GetFileContentType(image);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileStorageUrl = image,
ContentType = screenshotContentType,
FileSource = source
};
files.Add(model);
}
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
#endregion
}

0
src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs
View file

5
src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -9,11 +9,6 @@ public partial class LocalFileStorageService : IFileStorageService
private readonly IUserIdentity _user;
private readonly ILogger<LocalFileStorageService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _imageTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
};
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";

6
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs
View file

@ -10,8 +10,8 @@ public class MessageFileViewModel
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("file_extension")]
public string FileExtension { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
@ -30,7 +30,7 @@ public class MessageFileViewModel
{
FileUrl = model.FileUrl,
FileName = model.FileName,
FileType = model.FileType,
FileExtension = model.FileExtension,
ContentType = model.ContentType,
FileSource = model.FileSource
};

19
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -198,6 +198,7 @@ public class ChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
@ -262,13 +263,7 @@ public class ChatCompletionProvider : IChatCompletion
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileData))
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
@ -277,8 +272,14 @@ public class ChatCompletionProvider : IChatCompletion
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
}

4
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -1,8 +1,6 @@
using BotSharp.Abstraction.Files.Utilities;
using MailKit.Net.Smtp;
using MailKit.Security;
using MimeKit;
using System.IO;
namespace BotSharp.Plugin.EmailHandler.Functions;
@ -92,7 +90,7 @@ public class HandleEmailSenderFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl);
builder.Attachments.Add($"{file.FileName}.{file.FileType}", fileBytes, ContentType.Parse(file.ContentType));
builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType));
Thread.Sleep(100);
}
}

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -87,7 +87,7 @@ public class EditImageFn : IFunctionCallback
stream.Close();
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted.";
return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted.";
}
catch (Exception ex)
{

17
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<ReadImageFn> _logger;
private readonly IEnumerable<string> _imageContentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg,
};
public ReadImageFn(
IServiceProvider services,
ILogger<ReadImageFn> logger)
@ -29,7 +23,7 @@ public class ReadImageFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
{
@ -44,7 +38,7 @@ public class ReadImageFn : IFunctionCallback
return true;
}
private async Task<List<RoleDialogModel>> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
@ -52,7 +46,12 @@ public class ReadImageFn : IFunctionCallback
}
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var images = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes);
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
});
foreach (var dialog in dialogs)
{

7
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -51,11 +51,14 @@ public class ReadPdfFn : IFunctionCallback
}
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var files = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true);
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds);
if (screenshots.IsNullOrEmpty()) return dialogs;
foreach (var dialog in dialogs)
{
var found = files.Where(x => x.MessageId == dialog.MessageId).ToList();
var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile

19
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -199,6 +199,7 @@ public class ChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
@ -263,13 +264,7 @@ public class ChatCompletionProvider : IChatCompletion
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileData))
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
@ -278,8 +273,14 @@ public class ChatCompletionProvider : IChatCompletion
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
}

5
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs
View file

@ -7,12 +7,11 @@ public partial class TencentCosService
return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/";
}
public byte[] GetFileBytes(string filePath)
public byte[] GetFileBytes(string fileStorageUrl)
{
try
{
var fileData = _cosClient.BucketClient.DownloadFileBytes(filePath);
return fileData;
return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl);
}
catch (Exception ex)
{

170
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -7,38 +7,28 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
{
return files;
}
var messageIds = GetMessageIds(dialogs, offset);
var source = FileSourceType.User;
var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}";
foreach (var messageId in messageIds)
{
var dir = $"{pathPrefix}/{messageId}/{source}";
foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir))
{
var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var screenshots = await GetScreenshots(file, subDir, messageId, source);
if (screenshots.IsNullOrEmpty()) continue;
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
files.AddRange(foundFiles);
files.AddRange(screenshots);
}
}
@ -70,14 +60,14 @@ public partial class TencentCosService
}
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var fileExtension = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileUrl = BuilFileUrl(file),
FileStorageUrl = file,
FileName = fileName,
FileType = fileType,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
};
@ -89,12 +79,13 @@ public partial class TencentCosService
return files;
}
public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName)
{
var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/";
var fileList = _cosClient.BucketClient.GetDirFiles(dir);
var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName));
return found;
}
@ -246,88 +237,6 @@ public partial class TencentCosService
}
private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, string fileDir, string contentType,
string messageId, string source, bool includeScreenShot)
{
var files = new List<MessageFileModel>();
try
{
if (!_imageTypes.Contains(contentType) && includeScreenShot)
{
var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/";
var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir);
if (!fileList.IsNullOrEmpty())
{
foreach (var screenShot in fileList)
{
contentType = FileUtility.GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileUrl = BuilFileUrl(screenShot),
FileStorageUrl = screenShot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = FileUtility.GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileUrl = BuilFileUrl(image),
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
}
else
{
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileUrl = BuilFileUrl(file),
FileStorageUrl = file,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
@ -351,5 +260,64 @@ public partial class TencentCosService
{
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}";
}
private async Task<IEnumerable<MessageFileModel>> GetScreenshots(string file, string parentDir, string messageId, string source)
{
var files = new List<MessageFileModel>();
try
{
var contentType = FileUtility.GetFileContentType(file);
var screenshotDir = $"{parentDir}/{SCREENSHOT_FILE_FOLDER}/";
var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir);
if (!screenshots.IsNullOrEmpty())
{
foreach (var screenshot in screenshots)
{
var screenshotContentType = FileUtility.GetFileContentType(screenshot);
var fileName = Path.GetFileNameWithoutExtension(screenshot);
var fileExtension = Path.GetExtension(screenshot).Substring(1);
var model = new MessageFileModel
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileUrl = BuilFileUrl(screenshot),
FileStorageUrl = screenshot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenshotDir);
foreach (var image in images)
{
var fileName = Path.GetFileNameWithoutExtension(image);
var fileExtension = Path.GetExtension(image).Substring(1);
var screenshotContentType = FileUtility.GetFileContentType(image);
var model = new MessageFileModel
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileUrl = BuilFileUrl(image),
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
#endregion
}

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -40,7 +40,7 @@ public partial class TencentCosService : IFileStorageService
_user = user;
_logger = logger;
_services = services;
_fullBuketName = $"{_settings.BucketName}-{_settings.AppId}";
_fullBuketName = $"{settings.BucketName}-{settings.AppId}";
_cosClient = cosClient;
}
}