diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 8e255e76..ef40e11c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -7,9 +7,9 @@ public interface IFileStorageService #region Common string GetDirectory(string conversationId); IEnumerable GetFiles(string relativePath, string? searchQuery = null); - byte[] GetFileBytes(string fileStorageUrl); + BinaryData GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); - bool SaveFileBytesToPath(string filePath, byte[] bytes); + bool SaveFileBytesToPath(string filePath, BinaryData binary); string GetParentDir(string dir, int level = 1); bool ExistDirectory(string? dir); void CreateDirectory(string dir); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs index 7eaddd5a..e7372416 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -10,9 +10,16 @@ public class InstructFileModel : FileBase public string? FileExtension { get; set; } = string.Empty; /// - /// External file url + /// File url /// [JsonPropertyName("file_url")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileUrl { get; set; } = string.Empty; + + /// + /// File MIME type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index db9ccb65..d7867cf6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -11,11 +11,16 @@ public static class FileUtility /// /// /// - public static (string, byte[]) GetFileInfoFromData(string data) + public static (string?, BinaryData) GetFileInfoFromData(string data) { if (string.IsNullOrEmpty(data)) { - return (string.Empty, new byte[0]); + return (null, BinaryData.Empty); + } + + if (!data.StartsWith("data:")) + { + return (null, BinaryData.FromString(data)); } var typeStartIdx = data.IndexOf(':'); @@ -25,13 +30,13 @@ public static class FileUtility var base64startIdx = data.IndexOf(','); var base64Str = data.Substring(base64startIdx + 1); - return (contentType, Convert.FromBase64String(base64Str)); + return (contentType, BinaryData.FromString(base64Str)); } - public static string BuildFileDataFromFile(string fileName, byte[] bytes) + public static string BuildFileDataFromFile(string fileName, BinaryData binary) { var contentType = GetFileContentType(fileName); - var base64 = Convert.ToBase64String(bytes); + var base64 = Convert.ToBase64String(binary.ToArray()); return $"data:{contentType};base64,{base64}"; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index d61b6165..130baa19 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -14,9 +14,8 @@ public partial class FileInstructService } var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model); - var audioBytes = await DownloadFile(audio); - using var stream = new MemoryStream(); - stream.Write(audioBytes, 0, audioBytes.Length); + var audioBinary = await DownloadFile(audio); + using var stream = audioBinary.ToStream(); stream.Position = 0; var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index dd2e02d4..68b24f32 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,7 +1,5 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; -using System.IO; -using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Files.Services; @@ -21,7 +19,12 @@ public partial class FileInstructService { new RoleDialogModel(AgentRole.User, text) { - Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? [] + Files = images?.Select(x => new BotSharpFile + { + FileUrl = x.FileUrl, + FileData = x.FileData, + ContentType = x.ContentType + }).ToList() ?? [] } }); @@ -76,9 +79,8 @@ public partial class FileInstructService var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); + var binary = await DownloadFile(image); + using var stream = binary.ToStream(); stream.Position = 0; var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; @@ -113,9 +115,8 @@ public partial class FileInstructService var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); + var binary = await DownloadFile(image); + using var stream = binary.ToStream(); stream.Position = 0; var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; @@ -153,15 +154,13 @@ public partial class FileInstructService var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); - var imageBytes = await DownloadFile(image); - var maskBytes = await DownloadFile(mask); + var imageBinary = await DownloadFile(image); + var maskBinary = await DownloadFile(mask); - using var imageStream = new MemoryStream(); - imageStream.Write(imageBytes, 0, imageBytes.Length); + using var imageStream = imageBinary.ToStream(); imageStream.Position = 0; - using var maskStream = new MemoryStream(); - maskStream.Write(maskBytes, 0, maskBytes.Length); + using var maskStream = maskBinary.ToStream(); maskStream.Position = 0; var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index b593e949..5e89f428 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -22,7 +22,7 @@ public partial class FileInstructService try { var provider = options?.Provider ?? "openai"; - var pdfFiles = await DownloadFiles(sessionDir, files); + var pdfFiles = await DownloadAndSaveFiles(sessionDir, files); var targetFiles = pdfFiles; if (provider != "google-ai") @@ -78,7 +78,7 @@ public partial class FileInstructService } #region Private methods - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadAndSaveFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { @@ -90,32 +90,33 @@ public partial class FileInstructService { try { - var bytes = new byte[0]; + var binary = BinaryData.Empty; if (!string.IsNullOrEmpty(file.FileUrl)) { var http = _services.GetRequiredService(); using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); + var bytes = await client.GetByteArrayAsync(file.FileUrl); + binary = BinaryData.FromBytes(bytes); } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); } - if (!bytes.IsNullOrEmpty()) + if (!binary.IsEmpty) { var guid = Guid.NewGuid().ToString(); var fileDir = _fileStorage.BuildDirectory(dir, guid); - DeleteIfExistDirectory(fileDir, true); + DeleteIfExistDirectory(fileDir, createNew: true); var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileStorage.SaveFileBytesToPath(outputDir, bytes); + _fileStorage.SaveFileBytesToPath(outputDir, binary); locs.Add(outputDir); } } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when saving pdf file."); + _logger.LogWarning(ex, $"Error when saving {extension} file."); continue; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 9ff3b46a..98321347 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -32,21 +32,22 @@ public partial class FileInstructService : IFileInstructService } } - private async Task DownloadFile(InstructFileModel file) + private async Task DownloadFile(InstructFileModel file) { - var bytes = new byte[0]; + var binary = BinaryData.Empty; if (!string.IsNullOrEmpty(file.FileUrl)) { var http = _services.GetRequiredService(); using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); + var bytes = await client.GetByteArrayAsync(file.FileUrl); + binary = BinaryData.FromBytes(bytes); } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); } - return bytes; + return binary; } private async Task GetAgentTemplate(string agentId, string? templateName) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index b16c2c2c..80a10dee 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -30,12 +30,12 @@ public partial class LocalFileStorageService return Directory.GetFiles(path); } - public byte[] GetFileBytes(string fileStorageUrl) + public BinaryData GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); - return bytes; + return BinaryData.FromBytes(bytes); } public bool SaveFileStreamToPath(string filePath, Stream stream) @@ -49,11 +49,11 @@ public partial class LocalFileStorageService return true; } - public bool SaveFileBytesToPath(string filePath, byte[] bytes) + public bool SaveFileBytesToPath(string filePath, BinaryData binary) { using (var fs = new FileStream(filePath, FileMode.Create)) { - fs.Write(bytes, 0, bytes.Length); + fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(); fs.Close(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index c1accd30..9aac4df3 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -136,7 +136,7 @@ public partial class LocalFileStorageService try { - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { @@ -145,7 +145,7 @@ public partial class LocalFileStorageService using (var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.Create)) { - fs.Write(bytes, 0, bytes.Length); + fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(true); fs.Close(); Thread.Sleep(100); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index 21f522b2..9384bd83 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -34,8 +34,8 @@ public partial class LocalFileStorageService } dir = GetUserAvatarDir(user?.Id, createNewDir: true); - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); + File.WriteAllBytes(Path.Combine(dir, file.FileName), binary.ToArray()); return true; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 430a4563..71c67e7e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -135,7 +135,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in reading images. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); return error; } } @@ -170,7 +170,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in reading image upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -202,7 +202,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image generation. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -239,7 +239,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image variation. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -272,7 +272,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image variation upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -306,7 +306,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image edit. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -341,7 +341,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image edit upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -377,7 +377,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image mask edit. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -415,7 +415,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in image mask edit upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -446,7 +446,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in pdf completion. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -483,7 +483,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in pdf completion upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -519,7 +519,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in speech to text. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -553,7 +553,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { var error = $"Error in speech-to-text upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 88a10838..92e36f46 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -245,8 +245,8 @@ public class UserController : ControllerBase private FileContentResult BuildFileResult(string file) { var fileStorage = _services.GetRequiredService(); - var bytes = fileStorage.GetFileBytes(file); - return File(bytes, "application/octet-stream", Path.GetFileName(file)); + var binary = fileStorage.GetFileBytes(file); + return File(binary.ToArray(), "application/octet-stream", Path.GetFileName(file)); } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index f20a675a..a114c2fa 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -87,8 +87,8 @@ public class HandleAudioRequestFn : IFunctionCallback var fileName = Path.GetFileName(file.FileStorageUrl); if (!ParseAudioFileType(fileName)) continue; - var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); - using var stream = new MemoryStream(bytes); + var binary = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = binary.ToStream(); stream.Position = 0; var result = await audioCompletion.TranscriptTextAsync(stream, fileName); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 5c8a12d1..8e4a59fe 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -89,8 +89,8 @@ public class NativeWhisperProvider : IAudioTranscription DownloadModel(modelType, modelLoc); } - var bytes = _fileStorage.GetFileBytes(modelLoc); - _whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build(); + var binary = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(binary.ToArray()).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 114ea69d..b2eaaf46 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -322,15 +322,15 @@ public class ChatCompletionProvider : IChatCompletion { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index df46cdfe..35f817f4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -90,8 +90,8 @@ public class HandleEmailSenderFn : IFunctionCallback { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); - builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); + var fileBinary = fileStorage.GetFileBytes(file.FileStorageUrl); + builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBinary.ToArray(), ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs index 270f6177..b65401fd 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs @@ -113,8 +113,8 @@ public class HandleExcelRequestFn : IFunctionCallback _currentFileName = Path.GetFileName(file.FileStorageUrl); - var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); - var workbook = ConvertToWorkBook(bytes); + var binary = _fileStorage.GetFileBytes(file.FileStorageUrl); + var workbook = ConvertToWorkBook(binary.ToArray()); var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook); sqlCommandList.AddRange(currentCommandList); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 2895f152..aed98fea 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -77,9 +77,8 @@ public class EditImageFn : IFunctionCallback }; var fileStorage = _services.GetRequiredService(); - var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl); - using var stream = new MemoryStream(); - stream.Write(fileBytes); + var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + using var stream = fileBinary.ToStream(); stream.Position = 0; var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); stream.Close(); diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 9a23d817..a0e7cc17 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -307,26 +307,26 @@ public class GeminiChatCompletionProvider : IChatCompletion { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); contentParts.Add(new Part() { InlineData = new() { - MimeType = contentType, - Data = Convert.ToBase64String(bytes) + MimeType = contentType ?? file.ContentType, + Data = Convert.ToBase64String(binary.ToArray()) } }); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); contentParts.Add(new Part() { InlineData = new() { MimeType = contentType, - Data = Convert.ToBase64String(bytes) + Data = Convert.ToBase64String(binary.ToArray()) } }); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index a2461f18..491261d8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -48,12 +48,12 @@ public partial class KnowledgeService try { // Get document info - var (contentType, bytes) = await GetFileInfo(file); - var contents = await GetFileContent(contentType, bytes, option ?? ChunkOption.Default()); + var (contentType, binary) = await GetFileInfo(file); + var contents = await GetFileContent(contentType, binary, option ?? ChunkOption.Default()); // Save document var fileId = Guid.NewGuid(); - var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes); + var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, binary); if (!saved) { failedFiles.Add(file.FileName); @@ -342,11 +342,11 @@ public partial class KnowledgeService /// /// /// - private async Task<(string, byte[])> GetFileInfo(ExternalFileModel file) + private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file) { if (file == null) { - return (string.Empty, new byte[0]); + return (string.Empty, BinaryData.Empty); } if (!string.IsNullOrWhiteSpace(file.FileUrl)) @@ -355,37 +355,38 @@ public partial class KnowledgeService var contentType = FileUtility.GetFileContentType(file.FileName); using var client = http.CreateClient(); var bytes = await client.GetByteArrayAsync(file.FileUrl); - return (contentType, bytes); + return (contentType, BinaryData.FromBytes(bytes)); } else if (!string.IsNullOrWhiteSpace(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - return (contentType, bytes); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + return (contentType, binary); } - return (string.Empty, new byte[0]); + return (string.Empty, BinaryData.Empty); } #region Read doc content - private async Task> GetFileContent(string contentType, byte[] bytes, ChunkOption option) + private async Task> GetFileContent(string contentType, BinaryData binary, ChunkOption option) { IEnumerable results = new List(); if (contentType.IsEqualTo(MediaTypeNames.Text.Plain)) { - results = await ReadTxt(bytes, option); + results = await ReadTxt(binary, option); } else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf)) { - results = await ReadPdf(bytes); + results = await ReadPdf(binary); } return results; } - private async Task> ReadTxt(byte[] bytes, ChunkOption option) + private async Task> ReadTxt(BinaryData binary, ChunkOption option) { - using var stream = new MemoryStream(bytes); + using var stream = binary.ToStream(); + stream.Position = 0; using var reader = new StreamReader(stream); var content = await reader.ReadToEndAsync(); reader.Close(); @@ -395,18 +396,17 @@ public partial class KnowledgeService return lines; } - private async Task> ReadPdf(byte[] bytes) + private async Task> ReadPdf(BinaryData binary) { return Enumerable.Empty(); } #endregion - private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes) + private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData binary) { var fileStoreage = _services.GetRequiredService(); - var data = BinaryData.FromBytes(bytes); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data); + var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, binary); return saved; } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index d32fbece..d95a25ae 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -128,8 +128,8 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage!.GetFileBytes(file.FileStorageUrl); - contents.Add(new DataContent(bytes, contentType)); + var binary = fileStorage!.GetFileBytes(file.FileStorageUrl); + contents.Add(new DataContent(binary.ToMemory(), contentType)); } else if (!string.IsNullOrEmpty(file.FileUrl)) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index d9591a0a..bfec9e3e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -288,15 +288,15 @@ public class ChatCompletionProvider : IChatCompletion { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType ?? file.ContentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index fd458744..8219bf1c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -632,15 +632,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index 0659b0c8..fbc0a004 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -25,16 +25,17 @@ public partial class TencentCosService } } - public byte[] GetFileBytes(string fileStorageUrl) + public BinaryData GetFileBytes(string fileStorageUrl) { try { - return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); + var bytes = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); + return BinaryData.FromBytes(bytes); } catch (Exception ex) { _logger.LogWarning(ex, $"Error when getting file bytes (url: {fileStorageUrl})."); - return Array.Empty(); + return BinaryData.Empty; } } @@ -53,13 +54,13 @@ public partial class TencentCosService } } - public bool SaveFileBytesToPath(string filePath, byte[] bytes) + public bool SaveFileBytesToPath(string filePath, BinaryData binary) { if (string.IsNullOrEmpty(filePath)) return false; try { - return _cosClient.BucketClient.UploadBytes(filePath, bytes); + return _cosClient.BucketClient.UploadBytes(filePath, binary.ToArray()); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 4afec393..dfd6781f 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -132,9 +132,9 @@ public partial class TencentCosService try { - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = $"{dir}/{source}/{i + 1}"; - _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes); + _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", binary.ToArray()); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 72076528..9d65d00f 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -28,11 +28,11 @@ public partial class TencentCosService if (string.IsNullOrEmpty(dir)) return false; - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); var extension = Path.GetExtension(file.FileName); var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; - return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); + return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", binary.ToArray()); } catch (Exception ex) { diff --git a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs index 23514e4a..bb1d0c44 100644 --- a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs +++ b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs @@ -1,9 +1,9 @@ -using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Models; namespace BotSharp.Plugin.Google.Core { - public class NullFileStorageService:IFileStorageService + public class NullFileStorageService : IFileStorageService { public string GetDirectory(string conversationId) { @@ -15,9 +15,10 @@ namespace BotSharp.Plugin.Google.Core return new List { "FakeFile1.txt", "FakeFile2.txt" }; } - public byte[] GetFileBytes(string fileStorageUrl) + public BinaryData GetFileBytes(string fileStorageUrl) { - return new byte[] { 0x00, 0x01, 0x02 }; + var bytes = new byte[] { 0x00, 0x01, 0x02 }; + return BinaryData.FromBytes(bytes); } public bool SaveFileStreamToPath(string filePath, Stream stream) @@ -25,7 +26,7 @@ namespace BotSharp.Plugin.Google.Core return true; } - public bool SaveFileBytesToPath(string filePath, byte[] bytes) + public bool SaveFileBytesToPath(string filePath, BinaryData binary) { return true; }