From 0a0da61db394bfa8869e800c6b21e33c8dae5091 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 27 May 2025 21:23:20 -0500 Subject: [PATCH] temp save --- .../Files/Models/InstructFileModel.cs | 7 ---- .../Instruct/FileInstructService.Image.cs | 3 +- .../Instruct/FileInstructService.Pdf.cs | 41 ++++++++----------- .../Services/Instruct/FileInstructService.cs | 33 +++++++++------ .../Controllers/InstructModeController.cs | 27 ++++++------ .../Instructs/Request/InstructBaseRequest.cs | 3 ++ .../Providers/Chat/ChatCompletionProvider.cs | 2 +- 7 files changed, 55 insertions(+), 61 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs index e7372416..cacef5a0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -15,11 +15,4 @@ public class InstructFileModel : FileBase [JsonPropertyName("file_url")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileUrl { get; set; } = string.Empty; - - /// - /// File MIME type - /// - [JsonPropertyName("content_type")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ContentType { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 68b24f32..5d8deaa2 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -22,8 +22,7 @@ public partial class FileInstructService Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, - FileData = x.FileData, - ContentType = x.ContentType + FileData = x.FileData }).ToList() ?? [] } }); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 5e89f428..04e31092 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -85,38 +85,31 @@ public partial class FileInstructService return Enumerable.Empty(); } + var downloadTasks = files.Select(x => DownloadFile(x)); + await Task.WhenAll(downloadTasks); + var locs = new List(); - foreach (var file in files) + for (int i = 0; i < files.Count; i++) { + var binary = downloadTasks.ElementAt(i).Result; + if (binary == null || binary.IsEmpty) + { + continue; + } + try { - var binary = BinaryData.Empty; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - var bytes = await client.GetByteArrayAsync(file.FileUrl); - binary = BinaryData.FromBytes(bytes); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); - } + var guid = Guid.NewGuid().ToString(); + var fileDir = _fileStorage.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir, createNew: true); - if (!binary.IsEmpty) - { - var guid = Guid.NewGuid().ToString(); - var fileDir = _fileStorage.BuildDirectory(dir, guid); - DeleteIfExistDirectory(fileDir, createNew: true); - - var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileStorage.SaveFileBytesToPath(outputDir, binary); - locs.Add(outputDir); - } + var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(outputDir, binary); + locs.Add(outputDir); } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when saving {extension} file."); + _logger.LogWarning(ex, $"Error when saving #{i + 1} {extension} file."); continue; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 98321347..4a0729d0 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -35,19 +35,28 @@ public partial class FileInstructService : IFileInstructService private async Task DownloadFile(InstructFileModel file) { var binary = BinaryData.Empty; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - var bytes = await client.GetByteArrayAsync(file.FileUrl); - binary = BinaryData.FromBytes(bytes); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); - } - return binary; + try + { + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + var bytes = await client.GetByteArrayAsync(file.FileUrl); + binary = BinaryData.FromBytes(bytes); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); + } + + return binary; + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Error when downloading file {file.FileUrl}"); + return binary; + } } private async Task GetAgentTemplate(string agentId, string? templateName) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 71c67e7e..d7f31549 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -115,7 +115,7 @@ public class InstructModeController : ControllerBase #region Read image [HttpPost("/instruct/multi-modal")] - public async Task MultiModalCompletion([FromBody] MultiModalRequest input) + public async Task MultiModalCompletion([FromBody] MultiModalFileRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -141,28 +141,25 @@ public class InstructModeController : ControllerBase } [HttpPost("/instruct/multi-modal/upload")] - public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, - [FromForm] string? agentId = null, [FromForm] string? templateName = null) + public async Task MultiModalCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var viewModel = new MultiModalViewModel(); try { - var data = FileUtility.BuildFileDataFromFile(file); - var files = new List + var fileModels = files.Select(x => new InstructFileModel { - new InstructFileModel { FileData = data } - }; + FileData = FileUtility.BuildFileDataFromFile(x) + }).ToList(); var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadImages(text, files, new InstructOptions + var content = await fileInstruct.ReadImages(request?.Text ?? string.Empty, fileModels, new InstructOptions { - Provider = provider, - Model = model, - AgentId = agentId, - TemplateName = templateName + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName }); viewModel.Content = content; return viewModel; @@ -424,7 +421,7 @@ public class InstructModeController : ControllerBase #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] MultiModalRequest input) + public async Task PdfCompletion([FromBody] MultiModalFileRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 80368a70..cc3c11bd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -24,7 +24,10 @@ public class MultiModalRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; +} +public class MultiModalFileRequest : MultiModalRequest +{ [JsonPropertyName("files")] public List Files { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index bfec9e3e..d4082393 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -289,7 +289,7 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType ?? file.ContentType, ChatImageDetailLevel.Auto); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(binary.ToArray()), contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl))