diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs
index e7372416..cacef5a0 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs
@@ -15,11 +15,4 @@ public class InstructFileModel : FileBase
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
-
- ///
- /// File MIME type
- ///
- [JsonPropertyName("content_type")]
- [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
- public string? ContentType { get; set; }
}
diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
index 68b24f32..5d8deaa2 100644
--- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
+++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
@@ -22,8 +22,7 @@ public partial class FileInstructService
Files = images?.Select(x => new BotSharpFile
{
FileUrl = x.FileUrl,
- FileData = x.FileData,
- ContentType = x.ContentType
+ FileData = x.FileData
}).ToList() ?? []
}
});
diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
index 5e89f428..04e31092 100644
--- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
+++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
@@ -85,38 +85,31 @@ public partial class FileInstructService
return Enumerable.Empty();
}
+ var downloadTasks = files.Select(x => DownloadFile(x));
+ await Task.WhenAll(downloadTasks);
+
var locs = new List();
- foreach (var file in files)
+ for (int i = 0; i < files.Count; i++)
{
+ var binary = downloadTasks.ElementAt(i).Result;
+ if (binary == null || binary.IsEmpty)
+ {
+ continue;
+ }
+
try
{
- var binary = BinaryData.Empty;
- if (!string.IsNullOrEmpty(file.FileUrl))
- {
- var http = _services.GetRequiredService();
- using var client = http.CreateClient();
- var bytes = await client.GetByteArrayAsync(file.FileUrl);
- binary = BinaryData.FromBytes(bytes);
- }
- else if (!string.IsNullOrEmpty(file.FileData))
- {
- (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
- }
+ var guid = Guid.NewGuid().ToString();
+ var fileDir = _fileStorage.BuildDirectory(dir, guid);
+ DeleteIfExistDirectory(fileDir, createNew: true);
- if (!binary.IsEmpty)
- {
- var guid = Guid.NewGuid().ToString();
- var fileDir = _fileStorage.BuildDirectory(dir, guid);
- DeleteIfExistDirectory(fileDir, createNew: true);
-
- var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
- _fileStorage.SaveFileBytesToPath(outputDir, binary);
- locs.Add(outputDir);
- }
+ var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
+ _fileStorage.SaveFileBytesToPath(outputDir, binary);
+ locs.Add(outputDir);
}
catch (Exception ex)
{
- _logger.LogWarning(ex, $"Error when saving {extension} file.");
+ _logger.LogWarning(ex, $"Error when saving #{i + 1} {extension} file.");
continue;
}
}
diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
index 98321347..4a0729d0 100644
--- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
+++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
@@ -35,19 +35,28 @@ public partial class FileInstructService : IFileInstructService
private async Task DownloadFile(InstructFileModel file)
{
var binary = BinaryData.Empty;
- if (!string.IsNullOrEmpty(file.FileUrl))
- {
- var http = _services.GetRequiredService();
- using var client = http.CreateClient();
- var bytes = await client.GetByteArrayAsync(file.FileUrl);
- binary = BinaryData.FromBytes(bytes);
- }
- else if (!string.IsNullOrEmpty(file.FileData))
- {
- (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
- }
- return binary;
+ try
+ {
+ if (!string.IsNullOrEmpty(file.FileUrl))
+ {
+ var http = _services.GetRequiredService();
+ using var client = http.CreateClient();
+ var bytes = await client.GetByteArrayAsync(file.FileUrl);
+ binary = BinaryData.FromBytes(bytes);
+ }
+ else if (!string.IsNullOrEmpty(file.FileData))
+ {
+ (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
+ }
+
+ return binary;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, $"Error when downloading file {file.FileUrl}");
+ return binary;
+ }
}
private async Task GetAgentTemplate(string agentId, string? templateName)
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
index 71c67e7e..d7f31549 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
@@ -115,7 +115,7 @@ public class InstructModeController : ControllerBase
#region Read image
[HttpPost("/instruct/multi-modal")]
- public async Task MultiModalCompletion([FromBody] MultiModalRequest input)
+ public async Task MultiModalCompletion([FromBody] MultiModalFileRequest input)
{
var state = _services.GetRequiredService();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
@@ -141,28 +141,25 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/multi-modal/upload")]
- public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
- [FromForm] string? model = null, [FromForm] List? states = null,
- [FromForm] string? agentId = null, [FromForm] string? templateName = null)
+ public async Task MultiModalCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request)
{
var state = _services.GetRequiredService();
- states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
+ request?.States?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var viewModel = new MultiModalViewModel();
try
{
- var data = FileUtility.BuildFileDataFromFile(file);
- var files = new List
+ var fileModels = files.Select(x => new InstructFileModel
{
- new InstructFileModel { FileData = data }
- };
+ FileData = FileUtility.BuildFileDataFromFile(x)
+ }).ToList();
var fileInstruct = _services.GetRequiredService();
- var content = await fileInstruct.ReadImages(text, files, new InstructOptions
+ var content = await fileInstruct.ReadImages(request?.Text ?? string.Empty, fileModels, new InstructOptions
{
- Provider = provider,
- Model = model,
- AgentId = agentId,
- TemplateName = templateName
+ Provider = request?.Provider,
+ Model = request?.Model,
+ AgentId = request?.AgentId,
+ TemplateName = request?.TemplateName
});
viewModel.Content = content;
return viewModel;
@@ -424,7 +421,7 @@ public class InstructModeController : ControllerBase
#region Pdf
[HttpPost("/instruct/pdf-completion")]
- public async Task PdfCompletion([FromBody] MultiModalRequest input)
+ public async Task PdfCompletion([FromBody] MultiModalFileRequest input)
{
var state = _services.GetRequiredService();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
index 80368a70..cc3c11bd 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
@@ -24,7 +24,10 @@ public class MultiModalRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
+}
+public class MultiModalFileRequest : MultiModalRequest
+{
[JsonPropertyName("files")]
public List Files { get; set; } = [];
}
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
index bfec9e3e..d4082393 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
@@ -289,7 +289,7 @@ public class ChatCompletionProvider : IChatCompletion
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
- var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType ?? file.ContentType, ChatImageDetailLevel.Auto);
+ var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(binary.ToArray()), contentType, ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))