diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index f2e79b21..9918d5bf 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -23,8 +23,8 @@ public class ReadImageFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); + var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs); + var agent = await agentService.LoadAgent(message.CurrentAgentId ?? BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { Id = agent?.Id ?? Guid.Empty.ToString(), @@ -38,7 +38,7 @@ public class ReadImageFn : IFunctionCallback return true; } - private List AssembleFiles(string conversationId, List dialogs) + private List AssembleFiles(string conversationId, IEnumerable? imageUrls, List dialogs) { if (dialogs.IsNullOrEmpty()) { @@ -66,6 +66,18 @@ public class ReadImageFn : IFunctionCallback }).ToList(); } + if (!imageUrls.IsNullOrEmpty()) + { + var lastDialog = dialogs.Last(); + var files = lastDialog.Files ?? []; + var addnFiles = imageUrls.Select(x => x?.Trim()) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + + files.AddRange(addnFiles); + lastDialog.Files = files; + } + return dialogs; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs index 9917cb74..a35e83bd 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs @@ -11,4 +11,12 @@ public class LlmContextIn [JsonPropertyName("image_description")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ImageDescription { get; set; } + + //[JsonPropertyName("image_url")] + //[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + //public string? ImageUrl { get; set; } + + [JsonPropertyName("image_urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? ImageUrls { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json index a0f269a0..b8dda8b1 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json @@ -7,6 +7,14 @@ "user_request": { "type": "string", "description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time." + }, + "image_urls": { + "type": "array", + "description": "The image, photo or picture urls that user requests for analysis. They typically start with 'http' or 'https'. If user doesn't include any url, then leave this array empty. Please remove any duplicated urls", + "items": { + "type": "string", + "description": "The image, photo or picture url that user requests for analysis. It typically starts with http or https." + } } }, "required": [ "user_request" ]