From 6efe291526cf0e7b8a747ae25a0821e7846aebbe Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 12 Dec 2024 17:20:27 -0600 Subject: [PATCH 1/2] allow image urls in image-reader --- .../Functions/ReadImageFn.cs | 18 +++++++++++++++--- .../LlmContexts/LlmContextIn.cs | 8 ++++++++ .../functions/util-file-read_image.json | 8 ++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index f2e79b21..9918d5bf 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -23,8 +23,8 @@ public class ReadImageFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); + var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs); + var agent = await agentService.LoadAgent(message.CurrentAgentId ?? BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { Id = agent?.Id ?? Guid.Empty.ToString(), @@ -38,7 +38,7 @@ public class ReadImageFn : IFunctionCallback return true; } - private List AssembleFiles(string conversationId, List dialogs) + private List AssembleFiles(string conversationId, IEnumerable? imageUrls, List dialogs) { if (dialogs.IsNullOrEmpty()) { @@ -66,6 +66,18 @@ public class ReadImageFn : IFunctionCallback }).ToList(); } + if (!imageUrls.IsNullOrEmpty()) + { + var lastDialog = dialogs.Last(); + var files = lastDialog.Files ?? []; + var addnFiles = imageUrls.Select(x => x?.Trim()) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + + files.AddRange(addnFiles); + lastDialog.Files = files; + } + return dialogs; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs index 9917cb74..a35e83bd 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs @@ -11,4 +11,12 @@ public class LlmContextIn [JsonPropertyName("image_description")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ImageDescription { get; set; } + + //[JsonPropertyName("image_url")] + //[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + //public string? ImageUrl { get; set; } + + [JsonPropertyName("image_urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? ImageUrls { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json index a0f269a0..b8dda8b1 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json @@ -7,6 +7,14 @@ "user_request": { "type": "string", "description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time." + }, + "image_urls": { + "type": "array", + "description": "The image, photo or picture urls that user requests for analysis. They typically start with 'http' or 'https'. If user doesn't include any url, then leave this array empty. Please remove any duplicated urls", + "items": { + "type": "string", + "description": "The image, photo or picture url that user requests for analysis. It typically starts with http or https." + } } }, "required": [ "user_request" ] From 51a319a3bc3e3032e68ddb13e41168bbc7fe1ae1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 12 Dec 2024 17:22:11 -0600 Subject: [PATCH 2/2] minor change --- .../BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 9918d5bf..e2c4f8c2 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -24,7 +24,8 @@ public class ReadImageFn : IFunctionCallback var wholeDialogs = conv.GetDialogHistory(); var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs); - var agent = await agentService.LoadAgent(message.CurrentAgentId ?? BuiltInAgentId.UtilityAssistant); + var agentId = !string.IsNullOrWhiteSpace(message.CurrentAgentId) ? message.CurrentAgentId : BuiltInAgentId.UtilityAssistant; + var agent = await agentService.LoadAgent(agentId); var fileAgent = new Agent { Id = agent?.Id ?? Guid.Empty.ToString(),