SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

allow image urls in image-reader

Browse source
This commit is contained in:
Jicheng Lu 2024-12-12 17:20:27 -06:00
parent dc8715320b
commit 6efe291526
3 changed files with 31 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -23,8 +23,8 @@ public class ReadImageFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs);
var agent = await agentService.LoadAgent(message.CurrentAgentId ?? BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
@ -38,7 +38,7 @@ public class ReadImageFn : IFunctionCallback
return true;
}
private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
private List<RoleDialogModel> AssembleFiles(string conversationId, IEnumerable<string>? imageUrls, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
@ -66,6 +66,18 @@ public class ReadImageFn : IFunctionCallback
}).ToList();
}
if (!imageUrls.IsNullOrEmpty())
{
var lastDialog = dialogs.Last();
var files = lastDialog.Files ?? [];
var addnFiles = imageUrls.Select(x => x?.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => new BotSharpFile { FileUrl = x }).ToList();
files.AddRange(addnFiles);
lastDialog.Files = files;
}
return dialogs;
}

8
src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs
View file

@ -11,4 +11,12 @@ public class LlmContextIn
[JsonPropertyName("image_description")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ImageDescription { get; set; }
//[JsonPropertyName("image_url")]
//[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
//public string? ImageUrl { get; set; }
[JsonPropertyName("image_urls")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<string>? ImageUrls { get; set; }
}

8
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json
View file

@ -7,6 +7,14 @@
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time."
},
"image_urls": {
"type": "array",
"description": "The image, photo or picture urls that user requests for analysis. They typically start with 'http' or 'https'. If user doesn't include any url, then leave this array empty. Please remove any duplicated urls",
"items": {
"type": "string",
"description": "The image, photo or picture url that user requests for analysis. It typically starts with http or https."
}
}
},
"required": [ "user_request" ]