SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #796 from iceljc/master

Browse source 
allow image urls in image-reader
This commit is contained in:
iceljc 2024-12-12 17:22:36 -06:00 committed by GitHub
parent dc8715320b 51a319a3bc
commit bf6daa0e6d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 32 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -23,8 +23,9 @@ public class ReadImageFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs);
var agentId = !string.IsNullOrWhiteSpace(message.CurrentAgentId) ? message.CurrentAgentId : BuiltInAgentId.UtilityAssistant;
var agent = await agentService.LoadAgent(agentId);
var fileAgent = new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
@ -38,7 +39,7 @@ public class ReadImageFn : IFunctionCallback
return true;
}
private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
private List<RoleDialogModel> AssembleFiles(string conversationId, IEnumerable<string>? imageUrls, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
@ -66,6 +67,18 @@ public class ReadImageFn : IFunctionCallback
}).ToList();
}
if (!imageUrls.IsNullOrEmpty())
{
var lastDialog = dialogs.Last();
var files = lastDialog.Files ?? [];
var addnFiles = imageUrls.Select(x => x?.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => new BotSharpFile { FileUrl = x }).ToList();
files.AddRange(addnFiles);
lastDialog.Files = files;
}
return dialogs;
}

8
src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs
View file

@ -11,4 +11,12 @@ public class LlmContextIn
[JsonPropertyName("image_description")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ImageDescription { get; set; }
//[JsonPropertyName("image_url")]
//[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
//public string? ImageUrl { get; set; }
[JsonPropertyName("image_urls")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<string>? ImageUrls { get; set; }
}

8
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json
View file

@ -7,6 +7,14 @@
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested images. User can request for multiple images to process at one time."
},
"image_urls": {
"type": "array",
"description": "The image, photo or picture urls that user requests for analysis. They typically start with 'http' or 'https'. If user doesn't include any url, then leave this array empty. Please remove any duplicated urls",
"items": {
"type": "string",
"description": "The image, photo or picture url that user requests for analysis. It typically starts with http or https."
}
}
},
"required": [ "user_request" ]