SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine file select

Browse source
This commit is contained in:
Jicheng Lu 2025-09-12 17:17:03 -05:00
parent 93e904d5f4
commit e14a83a381
16 changed files with 377 additions and 181 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs
View file

@ -10,6 +10,7 @@ public interface IImageConverter
/// <param name="pdfLocation">Pdf file location</param>
/// <param name="imageFolderLocation">Image folder location</param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
Task<IEnumerable<string>> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException();
/// <summary>

5
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -1,4 +1,3 @@
namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile : FileInformation
@ -9,4 +8,8 @@ public class BotSharpFile : FileInformation
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
[JsonPropertyName("file_index")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileIndex { get; set; } = string.Empty;
}

20
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs
View file

@ -2,7 +2,21 @@ namespace BotSharp.Abstraction.Files.Models;
public class FileSelectContext
{
[JsonPropertyName("selected_ids")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<int>? Selecteds { get; set; }
[JsonPropertyName("selected_files")]
public List<FileSelectItem>? SelectedFiles { get; set; }
}
public class FileSelectItem
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
[JsonPropertyName("file_index")]
public string FileIndex { get; set; }
[JsonPropertyName("file_source")]
public string FileSource { get; set; }
[JsonPropertyName("file_name")]
public string? FileName { get; set; }
}

3
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -8,6 +8,9 @@ public class MessageFileModel : FileInformation
[JsonPropertyName("file_source")]
public string FileSource { get; set; } = FileSourceType.User;
[JsonPropertyName("file_index")]
public string FileIndex { get; set; } = string.Empty;
public MessageFileModel()
{

26
src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs
View file

@ -12,6 +12,16 @@ public class SelectFileOptions
/// </summary>
public string? Model { get; set; }
/// <summary>
/// Llm maximum output tokens
/// </summary>
public int? MaxOutputTokens { get; set; }
/// <summary>
/// Llm reasoning effort level
/// </summary>
public string? ReasoningEffortLevel { get; set; }
/// <summary>
/// Agent id
/// </summary>
@ -30,7 +40,7 @@ public class SelectFileOptions
/// <summary>
/// Whether include bot generated files
/// </summary>
public bool IncludeBotFile { get; set; }
public bool IsIncludeBotFiles { get; set; }
/// <summary>
/// Conversation breakpoint
@ -38,12 +48,22 @@ public class SelectFileOptions
public bool FromBreakpoint { get; set; }
/// <summary>
/// Message offset from last
/// The maximum number of messages
/// </summary>
public int? Offset { get; set; }
public int? MessageLimit { get; set; }
/// <summary>
/// Whehter attach files to messages
/// </summary>
public bool IsAttachFiles { get; set; }
/// <summary>
/// File content types. If null, all types of files will be retrived
/// </summary>
public IEnumerable<string>? ContentTypes { get; set; }
/// <summary>
/// Data that can be used to fill in the prompt
/// </summary>
public Dictionary<string, object>? Data { get; set; }
}

192
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Templating;
namespace BotSharp.Core.Files.Services;
@ -12,12 +13,23 @@ public partial class FileInstructService
return Enumerable.Empty<MessageFileModel>();
}
var routeContext = _services.GetRequiredService<IRoutingContext>();
var convService = _services.GetRequiredService<IConversationService>();
var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint);
var messageIds = GetMessageIds(dialogs, options.Offset);
var dialogs = routeContext.GetDialogs();
if (dialogs.IsNullOrEmpty())
{
dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint);
}
if (options.MessageLimit > 0)
{
dialogs = dialogs.TakeLast(options.MessageLimit.Value).ToList();
}
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes);
if (options.IncludeBotFile)
if (options.IsIncludeBotFiles)
{
var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes);
files = MergeMessageFiles(messageIds, files, botFiles);
@ -39,8 +51,8 @@ public partial class FileInstructService
foreach (var messageId in messageIds)
{
var users = userFiles.Where(x => x.MessageId == messageId).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).ToList();
var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList();
if (!users.IsNullOrEmpty()) files.AddRange(users);
if (!bots.IsNullOrEmpty()) files.AddRange(bots);
@ -51,87 +63,175 @@ public partial class FileInstructService
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, IEnumerable<RoleDialogModel> dialogs, SelectFileOptions options)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var res = new List<MessageFileModel>();
if (files.IsNullOrEmpty())
{
return res;
}
var agentService = _services.GetRequiredService<IAgentService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
// Handle dialogs and files
var innerDialogs = (dialogs ?? []).ToList();
var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s).";
innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList();
if (options.IsAttachFiles)
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}";
AssembleMessageFiles(innerDialogs, files, options);
}
// Handle instruction
var promptMessages = innerDialogs.Select(x =>
{
var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}";
var fileDescs = x.Files?.Select((f, fidx) => $"message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " +
$"file_name: '{f.FileFullName}', content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'");
var desc = string.Empty;
if (!fileDescs.IsNullOrEmpty())
{
desc = $"[Files]: \r\n\t{string.Join("\r\n\t", fileDescs)}";
}
return new NameDesc(text, desc);
}).ToList();
var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant;
var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "util-file-select_file_instruction";
var prompt = db.GetAgentTemplate(agentId, template);
var foundAgent = db.GetAgent(agentId);
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template);
prompt = render.Render(prompt, new Dictionary<string, object>
var data = new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
{ "message_files", promptMessages }
};
if (!options.Data.IsNullOrEmpty())
{
foreach (var item in options.Data)
{
data[item.Key] = item.Value;
}
}
prompt = render.Render(prompt, data);
// Build agent
var foundAgent = await agentService.GetAgent(agentId);
var agent = new Agent
{
Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = foundAgent?.Name ?? "Utility Assistant",
Instruction = prompt
Instruction = prompt,
LlmConfig = new AgentLlmConfig
{
MaxOutputTokens = options.MaxOutputTokens,
ReasoningEffortLevel = options.ReasoningEffortLevel
}
};
var message = dialogs.LastOrDefault();
var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content;
if (message == null)
{
message = new RoleDialogModel(AgentRole.User, text);
}
else
{
message = RoleDialogModel.From(message, AgentRole.User, text);
}
var providerName = options.Provider ?? "openai";
var model = options?.Model ?? "gpt-4.1-mini";
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
// Get ai response
var provider = options.Provider ?? "openai";
var model = options?.Model ?? "gpt-5-mini";
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
var content = response?.Content ?? string.Empty;
var response = await completion.GetChatCompletions(agent, innerDialogs);
var content = response?.Content ?? "{}";
var selecteds = JsonSerializer.Deserialize<FileSelectContext>(content, new JsonSerializerOptions
{
AllowTrailingCommas = true
});
var fids = selecteds?.Selecteds ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
var selectedFiles = selecteds?.SelectedFiles ?? new List<FileSelectItem>();
if (!selectedFiles.IsNullOrEmpty())
{
res = files.Where(file => selectedFiles.Any(x => x.MessageId.IsEqualTo(file.MessageId)
&& x.FileIndex.IsEqualTo(file.FileIndex)
&& x.FileSource.IsEqualTo(file.FileSource))).ToList();
}
return res;
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when selecting files.");
return new List<MessageFileModel>();
return [];
}
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> conversations, int? offset = null)
private void AssembleMessageFiles(IEnumerable<RoleDialogModel> dialogs, IEnumerable<MessageFileModel> files, SelectFileOptions options)
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset.HasValue && offset < 1)
if (dialogs.IsNullOrEmpty() || files.IsNullOrEmpty())
{
offset = 1;
return;
}
var messageIds = new List<string>();
if (offset.HasValue)
var groupedDialogs = dialogs.GroupBy(x => x.MessageId);
foreach (var group in groupedDialogs)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
var targetMessageId = group.Key;
var found = files.Where(x => x.MessageId == targetMessageId);
return messageIds;
if (found.IsNullOrEmpty())
{
continue;
}
var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User);
if (userMsg != null)
{
var userFiles = found.Where(x => x.FileSource == FileSourceType.User);
userMsg.Files = userFiles.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl,
FileName = x.FileName,
FileExtension = x.FileExtension,
FileIndex = x.FileIndex
}).ToList();
}
var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant);
if (botMsg != null)
{
var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot);
botMsg.Files = botFiles.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl,
FileName = x.FileName,
FileExtension = x.FileExtension,
FileIndex = x.FileIndex
}).ToList();
}
}
}
private sealed class MessageFileIndexComparer : IComparer<string>
{
public int Compare(string? x, string? y)
{
if (x == null) return -1;
if (y == null) return 1;
var isNumx = int.TryParse(x, out var xNum);
var isNumy = int.TryParse(y, out var yNum);
if (isNumx && isNumy)
{
return xNum.CompareTo(yNum);
}
return string.Compare(x, y, StringComparison.OrdinalIgnoreCase);
}
}
}

5
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -96,7 +96,8 @@ public partial class LocalFileStorageService
FileName = fileName,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
FileSource = source,
FileIndex = index
};
files.Add(model);
}
@ -195,7 +196,7 @@ public partial class LocalFileStorageService
fs.Write(binary.ToArray(), 0, binary.Length);
fs.Flush(true);
fs.Close();
Thread.Sleep(100);
Thread.Sleep(50);
}
}
catch (Exception ex)

2
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json
View file

@ -11,7 +11,7 @@
"llmConfig": {
"is_inherit": false,
"provider": "openai",
"model": "gpt-4o-mini",
"model": "gpt-5-mini",
"max_recursion_depth": 3
}
}

68
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid
View file

@ -1,44 +1,34 @@
Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
You are a File Selector that helps user select files from the chat context.
- Please take a look at the messages and files in the [MESSAGES AND FILES] and [REQUIREMENTS] section, and select the files that user asks for.
- After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response.
** Ensure the output is only in JSON format without any additional text.
** If no files are selected, you must output an empty list [].
** You may need to look at the file_name as a reference to find the correct file id or ids.
[REQUIREMENTS]
- You need to select the files based on chat context with user.
- You need to check starting from the latest message and its files and go backwards. Usually the recent files are more relevant.
- You can select multiple files if necessary. DO NOT select the duplicate files.
- You need to select the files based on the relevance with the user's request. The output selected files must be in the descending order of relevance.
- You must ensure the information (e.g., role, message_id, file_index, file_name) of each selected file is from the same message.
- If no files are selected, you must output an empty list [] for "selected_files".
[MESSAGES AND FILES]
{% for item in message_files -%}
================
{{ item.name }}
{{ item.description }}{{ "\r\n" }}
================
{%- endfor %}
[RESPONSE FORMAT]
*** Your output must be in the following JSON format:
{
"selected_ids": a list of id selected from the [FILES] section
"selected_files": a list of files selected from the [MESSAGES AND FILES] section, each item in this list should be in JSON format:
{
"message_id": "the message_id of the file",
"file_source": "the file_source of the file: 'user' | 'bot'",
"file_index": "the file_index of the selected file",
"file_name": "the file_name of the selected file"
}
}
Suppose there are four files:
id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user
id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user
id: 3, file_name: example_file.png, content_type: image/png, author: bot
id: 4, file_name: example_file.png, content_type: image/png, author: bot
=====
Example 1:
USER: I want to send the first file and the third file.
OUTPUT: { "selected_ids": [1, 3] }
Example 2:
USER: Send all the images.
OUTPUT: { "selected_ids": [1, 2, 4] }
Example 3:
USER: Send all the images I uploaded.
OUTPUT: { "selected_ids": [1] }
Example 4:
USER: Send the image and the pdf file.
OUTPUT: { "selected_ids": [1, 2] }
Example 5:
USER: Send the images generated by bot
OUTPUT: { "selected_ids": [3, 4] }
=====
[FILES]
{% for file in file_list -%}
{{ file }}{{ "\r\n" }}
{%- endfor %}

17
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -11,23 +11,17 @@ public class HandleEmailSenderFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<HandleEmailSenderFn> _logger;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IHttpContextAccessor _context;
private readonly BotSharpOptions _options;
private readonly EmailSenderSettings _emailSettings;
public HandleEmailSenderFn(
IServiceProvider services,
ILogger<HandleEmailSenderFn> logger,
IHttpClientFactory httpClientFactory,
IHttpContextAccessor context,
BotSharpOptions options,
EmailSenderSettings emailPluginSettings)
{
_services = services;
_logger = logger;
_httpClientFactory = httpClientFactory;
_context = context;
_options = options;
_emailSettings = emailPluginSettings;
}
@ -76,7 +70,16 @@ public class HandleEmailSenderFn : IFunctionCallback
var conversationId = convService.ConversationId;
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true });
var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions
{
IsIncludeBotFiles = true,
IsAttachFiles = true,
MessageLimit = 50,
Provider = "openai",
Model = "gpt-5-mini",
MaxOutputTokens = 8192,
ReasoningEffortLevel = "low"
});
return selecteds;
}

34
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -9,6 +9,7 @@ public class EditImageFn : IFunctionCallback
private readonly ILogger<EditImageFn> _logger;
private readonly FileHandlerSettings _settings;
private string _agentId;
private string _conversationId;
private string _messageId;
@ -39,6 +40,7 @@ public class EditImageFn : IFunctionCallback
private void Init(RoleDialogModel message)
{
var convService = _services.GetRequiredService<IConversationService>();
_agentId = message.CurrentAgentId;
_conversationId = convService.ConversationId;
_messageId = message.MessageId;
}
@ -56,8 +58,14 @@ public class EditImageFn : IFunctionCallback
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
{
Description = description,
IncludeBotFile = true,
ContentTypes = [MediaTypeNames.Image.Png]
IsIncludeBotFiles = true,
IsAttachFiles = true,
ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg],
MessageLimit = 50,
Provider = "openai",
Model = "gpt-5-mini",
MaxOutputTokens = 8192,
ReasoningEffortLevel = "low"
});
return selecteds?.FirstOrDefault();
}
@ -92,7 +100,12 @@ public class EditImageFn : IFunctionCallback
stream.Close();
SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault());
return $"Your image is successfylly editted.";
if (!string.IsNullOrWhiteSpace(response?.Content))
{
return response.Content;
}
return await GetImageEditGreetingResponse(description);
}
catch (Exception ex)
{
@ -102,6 +115,21 @@ public class EditImageFn : IFunctionCallback
}
}
private async Task<string> GetImageEditGreetingResponse(string description)
{
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant"
};
var text = $"Please generate a user-friendly response from the following description to inform user that you have completed the required image: {description}";
var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4o-mini");
var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]);
return response?.Content ?? "Your image is successfully edited.";
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();

19
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -58,13 +58,17 @@ public class ReadImageFn : IFunctionCallback
return new List<RoleDialogModel>();
}
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List<string>
var contentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
});
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes);
var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes);
var images = userImages.Concat(botImages);
foreach (var dialog in dialogs)
{
@ -82,13 +86,12 @@ public class ReadImageFn : IFunctionCallback
if (!imageUrls.IsNullOrEmpty())
{
var lastDialog = dialogs.LastOrDefault(x => x.Role == AgentRole.User) ?? dialogs.Last();
var files = lastDialog.Files ?? [];
lastDialog.Files ??= [];
var addnFiles = imageUrls.Select(x => x?.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => new BotSharpFile { FileUrl = x }).ToList();
files.AddRange(addnFiles);
lastDialog.Files = files;
lastDialog.Files.AddRange(addnFiles);
}
return dialogs;

2
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid
View file

@ -1 +1 @@
Please call util-file-edit_image if user wants to edit or change an image in the conversation.
Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation.

2
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid
View file

@ -1,2 +1,2 @@
** Please call util-file-generate_image if user wants you to provide or generate an image or picture.
** If user does not generate image explicitly, please do not call generate_image.
** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image.

95
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -1,9 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Hooks;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Types;
using Google.Ai.Generativelanguage.V1Beta2;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;
@ -272,51 +274,22 @@ public class GeminiChatCompletionProvider : IChatCompletion
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
contentParts.Add(new Part()
{
FileData = new()
{
FileUri = file.FileUrl
}
});
}
}
CollectMessageContentParts(contentParts, message.Files);
}
contents.Add(new Content(contentParts, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)
{
contents.Add(new Content(message.Content, AgentRole.Model));
var text = message.Content;
var contentParts = new List<Part> { new() { Text = text } };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files);
}
contents.Add(new Content(contentParts, AgentRole.Model));
convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
}
}
@ -342,6 +315,50 @@ public class GeminiChatCompletionProvider : IChatCompletion
return (prompt, request);
}
private void CollectMessageContentParts(List<Part> contentParts, List<BotSharpFile> files)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
foreach (var file in files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
contentParts.Add(new Part()
{
FileData = new()
{
FileUri = file.FileUrl
}
});
}
}
}
private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string> funcPrompts, IEnumerable<string> convPrompts)
{
var prompt = string.Empty;

61
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -321,7 +321,6 @@ public class ChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
@ -397,34 +396,21 @@ public class ChatCompletionProvider : IChatCompletion
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel);
contentParts.Add(contentPart);
}
}
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
}
else if (message.Role == AgentRole.Assistant)
{
messages.Add(new AssistantChatMessage(message.Content));
var text = message.Content;
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new AssistantChatMessage(contentParts));
}
}
@ -432,6 +418,33 @@ public class ChatCompletionProvider : IChatCompletion
return (prompt, messages, options);
}
private void CollectMessageContentParts(List<ChatMessageContentPart> contentParts, List<BotSharpFile> files, ChatImageDetailLevel imageDetailLevel)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
foreach (var file in files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel);
contentParts.Add(contentPart);
}
}
}
private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
{