SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1145 from iceljc/features/refine-model-settings

Browse source 
Features/refine model settings
This commit is contained in:
iceljc 2025-09-16 13:16:35 -05:00 committed by GitHub
parent 6fab3e05ff 70477fdcd9
commit 611d5caf96
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
61 changed files with 1613 additions and 590 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Directory.Packages.props
View file

@ -6,8 +6,8 @@
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="EntityFramework" Version="6.4.4" />
<PackageVersion Include="Google_GenerativeAI" Version="2.5.8" />
<PackageVersion Include="Google_GenerativeAI.Live" Version="2.5.8" />
<PackageVersion Include="Google_GenerativeAI" Version="3.2.0" />
<PackageVersion Include="Google_GenerativeAI.Live" Version="3.2.0" />
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="$(AspNetCoreVersion)" />
<PackageVersion Include="Microsoft.AspNetCore.StaticFiles" Version="$(AspNetCoreVersion)" />
@ -20,7 +20,7 @@
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="SharpHook" Version="5.3.9" />
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.7" />
<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.11" />
<PackageVersion Include="System.ClientModel" Version="1.3.0" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
@ -45,8 +45,8 @@
<PackageVersion Include="Whisper.net" Version="1.8.1" />
<PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
<PackageVersion Include="NCrontab" Version="3.3.3" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.5" />
<PackageVersion Include="OpenAI" Version="2.3.0" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.3.0-beta.2" />
<PackageVersion Include="OpenAI" Version="2.4.0" />
<PackageVersion Include="MailKit" Version="4.11.0" />
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageVersion Include="MySql.Data" Version="9.0.0" />

3
src/Infrastructure/BotSharp.Abstraction/Conversations/Dtos/ChatResponseDto.cs
View file

@ -46,9 +46,6 @@ public class ChatResponseDto : InstructResult
[JsonPropertyName("is_streaming")]
public bool IsStreaming { get; set; }
[JsonPropertyName("is_append")]
public bool IsAppend { get; set; }
[JsonPropertyName("created_at")]
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
}

27
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -132,12 +132,6 @@ public class RoleDialogModel : ITrackableMessage
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public bool IsStreaming { get; set; }
/// <summary>
/// Additional messages that can be sent sequentially and save to db
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public ChatMessageWrapper? AdditionalMessageWrapper { get; set; }
public RoleDialogModel()
{
@ -184,26 +178,7 @@ public class RoleDialogModel : ITrackableMessage
Instruction = source.Instruction,
Data = source.Data,
IsStreaming = source.IsStreaming,
Annotations = source.Annotations,
AdditionalMessageWrapper = source.AdditionalMessageWrapper
Annotations = source.Annotations
};
}
}
public class ChatMessageWrapper
{
/// <summary>
/// Messages sending interval in milliseconds
/// </summary>
public int SendingInterval { get; set; }
/// <summary>
/// Whether the Messages are saved to db
/// </summary>
public bool SaveToDb { get; set; }
/// <summary>
/// Messages to send or save
/// </summary>
public List<RoleDialogModel>? Messages { get; set; }
}

6
src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs
View file

@ -14,6 +14,7 @@ public class ConversationSetting
public bool EnableTranslationMemory { get; set; }
public CleanConversationSetting CleanSetting { get; set; } = new();
public RateLimitSetting RateLimit { get; set; } = new();
public FileSelectSetting? FileSelect { get; set; }
}
public class CleanConversationSetting
@ -24,3 +25,8 @@ public class CleanConversationSetting
public int BufferHours { get; set; }
public IEnumerable<string> ExcludeAgentIds { get; set; } = new List<string>();
}
public class FileSelectSetting : LlmConfigBase
{
public int? MessageLimit { get; set; }
}

24
src/Infrastructure/BotSharp.Abstraction/Files/Converters/IImageConverter.cs Normal file
View file

@ -0,0 +1,24 @@
namespace BotSharp.Abstraction.Files.Converters;
public interface IImageConverter
{
public string Provider { get; }
/// <summary>
/// Convert pdf pages to images, and return a list of image file paths
/// </summary>
/// <param name="pdfLocation">Pdf file location</param>
/// <param name="imageFolderLocation">Image folder location</param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
Task<IEnumerable<string>> ConvertPdfToImages(string pdfLocation, string imageFolderLocation) => throw new NotImplementedException();
/// <summary>
/// Convert an image to PNG with RGBA
/// </summary>
/// <param name="binary"></param>
/// <param name="options"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
Task<BinaryData> ConvertImage(BinaryData binary, ImageConvertOptions? options = null) => throw new NotImplementedException();
}

14
src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs
View file

@ -1,14 +0,0 @@
namespace BotSharp.Abstraction.Files.Converters;
public interface IPdf2ImageConverter
{
public string Provider { get; }
/// <summary>
/// Convert pdf pages to images, and return a list of image file paths
/// </summary>
/// <param name="pdfLocation">Pdf file location</param>
/// <param name="imageFolderLocation">Image folder location</param>
/// <returns></returns>
Task<IEnumerable<string>> ConvertPdfToImages(string pdfLocation, string imageFolderLocation);
}

5
src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs
View file

@ -5,8 +5,9 @@ namespace BotSharp.Abstraction.Files;
public class FileCoreSettings
{
public string Storage { get; set; } = FileStorageEnum.LocalFileStorage;
public SettingBase Pdf2TextConverter { get; set; }
public SettingBase Pdf2ImageConverter { get; set; }
public SettingBase? Pdf2TextConverter { get; set; }
public SettingBase? Pdf2ImageConverter { get; set; }
public SettingBase? ImageConverter { get; set; }
}
public class SettingBase

5
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -1,4 +1,3 @@
namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile : FileInformation
@ -9,4 +8,8 @@ public class BotSharpFile : FileInformation
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
[JsonPropertyName("file_index")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileIndex { get; set; } = string.Empty;
}

3
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs
View file

@ -42,4 +42,7 @@ public class FileInformation
[JsonPropertyName("file_download_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileDownloadUrl { get; set; } = string.Empty;
public string FileFullName => $"{FileName}.{FileExtension}";
}

20
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs
View file

@ -2,7 +2,21 @@ namespace BotSharp.Abstraction.Files.Models;
public class FileSelectContext
{
[JsonPropertyName("selected_ids")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<int>? Selecteds { get; set; }
[JsonPropertyName("selected_files")]
public List<FileSelectItem>? SelectedFiles { get; set; }
}
public class FileSelectItem
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
[JsonPropertyName("file_index")]
public string FileIndex { get; set; }
[JsonPropertyName("file_source")]
public string FileSource { get; set; }
[JsonPropertyName("file_name")]
public string? FileName { get; set; }
}

7
src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageConvertOptions.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Abstraction.Files.Models;
public class ImageConvertOptions
{
public string ImageType { get; set; } = "png";
public string ColorType { get; set; } = "rgba";
}

3
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -8,6 +8,9 @@ public class MessageFileModel : FileInformation
[JsonPropertyName("file_source")]
public string FileSource { get; set; } = FileSourceType.User;
[JsonPropertyName("file_index")]
public string FileIndex { get; set; } = string.Empty;
public MessageFileModel()
{

28
src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs
View file

@ -1,17 +1,7 @@
namespace BotSharp.Abstraction.Files.Models;
public class SelectFileOptions
public class SelectFileOptions : LlmConfigBase
{
/// <summary>
/// Llm provider
/// </summary>
public string? Provider { get; set; }
/// <summary>
/// Llm model
/// </summary>
public string? Model { get; set; }
/// <summary>
/// Agent id
/// </summary>
@ -30,7 +20,7 @@ public class SelectFileOptions
/// <summary>
/// Whether include bot generated files
/// </summary>
public bool IncludeBotFile { get; set; }
public bool IsIncludeBotFiles { get; set; }
/// <summary>
/// Conversation breakpoint
@ -38,12 +28,22 @@ public class SelectFileOptions
public bool FromBreakpoint { get; set; }
/// <summary>
/// Message offset from last
/// The maximum number of messages
/// </summary>
public int? Offset { get; set; }
public int? MessageLimit { get; set; }
/// <summary>
/// Whehter attach files to messages
/// </summary>
public bool IsAttachFiles { get; set; }
/// <summary>
/// File content types. If null, all types of files will be retrived
/// </summary>
public IEnumerable<string>? ContentTypes { get; set; }
/// <summary>
/// Data that can be used to fill in the prompt
/// </summary>
public Dictionary<string, object>? Data { get; set; }
}

5
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructOptions.cs
View file

@ -31,4 +31,9 @@ public class InstructOptions
/// Data to fill in prompt
/// </summary>
public Dictionary<string, object> Data { get; set; } = new();
/// <summary>
/// Image convert provider
/// </summary>
public string? ImageConvertProvider { get; set; }
}

12
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -73,23 +73,31 @@ public class LlmModelSetting
}
}
#region Embedding model settings
public class EmbeddingSetting
{
public int Dimension { get; set; }
}
#endregion
#region Reasoning model settings
public class ReasoningSetting
{
public float Temperature { get; set; } = 1.0f;
public string? EffortLevel { get; set; }
}
#endregion
#region Web search model settings
public class WebSearchSetting
{
public bool IsDefault { get; set; }
public string? SearchContextSize { get; set; }
}
#endregion
#region Image model settings
public class ImageSetting
{
public ImageGenerationSetting? Generation { get; set; }
@ -103,12 +111,14 @@ public class ImageGenerationSetting
public ModelSettingBase? Size { get; set; }
public ModelSettingBase? Quality { get; set; }
public ModelSettingBase? ResponseFormat { get; set; }
public ModelSettingBase? Background { get; set; }
}
public class ImageEditSetting
{
public ModelSettingBase? Size { get; set; }
public ModelSettingBase? ResponseFormat { get; set; }
public ModelSettingBase? Background { get; set; }
}
public class ImageVariationSetting
@ -116,7 +126,7 @@ public class ImageVariationSetting
public ModelSettingBase? Size { get; set; }
public ModelSettingBase? ResponseFormat { get; set; }
}
#endregion
public class ModelSettingBase
{

24
src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs Normal file
View file

@ -0,0 +1,24 @@
namespace BotSharp.Abstraction.Models;
public class LlmConfigBase
{
/// <summary>
/// Llm provider
/// </summary>
public string? LlmProvider { get; set; }
/// <summary>
/// Llm model
/// </summary>
public string? LlmModel { get; set; }
/// <summary>
/// Llm maximum output tokens
/// </summary>
public int? MaxOutputTokens { get; set; }
/// <summary>
/// Llm reasoning effort level
/// </summary>
public string? ReasoningEffortLevel { get; set; }
}

17
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file

@ -31,21 +31,10 @@ public class ConversationStorage : IConversationStorage
foreach ( var dialog in dialogs)
{
var innerList = new List<RoleDialogModel> { dialog };
if (dialog.AdditionalMessageWrapper != null
&& dialog.AdditionalMessageWrapper.SaveToDb
&& dialog.AdditionalMessageWrapper.Messages?.Count > 0)
var element = BuildDialogElement(dialog);
if (element != null)
{
innerList.AddRange(dialog.AdditionalMessageWrapper.Messages);
}
foreach (var item in innerList)
{
var element = BuildDialogElement(item);
if (element != null)
{
dialogElements.Add(element);
}
dialogElements.Add(element);
}
}

48
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -48,12 +48,13 @@ public partial class FileInstructService
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-3");
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1");
var message = await completion.GetImageGeneration(new Agent()
{
Id = innerAgentId,
Instruction = instruction
}, new RoleDialogModel(AgentRole.User, instruction ?? text));
}, new RoleDialogModel(AgentRole.User, textContent));
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
@ -80,6 +81,15 @@ public partial class FileInstructService
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var binary = await DownloadFile(image);
// Convert image
var converter = GetImageConverter(options?.ImageConvertProvider);
if (converter != null)
{
binary = await converter.ConvertImage(binary);
image.FileExtension = "png";
}
using var stream = binary.ToStream();
stream.Position = 0;
@ -114,16 +124,26 @@ public partial class FileInstructService
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1");
var binary = await DownloadFile(image);
// Convert image
var converter = GetImageConverter(options?.ImageConvertProvider);
if (converter != null)
{
binary = await converter.ConvertImage(binary);
image.FileExtension = "png";
}
using var stream = binary.ToStream();
stream.Position = 0;
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var fileName = BuildFileName(image.FileName,image.FileExtension, "image", "png");
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
}, new RoleDialogModel(AgentRole.User, instruction ?? text), stream, fileName);
}, new RoleDialogModel(AgentRole.User, textContent), stream, fileName);
stream.Close();
@ -153,10 +173,21 @@ public partial class FileInstructService
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-image-1");
var imageBinary = await DownloadFile(image);
var maskBinary = await DownloadFile(mask);
// Convert image
var converter = GetImageConverter(options?.ImageConvertProvider);
if (converter != null)
{
imageBinary = await converter.ConvertImage(imageBinary);
image.FileExtension = "png";
maskBinary = await converter.ConvertImage(maskBinary);
mask.FileExtension = "png";
}
using var imageStream = imageBinary.ToStream();
imageStream.Position = 0;
@ -164,11 +195,12 @@ public partial class FileInstructService
maskStream.Position = 0;
var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png");
var maskName = BuildFileName(mask.FileName, mask.FileExtension, "mask", "png");
var textContent = text.IfNullOrEmptyAs(instruction).IfNullOrEmptyAs(string.Empty);
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
}, new RoleDialogModel(AgentRole.User, instruction ?? text), imageStream, imageName, maskStream, maskName);
}, new RoleDialogModel(AgentRole.User, textContent), imageStream, imageName, maskStream, maskName);
imageStream.Close();
maskStream.Close();

10
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Files.Converters;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
@ -27,7 +26,7 @@ public partial class FileInstructService
var targetFiles = pdfFiles;
if (provider != "google-ai")
{
targetFiles = await ConvertPdfToImages(pdfFiles);
targetFiles = await ConvertPdfToImages(pdfFiles, options);
}
if (targetFiles.IsNullOrEmpty())
@ -39,7 +38,7 @@ public partial class FileInstructService
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider,
model: options?.Model ?? "gpt-4o", multiModal: true);
model: options?.Model ?? "gpt-5-mini", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = innerAgentId,
@ -116,11 +115,12 @@ public partial class FileInstructService
return locs;
}
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files)
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files, InstructOptions? options = null)
{
var images = new List<string>();
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
var converter = GetImageConverter(options?.ImageConvertProvider);
if (converter == null || files.IsNullOrEmpty())
{
return images;

194
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Templating;
namespace BotSharp.Core.Files.Services;
@ -12,12 +13,23 @@ public partial class FileInstructService
return Enumerable.Empty<MessageFileModel>();
}
var routeContext = _services.GetRequiredService<IRoutingContext>();
var convService = _services.GetRequiredService<IConversationService>();
var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint);
var messageIds = GetMessageIds(dialogs, options.Offset);
var dialogs = routeContext.GetDialogs();
if (dialogs.IsNullOrEmpty())
{
dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint);
}
if (options.MessageLimit > 0)
{
dialogs = dialogs.TakeLast(options.MessageLimit.Value).ToList();
}
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes);
if (options.IncludeBotFile)
if (options.IsIncludeBotFiles)
{
var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes);
files = MergeMessageFiles(messageIds, files, botFiles);
@ -39,8 +51,8 @@ public partial class FileInstructService
foreach (var messageId in messageIds)
{
var users = userFiles.Where(x => x.MessageId == messageId).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).ToList();
var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList();
if (!users.IsNullOrEmpty()) files.AddRange(users);
if (!bots.IsNullOrEmpty()) files.AddRange(bots);
@ -51,87 +63,175 @@ public partial class FileInstructService
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, IEnumerable<RoleDialogModel> dialogs, SelectFileOptions options)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var res = new List<MessageFileModel>();
if (files.IsNullOrEmpty())
{
return res;
}
var agentService = _services.GetRequiredService<IAgentService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
// Handle dialogs and files
var innerDialogs = (dialogs ?? []).ToList();
var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s).";
innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList();
if (options.IsAttachFiles)
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}";
AssembleMessageFiles(innerDialogs, files, options);
}
// Handle instruction
var promptMessages = innerDialogs.Select(x =>
{
var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}";
var fileDescs = x.Files?.Select((f, fidx) => $"- message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " +
$"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'");
var desc = string.Empty;
if (!fileDescs.IsNullOrEmpty())
{
desc = $"[Files]: \r\n\t{string.Join("\r\n\t", fileDescs)}";
}
return new NameDesc(text, desc);
}).ToList();
var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant;
var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "util-file-select_file_instruction";
var prompt = db.GetAgentTemplate(agentId, template);
var foundAgent = db.GetAgent(agentId);
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template);
prompt = render.Render(prompt, new Dictionary<string, object>
var data = new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
{ "message_files", promptMessages }
};
if (!options.Data.IsNullOrEmpty())
{
foreach (var item in options.Data)
{
data[item.Key] = item.Value;
}
}
prompt = render.Render(prompt, data);
// Build agent
var foundAgent = await agentService.GetAgent(agentId);
var agent = new Agent
{
Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = foundAgent?.Name ?? "Utility Assistant",
Instruction = prompt
Instruction = prompt,
LlmConfig = new AgentLlmConfig
{
MaxOutputTokens = options.MaxOutputTokens,
ReasoningEffortLevel = options.ReasoningEffortLevel
}
};
var message = dialogs.LastOrDefault();
var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content;
if (message == null)
{
message = new RoleDialogModel(AgentRole.User, text);
}
else
{
message = RoleDialogModel.From(message, AgentRole.User, text);
}
var providerName = options.Provider ?? "openai";
var model = options?.Model ?? "gpt-4o-mini";
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
// Get ai response
var provider = options.LlmProvider ?? "openai";
var model = options?.LlmModel ?? "gpt-5-mini";
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
var content = response?.Content ?? string.Empty;
var response = await completion.GetChatCompletions(agent, innerDialogs);
var content = response?.Content ?? "{}";
var selecteds = JsonSerializer.Deserialize<FileSelectContext>(content, new JsonSerializerOptions
{
AllowTrailingCommas = true
});
var fids = selecteds?.Selecteds ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
var selectedFiles = selecteds?.SelectedFiles ?? new List<FileSelectItem>();
if (!selectedFiles.IsNullOrEmpty())
{
res = files.Where(file => selectedFiles.Any(x => x.MessageId.IsEqualTo(file.MessageId)
&& x.FileIndex.IsEqualTo(file.FileIndex)
&& x.FileSource.IsEqualTo(file.FileSource))).ToList();
}
return res;
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when selecting files.");
return new List<MessageFileModel>();
return [];
}
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> conversations, int? offset = null)
private void AssembleMessageFiles(IEnumerable<RoleDialogModel> dialogs, IEnumerable<MessageFileModel> files, SelectFileOptions options)
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset.HasValue && offset < 1)
if (dialogs.IsNullOrEmpty() || files.IsNullOrEmpty())
{
offset = 1;
return;
}
var messageIds = new List<string>();
if (offset.HasValue)
var groupedDialogs = dialogs.GroupBy(x => x.MessageId);
foreach (var group in groupedDialogs)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
var targetMessageId = group.Key;
var found = files.Where(x => x.MessageId == targetMessageId);
return messageIds;
if (found.IsNullOrEmpty())
{
continue;
}
var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User);
if (userMsg != null)
{
var userFiles = found.Where(x => x.FileSource == FileSourceType.User);
userMsg.Files = userFiles.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl,
FileName = x.FileName,
FileExtension = x.FileExtension,
FileIndex = x.FileIndex
}).ToList();
}
var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant);
if (botMsg != null)
{
var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot);
botMsg.Files = botFiles.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl,
FileName = x.FileName,
FileExtension = x.FileExtension,
FileIndex = x.FileIndex
}).ToList();
}
}
}
}
private sealed class MessageFileIndexComparer : IComparer<string>
{
public int Compare(string? x, string? y)
{
if (x == null) return -1;
if (y == null) return 1;
var isNumx = int.TryParse(x, out var xNum);
var isNumy = int.TryParse(y, out var yNum);
if (isNumx && isNumy)
{
return xNum.CompareTo(yNum);
}
return string.Compare(x, y, StringComparison.OrdinalIgnoreCase);
}
}
}

10
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Files.Converters;
using Microsoft.Extensions.Options;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.Core.Files.Services;
@ -86,5 +88,13 @@ public partial class FileInstructService : IFileInstructService
fextension = fextension.StartsWith(".") ? fextension.Substring(1) : fextension;
return $"{name}.{fextension}";
}
private IImageConverter? GetImageConverter(string? provider)
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var convertProvider = provider ?? settings?.ImageConverter?.Provider;
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == convertProvider);
return converter;
}
#endregion
}

11
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -96,7 +96,8 @@ public partial class LocalFileStorageService
FileName = fileName,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
FileSource = source,
FileIndex = index
};
files.Add(model);
}
@ -195,7 +196,7 @@ public partial class LocalFileStorageService
fs.Write(binary.ToArray(), 0, binary.Length);
fs.Flush(true);
fs.Close();
Thread.Sleep(100);
Thread.Sleep(50);
}
}
catch (Exception ex)
@ -325,7 +326,7 @@ public partial class LocalFileStorageService
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
var converters = _services.GetServices<IImageConverter>();
if (converters.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
@ -340,10 +341,10 @@ public partial class LocalFileStorageService
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IPdf2ImageConverter? GetPdf2ImageConverter()
private IImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

2
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs
View file

@ -60,7 +60,6 @@ public partial class RoutingService
message.CurrentAgentId = agent.Id;
message.IsStreaming = response.IsStreaming;
message.MessageLabel = response.MessageLabel;
message.AdditionalMessageWrapper = null;
await InvokeFunction(message, dialogs, options);
}
@ -77,7 +76,6 @@ public partial class RoutingService
message.CurrentAgentId = agent.Id;
message.IsStreaming = response.IsStreaming;
message.MessageLabel = response.MessageLabel;
message.AdditionalMessageWrapper = null;
dialogs.Add(message);
Context.SetDialogs(dialogs);
}

1
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs
View file

@ -66,7 +66,6 @@ public partial class RoutingService
message.RichContent = clonedMessage.RichContent;
message.Data = clonedMessage.Data;
message.MessageLabel = clonedMessage.MessageLabel;
message.AdditionalMessageWrapper = clonedMessage.AdditionalMessageWrapper;
}
catch (JsonException ex)
{

38
src/Infrastructure/BotSharp.Core/WebSearch/Functions/WebIntelligentSearchFn.cs
View file

@ -34,7 +34,7 @@ public class WebIntelligentSearchFn : IFunctionCallback
var agent = new Agent
{
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = fromAgent?.Name ?? "AI Agent",
Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = "Please search the websites to handle user's request."
};
@ -53,16 +53,8 @@ public class WebIntelligentSearchFn : IFunctionCallback
{
try
{
var provider = "openai";
var defaultModel = "gpt-4o-mini-search-preview";
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var models = llmProviderService.GetProviderModels(provider);
var webSearchModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)?.Name
?? models.FirstOrDefault(x => x.WebSearch != null)?.Name
?? defaultModel;
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: webSearchModel);
var (provider, model) = GetLlmProviderModel();
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;
}
@ -73,4 +65,28 @@ public class WebIntelligentSearchFn : IFunctionCallback
return error;
}
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = state.GetState("web_search_llm_provider");
var model = state.GetState("web_search_llm_model");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-4o-mini-search-preview";
var models = llmProviderService.GetProviderModels(provider);
var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)
?? models.FirstOrDefault(x => x.WebSearch != null);
model = foundModel?.Name ?? model;
return (provider, model);
}
}

5
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json
View file

@ -11,7 +11,8 @@
"llmConfig": {
"is_inherit": false,
"provider": "openai",
"model": "gpt-4o-mini",
"max_recursion_depth": 3
"model": "gpt-5-mini",
"max_recursion_depth": 3,
"reasoning_effort_level": "minimal"
}
}

76
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-select_file_instruction.liquid
View file

@ -1,44 +1,38 @@
Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
You are a File Selector designed to identify and return the most relevant files from the chat messages and file uploads.
** Ensure the output is only in JSON format without any additional text.
** If no files are selected, you must output an empty list [].
** You may need to look at the file_name as a reference to find the correct file id or ids.
[INSTRUCTION STEPS]
1. Analyze the user’s intent by reasoning through the chat context in [MESSAGES AND FILES], starting from the latest message and moving backward.
2. Based on that, select the relevant files in descending order of relevance according to the [REQUIREMENTS] below.
3. After you figure out what files the user is asking for, please follow the [RESPONSE FORMAT] section and generate your response.
Here is the JSON format to use:
[REQUIREMENTS] (Follow All Precisely)
- Infer what files the user is asking for based on the chat context. ALWAYS select files from [MESSAGES AND FILES].
- Begin from the most recent message and go backward. Newer files are generally more relevant.
- You must check the content of the message files to determine if they are relevant.
- You must select files in descending order of relevance based on the chat context. Do not include irrelevant files.
- You must not include duplicate files (even across different messages).
- You must ensure the information (e.g., message_id, file_source, file_index) of each selected file is from the same message.
- You may select more than one file if required to satisfy the user's request.
- If you think no files are relevant, output an empty list [] for "selected_files".
[MESSAGES AND FILES]
{% for item in message_files -%}
================
{{ item.name }}
{{ item.description }}{{ "\r\n" }}
================
{%- endfor %}
[RESPONSE FORMAT]
*** Your output must be in the following JSON format:
{
"selected_ids": a list of id selected from the [FILES] section
}
Suppose there are four files:
id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user
id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user
id: 3, file_name: example_file.png, content_type: image/png, author: bot
id: 4, file_name: example_file.png, content_type: image/png, author: bot
=====
Example 1:
USER: I want to send the first file and the third file.
OUTPUT: { "selected_ids": [1, 3] }
Example 2:
USER: Send all the images.
OUTPUT: { "selected_ids": [1, 2, 4] }
Example 3:
USER: Send all the images I uploaded.
OUTPUT: { "selected_ids": [1] }
Example 4:
USER: Send the image and the pdf file.
OUTPUT: { "selected_ids": [1, 2] }
Example 5:
USER: Send the images generated by bot
OUTPUT: { "selected_ids": [3, 4] }
=====
[FILES]
{% for file in file_list -%}
{{ file }}{{ "\r\n" }}
{%- endfor %}
"selected_files": a list of files selected from the [MESSAGES AND FILES] section, each item in this list should be in JSON format:
{
"message_id": "the message_id of the file",
"file_source": "the file_source of the file: 'user' | 'bot'",
"file_index": "the file_index of the selected file"
}
}

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -375,7 +375,6 @@ public class ConversationController : ControllerBase
response.RichContent = msg.SecondaryRichContent ?? msg.RichContent;
response.Instruction = msg.Instruction;
response.Data = msg.Data;
response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId);
});
var state = _services.GetRequiredService<IConversationStateService>();
@ -434,8 +433,7 @@ public class ConversationController : ControllerBase
response.Instruction = msg.Instruction;
response.Data = msg.Data;
response.States = state.GetStates();
response.AdditionalMessageWrapper = ChatResponseWrapper.From(msg.AdditionalMessageWrapper, conversationId, inputMsg.MessageId);
await OnChunkReceived(Response, response);
});

142
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -144,7 +144,7 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/multi-modal/upload")]
[HttpPost("/instruct/multi-modal/form")]
public async Task<MultiModalViewModel> MultiModalCompletion([FromForm] IEnumerable<IFormFile> files, [FromForm] MultiModalRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
@ -182,21 +182,21 @@ public class InstructModeController : ControllerBase
#region Generate image
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest input)
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.GenerateImage(input.Text, new InstructOptions
var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
@ -214,29 +214,30 @@ public class InstructModeController : ControllerBase
#region Edit image
[HttpPost("/instruct/image-variation")]
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] ImageVariationRequest input)
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] ImageVariationFileRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
if (input.File == null)
if (request.File == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
}
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.VaryImage(input.File, new InstructOptions
var message = await fileInstruct.VaryImage(request.File, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
ImageConvertProvider = request.ImageConvertProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
return imageViewModel;
}
catch (Exception ex)
@ -248,8 +249,8 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-variation/upload")]
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] MultiModalRequest request)
[HttpPost("/instruct/image-variation/form")]
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
@ -269,7 +270,8 @@ public class InstructModeController : ControllerBase
{
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId
AgentId = request?.AgentId,
ImageConvertProvider = request?.ImageConvertProvider
});
imageViewModel.Content = message.Content;
@ -286,25 +288,26 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-edit")]
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditRequest input)
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditFileRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
if (input.File == null)
if (request.File == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" };
}
var message = await fileInstruct.EditImage(input.Text, input.File, new InstructOptions
var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName,
ImageConvertProvider = request.ImageConvertProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
@ -319,8 +322,8 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-edit/upload")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] MultiModalRequest request)
[HttpPost("/instruct/image-edit/form")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] ImageEditRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -341,12 +344,12 @@ public class InstructModeController : ControllerBase
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
TemplateName = request?.TemplateName,
ImageConvertProvider = request?.ImageConvertProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
return imageViewModel;
}
catch (Exception ex)
@ -359,27 +362,28 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/image-mask-edit")]
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditRequest input)
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditFileRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var image = input.File;
var mask = input.Mask;
var image = request.File;
var mask = request.Mask;
if (image == null || mask == null)
{
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" };
}
var message = await fileInstruct.EditImage(input.Text, image, mask, new InstructOptions
var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName,
ImageConvertProvider = request.ImageConvertProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
@ -394,8 +398,8 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/image-mask-edit/upload")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request)
[HttpPost("/instruct/image-mask-edit/form")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -424,12 +428,12 @@ public class InstructModeController : ControllerBase
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
TemplateName = request?.TemplateName,
ImageConvertProvider = request?.ImageConvertProvider
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
return imageViewModel;
}
catch (Exception ex)
@ -444,21 +448,22 @@ public class InstructModeController : ControllerBase
#region Pdf
[HttpPost("/instruct/pdf-completion")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] MultiModalFileRequest input)
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] PdfReadFileRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new PdfCompletionViewModel();
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(input.Text, input.Files, new InstructOptions
var content = await fileInstruct.ReadPdf(request.Text, request.Files, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName,
ImageConvertProvider = request.ImageConvertProvider
});
viewModel.Content = content;
return viewModel;
@ -472,8 +477,8 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/pdf-completion/upload")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromForm] IEnumerable<IFormFile> files, [FromForm] MultiModalRequest request)
[HttpPost("/instruct/pdf-completion/form")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromForm] IEnumerable<IFormFile> files, [FromForm] PdfReadRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
@ -493,7 +498,8 @@ public class InstructModeController : ControllerBase
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
TemplateName = request?.TemplateName,
ImageConvertProvider = request?.ImageConvertProvider
});
viewModel.Content = content;
return viewModel;
@ -510,26 +516,26 @@ public class InstructModeController : ControllerBase
#region Audio
[HttpPost("/instruct/speech-to-text")]
public async Task<SpeechToTextViewModel> SpeechToText([FromBody] SpeechToTextRequest input)
public async Task<SpeechToTextViewModel> SpeechToText([FromBody] SpeechToTextFileRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new SpeechToTextViewModel();
try
{
var audio = input.File;
var audio = request.File;
if (audio == null)
{
return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" };
}
var content = await fileInstruct.SpeechToText(audio, input.Text, new InstructOptions
var content = await fileInstruct.SpeechToText(audio, request.Text, new InstructOptions
{
Provider = input.Provider,
Model = input.Model,
AgentId = input.AgentId,
TemplateName = input.TemplateName
Provider = request.Provider,
Model = request.Model,
AgentId = request.AgentId,
TemplateName = request.TemplateName
});
viewModel.Content = content;
return viewModel;
@ -543,8 +549,8 @@ public class InstructModeController : ControllerBase
}
}
[HttpPost("/instruct/speech-to-text/upload")]
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm] MultiModalRequest request)
[HttpPost("/instruct/speech-to-text/form")]
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm] SpeechToTextRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -582,13 +588,13 @@ public class InstructModeController : ControllerBase
}
[HttpPost("/instruct/text-to-speech")]
public async Task<IActionResult> TextToSpeech([FromBody] TextToSpeechRequest input)
public async Task<IActionResult> TextToSpeech([FromBody] TextToSpeechRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model);
var binaryData = await completion.GenerateAudioAsync(input.Text);
var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: request.Provider, model: request.Model);
var binaryData = await completion.GenerateAudioAsync(request.Text);
var stream = binaryData.ToStream();
stream.Position = 0;

39
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/Response/ChatResponseModel.cs
View file

@ -1,46 +1,7 @@
using BotSharp.Abstraction.Conversations.Dtos;
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Conversations;
public class ChatResponseModel : ChatResponseDto
{
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonPropertyName("additional_message_wrapper")]
public ChatResponseWrapper? AdditionalMessageWrapper { get; set; }
}
public class ChatResponseWrapper
{
[JsonPropertyName("sending_interval")]
public int SendingInterval { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonPropertyName("messages")]
public List<ChatResponseModel>? Messages { get; set; }
public static ChatResponseWrapper? From(ChatMessageWrapper? wrapper, string conversationId, string? messageId = null)
{
if (wrapper == null)
{
return null;
}
return new ChatResponseWrapper
{
SendingInterval = wrapper.SendingInterval,
Messages = wrapper?.Messages?.Select(x => new ChatResponseModel
{
ConversationId = conversationId,
MessageId = messageId ?? x.MessageId,
Text = !string.IsNullOrEmpty(x.SecondaryContent) ? x.SecondaryContent : x.Content,
MessageLabel = x.MessageLabel,
Function = x.FunctionName,
RichContent = x.SecondaryRichContent ?? x.RichContent,
Instruction = x.Instruction,
Data = x.Data,
IsAppend = true
})?.ToList()
};
}
}

43
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
View file

@ -21,6 +21,7 @@ public class InstructBaseRequest
public List<InstructState> States { get; set; } = [];
}
public class MultiModalRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
@ -33,32 +34,54 @@ public class MultiModalFileRequest : MultiModalRequest
public List<InstructFileModel> Files { get; set; } = [];
}
public class ImageGenerationRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
}
public class ImageVariationRequest : InstructBaseRequest
{
[JsonPropertyName("image_convert_provider")]
public string? ImageConvertProvider { get; set; }
}
public class ImageVariationFileRequest : ImageVariationRequest
{
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}
public class ImageEditRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
[JsonPropertyName("image_convert_provider")]
public string? ImageConvertProvider { get; set; }
}
public class ImageEditFileRequest : ImageEditRequest
{
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}
public class ImageMaskEditRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
[JsonPropertyName("image_convert_provider")]
public string? ImageConvertProvider { get; set; }
}
public class ImageMaskEditFileRequest : ImageMaskEditRequest
{
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
@ -66,11 +89,31 @@ public class ImageMaskEditRequest : InstructBaseRequest
public InstructFileModel Mask { get; set; }
}
public class PdfReadRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
[JsonPropertyName("image_convert_provider")]
public string? ImageConvertProvider { get; set; }
}
public class PdfReadFileRequest : PdfReadRequest
{
[JsonPropertyName("files")]
public List<InstructFileModel> Files { get; set; } = [];
}
public class SpeechToTextRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string? Text { get; set; }
}
public class SpeechToTextFileRequest : SpeechToTextRequest
{
[JsonPropertyName("file")]
public InstructFileModel File { get; set; }
}

208
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,3 +1,4 @@
#pragma warning disable OPENAI001
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Hooks;
@ -65,6 +66,7 @@ public class ChatCompletionProvider : IChatCompletion
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
ToolCallId = toolCall?.Id,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString(),
RenderedInstruction = string.Join("\r\n", renderedInstructions)
@ -82,7 +84,14 @@ public class ChatCompletionProvider : IChatCompletion
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
RenderedInstruction = string.Join("\r\n", renderedInstructions)
RenderedInstruction = string.Join("\r\n", renderedInstructions),
Annotations = value.Annotations?.Select(x => new ChatAnnotation
{
Title = x.WebResourceTitle,
Url = x.WebResourceUri.AbsoluteUri,
StartIndex = x.StartIndex,
EndIndex = x.EndIndex
})?.ToList()
};
}
}
@ -200,6 +209,19 @@ public class ChatCompletionProvider : IChatCompletion
else
{
// Text response received
msg = new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
RenderedInstruction = string.Join("\r\n", renderedInstructions),
Annotations = value.Annotations?.Select(x => new ChatAnnotation
{
Title = x.WebResourceTitle,
Url = x.WebResourceUri.AbsoluteUri,
StartIndex = x.StartIndex,
EndIndex = x.EndIndex
})?.ToList()
};
await onMessageReceived(msg);
}
@ -347,17 +369,7 @@ public class ChatCompletionProvider : IChatCompletion
renderedInstructions = [];
var messages = new List<ChatMessage>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var options = new ChatCompletionOptions()
{
Temperature = temperature,
MaxOutputTokenCount = maxTokens
};
var options = InitChatCompletionOption(agent);
// Prepare instruction and functions
var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent);
@ -367,6 +379,22 @@ public class ChatCompletionProvider : IChatCompletion
messages.Add(new SystemChatMessage(instruction));
}
// Render functions
if (options.WebSearchOptions == null)
{
foreach (var function in functions)
{
if (!agentService.RenderFunction(agent, function)) continue;
var property = agentService.RenderFunctionProperty(agent, function);
options.Tools.Add(ChatTool.CreateFunctionTool(
functionName: function.Name,
functionDescription: function.Description,
functionParameters: BinaryData.FromObjectAsJson(property)));
}
}
foreach (var function in functions)
{
if (!agentService.RenderFunction(agent, function)) continue;
@ -397,6 +425,12 @@ public class ChatCompletionProvider : IChatCompletion
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}
var imageDetailLevel = ChatImageDetailLevel.Auto;
if (allowMultiModal)
{
imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level"));
}
foreach (var message in filteredMessages)
{
if (message.Role == AgentRole.Function)
@ -416,34 +450,21 @@ public class ChatCompletionProvider : IChatCompletion
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
}
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
}
else if (message.Role == AgentRole.Assistant)
{
messages.Add(new AssistantChatMessage(message.Content));
var text = message.Content;
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new AssistantChatMessage(contentParts));
}
}
@ -451,6 +472,34 @@ public class ChatCompletionProvider : IChatCompletion
return (prompt, messages, options);
}
private void CollectMessageContentParts(List<ChatMessageContentPart> contentParts, List<BotSharpFile> files, ChatImageDetailLevel imageDetailLevel)
{
foreach (var file in files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel);
contentParts.Add(contentPart);
}
}
}
private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
{
var prompt = string.Empty;
@ -518,6 +567,95 @@ public class ChatCompletionProvider : IChatCompletion
return prompt;
}
private ChatCompletionOptions InitChatCompletionOption(Agent agent)
{
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
// Reasoning effort
ChatReasoningEffortLevel? reasoningEffortLevel = null;
float? temperature = float.Parse(state.GetState("temperature", "0.0"));
if (settings?.Reasoning != null)
{
temperature = settings.Reasoning.Temperature;
var level = state.GetState("reasoning_effort_level")
.IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel)
.IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel);
reasoningEffortLevel = ParseReasoningEffortLevel(level);
}
// Web search
ChatWebSearchOptions? webSearchOptions = null;
if (settings?.WebSearch != null)
{
temperature = null;
reasoningEffortLevel = null;
webSearchOptions = new();
}
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
return new ChatCompletionOptions()
{
Temperature = temperature,
MaxOutputTokenCount = maxTokens,
ReasoningEffortLevel = reasoningEffortLevel,
WebSearchOptions = webSearchOptions
};
}
private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level)
{
if (string.IsNullOrWhiteSpace(level))
{
return null;
}
var effortLevel = new ChatReasoningEffortLevel("minimal");
switch (level.ToLower())
{
case "low":
effortLevel = ChatReasoningEffortLevel.Low;
break;
case "medium":
effortLevel = ChatReasoningEffortLevel.Medium;
break;
case "high":
effortLevel = ChatReasoningEffortLevel.High;
break;
default:
break;
}
return effortLevel;
}
private ChatImageDetailLevel ParseChatImageDetailLevel(string? level)
{
if (string.IsNullOrWhiteSpace(level))
{
return ChatImageDetailLevel.Auto;
}
var imageLevel = ChatImageDetailLevel.Auto;
switch (level.ToLower())
{
case "low":
imageLevel = ChatImageDetailLevel.Low;
break;
case "high":
imageLevel = ChatImageDetailLevel.High;
break;
default:
break;
}
return imageLevel;
}
public void SetModelName(string model)
{
_model = model;

43
src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs
View file

@ -61,40 +61,33 @@ public class PlotChartFn : IFunctionCallback
});
var response = await GetChatCompletion(innerAgent, dialogs);
var obj = response.JsonContent<LlmContextOut>();
message.Content = obj?.GreetingMessage ?? "Here is the chart you ask for:";
LlmContextOut? ret = null;
var errorMsg = "Error when deserializing ai chart response";
try
{
ret = response.JsonContent<LlmContextOut>();
}
catch (Exception ex)
{
_logger.LogError(ex, errorMsg);
ret = new LlmContextOut
{
GreetingMessage = errorMsg
};
}
message.Content = ret?.GreetingMessage ?? "Here is the chart you ask for:";
message.RichContent = new RichContent<IRichMessage>
{
Recipient = new Recipient { Id = convService.ConversationId },
Message = new ProgramCodeTemplateMessage
{
Text = obj?.JsCode ?? string.Empty,
Text = ret?.JsCode ?? string.Empty,
Language = "javascript"
}
};
if (!string.IsNullOrEmpty(obj?.ReportSummary))
{
message.AdditionalMessageWrapper = new()
{
SendingInterval = 1500,
SaveToDb = true,
Messages = new List<RoleDialogModel>
{
new(AgentRole.Assistant, obj.ReportSummary)
{
MessageId = message.MessageId,
MessageLabel = "chart_report_summary",
Indication = "Summarizing",
CurrentAgentId = message.CurrentAgentId,
FunctionName = message.FunctionName,
FunctionArgs = message.FunctionArgs,
CreatedAt = DateTime.UtcNow
}
}
};
}
message.StopCompletion = true;
return true;
}

8
src/Plugins/BotSharp.Plugin.ChartHandler/Settings/ChartHandlerSettings.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Models;
namespace BotSharp.Plugin.ChartHandler.Settings;
public class ChartHandlerSettings
@ -5,11 +7,7 @@ public class ChartHandlerSettings
public ChartPlotSetting ChartPlot { get; set; }
}
public class ChartPlotSetting
public class ChartPlotSetting : LlmConfigBase
{
public string? LlmProvider { get; set; }
public string? LlmModel { get; set; }
public int? MaxOutputTokens { get; set; }
public string? ReasoningEffortLevel { get; set; }
public int? MessageLimit { get; set; }
}

3
src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid
View file

@ -41,6 +41,5 @@ You must strictly follow the "Hard Requirements", "Render Requirements", "Code R
You must output the response in the following JSON format:
{
"greeting_message": "A short polite message that informs user that the charts have been generated.",
"js_code": "The javascript code that can generate the charts as requested.",
"report_summary": "Generate an insightful summary report in markdown format based on the data. You can summarize using one or multiple titles and list the key findings under each title. Bold all key findings and use level-4 headings or smaller (####, #####, etc.). DO NOT make everything in one line."
"js_code": "The javascript code that can generate the charts as requested."
}

43
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs
View file

@ -126,49 +126,6 @@ public class ChatHubConversationHook : ConversationHookBase
await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action);
await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data);
var wrapper = message.AdditionalMessageWrapper;
if (wrapper?.SendingInterval > 0 && wrapper?.Messages?.Count > 0)
{
action.SenderAction = SenderActionEnum.TypingOn;
await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action);
foreach (var item in wrapper.Messages)
{
if (!string.IsNullOrWhiteSpace(item.Indication))
{
data = new ChatResponseDto
{
ConversationId = conv.ConversationId,
MessageId = item.MessageId,
MessageLabel = item.MessageLabel,
Indication = item.Indication,
Sender = sender
};
await SendEvent(ChatEvent.OnIndicationReceived, conv.ConversationId, data);
}
await Task.Delay(wrapper.SendingInterval);
data = new ChatResponseDto
{
ConversationId = conv.ConversationId,
MessageId = item.MessageId,
MessageLabel = item.MessageLabel,
Text = !string.IsNullOrEmpty(item.SecondaryContent) ? item.SecondaryContent : item.Content,
Function = item.FunctionName,
RichContent = item.SecondaryRichContent ?? item.RichContent,
Data = item.Data,
States = state.GetStates(),
IsAppend = true,
Sender = sender
};
await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data);
}
action.SenderAction = SenderActionEnum.TypingOff;
await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action);
}
await base.OnResponseGenerated(message);
}

195
src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,5 +1,8 @@
#pragma warning disable OPENAI001
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.MessageHub.Models;
using BotSharp.Core.Infrastructures.Streams;
@ -74,7 +77,14 @@ public class ChatCompletionProvider : IChatCompletion
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
RenderedInstruction = string.Join("\r\n", renderedInstructions)
RenderedInstruction = string.Join("\r\n", renderedInstructions),
Annotations = value.Annotations?.Select(x => new ChatAnnotation
{
Title = x.WebResourceTitle,
Url = x.WebResourceUri.AbsoluteUri,
StartIndex = x.StartIndex,
EndIndex = x.EndIndex
})?.ToList()
};
}
@ -168,6 +178,19 @@ public class ChatCompletionProvider : IChatCompletion
else
{
// Text response received
msg = new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
RenderedInstruction = string.Join("\r\n", renderedInstructions),
Annotations = value.Annotations?.Select(x => new ChatAnnotation
{
Title = x.WebResourceTitle,
Url = x.WebResourceUri.AbsoluteUri,
StartIndex = x.StartIndex,
EndIndex = x.EndIndex
})?.ToList()
};
await onMessageReceived(msg);
}
@ -313,23 +336,13 @@ public class ChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
renderedInstructions = [];
var messages = new List<ChatMessage>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var options = new ChatCompletionOptions()
{
Temperature = temperature,
MaxOutputTokenCount = maxTokens
};
var options = InitChatCompletionOption(agent);
// Prepare instruction and functions
var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent);
@ -339,16 +352,20 @@ public class ChatCompletionProvider : IChatCompletion
messages.Add(new SystemChatMessage(instruction));
}
foreach (var function in functions)
// Render functions
if (options.WebSearchOptions == null)
{
if (!agentService.RenderFunction(agent, function)) continue;
foreach (var function in functions)
{
if (!agentService.RenderFunction(agent, function)) continue;
var property = agentService.RenderFunctionProperty(agent, function);
var property = agentService.RenderFunctionProperty(agent, function);
options.Tools.Add(ChatTool.CreateFunctionTool(
functionName: function.Name,
functionDescription: function.Description,
functionParameters: BinaryData.FromObjectAsJson(property)));
options.Tools.Add(ChatTool.CreateFunctionTool(
functionName: function.Name,
functionDescription: function.Description,
functionParameters: BinaryData.FromObjectAsJson(property)));
}
}
if (!string.IsNullOrEmpty(agent.Knowledges))
@ -363,6 +380,12 @@ public class ChatCompletionProvider : IChatCompletion
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}
var imageDetailLevel = ChatImageDetailLevel.Auto;
if (allowMultiModal)
{
imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level"));
}
foreach (var message in filteredMessages)
{
if (message.Role == AgentRole.Function)
@ -377,11 +400,26 @@ public class ChatCompletionProvider : IChatCompletion
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
messages.Add(new UserChatMessage(text));
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
}
else if (message.Role == AgentRole.Assistant)
{
messages.Add(new AssistantChatMessage(message.Content));
var text = message.Content;
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new AssistantChatMessage(contentParts));
}
}
@ -389,6 +427,32 @@ public class ChatCompletionProvider : IChatCompletion
return (prompt, messages, options);
}
private void CollectMessageContentParts(List<ChatMessageContentPart> contentParts, List<BotSharpFile> files, ChatImageDetailLevel imageDetailLevel)
{
foreach (var file in files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel);
contentParts.Add(contentPart);
}
}
}
private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
{
@ -456,4 +520,93 @@ public class ChatCompletionProvider : IChatCompletion
return prompt;
}
private ChatCompletionOptions InitChatCompletionOption(Agent agent)
{
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
// Reasoning effort
ChatReasoningEffortLevel? reasoningEffortLevel = null;
float? temperature = float.Parse(state.GetState("temperature", "0.0"));
if (settings?.Reasoning != null)
{
temperature = settings.Reasoning.Temperature;
var level = state.GetState("reasoning_effort_level")
.IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel)
.IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel);
reasoningEffortLevel = ParseReasoningEffortLevel(level);
}
// Web search
ChatWebSearchOptions? webSearchOptions = null;
if (settings?.WebSearch != null)
{
temperature = null;
reasoningEffortLevel = null;
webSearchOptions = new();
}
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
return new ChatCompletionOptions()
{
Temperature = temperature,
MaxOutputTokenCount = maxTokens,
ReasoningEffortLevel = reasoningEffortLevel,
WebSearchOptions = webSearchOptions
};
}
private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level)
{
if (string.IsNullOrWhiteSpace(level))
{
return null;
}
var effortLevel = new ChatReasoningEffortLevel("minimal");
switch (level.ToLower())
{
case "low":
effortLevel = ChatReasoningEffortLevel.Low;
break;
case "medium":
effortLevel = ChatReasoningEffortLevel.Medium;
break;
case "high":
effortLevel = ChatReasoningEffortLevel.High;
break;
default:
break;
}
return effortLevel;
}
private ChatImageDetailLevel ParseChatImageDetailLevel(string? level)
{
if (string.IsNullOrWhiteSpace(level))
{
return ChatImageDetailLevel.Auto;
}
var imageLevel = ChatImageDetailLevel.Auto;
switch (level.ToLower())
{
case "low":
imageLevel = ChatImageDetailLevel.Low;
break;
case "high":
imageLevel = ChatImageDetailLevel.High;
break;
default:
break;
}
return imageLevel;
}
}

22
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Conversations.Settings;
using MailKit.Net.Smtp;
using MailKit.Security;
using MimeKit;
@ -11,23 +12,17 @@ public class HandleEmailSenderFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<HandleEmailSenderFn> _logger;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IHttpContextAccessor _context;
private readonly BotSharpOptions _options;
private readonly EmailSenderSettings _emailSettings;
public HandleEmailSenderFn(
IServiceProvider services,
ILogger<HandleEmailSenderFn> logger,
IHttpClientFactory httpClientFactory,
IHttpContextAccessor context,
BotSharpOptions options,
EmailSenderSettings emailPluginSettings)
{
_services = services;
_logger = logger;
_httpClientFactory = httpClientFactory;
_context = context;
_options = options;
_emailSettings = emailPluginSettings;
}
@ -73,10 +68,19 @@ public class HandleEmailSenderFn : IFunctionCallback
private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
{
var convService = _services.GetRequiredService<IConversationService>();
var conversationId = convService.ConversationId;
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true });
var convSettings = _services.GetRequiredService<ConversationSetting>();
var selecteds = await fileInstruct.SelectMessageFiles(convService.ConversationId, new SelectFileOptions
{
IsIncludeBotFiles = true,
IsAttachFiles = true,
MessageLimit = convSettings?.FileSelect?.MessageLimit,
LlmProvider = convSettings?.FileSelect?.LlmProvider,
LlmModel = convSettings?.FileSelect?.LlmModel,
MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens,
ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel
});
return selecteds;
}

4
src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj
View file

@ -48,6 +48,10 @@
</Content>
</ItemGroup>
<ItemGroup>
<PackageReference Include="SixLabors.ImageSharp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>

71
src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs Normal file
View file

@ -0,0 +1,71 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Formats.Png;
using System.IO;
namespace BotSharp.Plugin.FileHandler.Converters;
public class FileHandlerImageConverter : IImageConverter
{
private readonly IServiceProvider _services;
private readonly ILogger<FileHandlerImageConverter> _logger;
public FileHandlerImageConverter(
IServiceProvider services,
ILogger<FileHandlerImageConverter> logger)
{
_services = services;
_logger = logger;
}
public string Provider => "file-handler";
public async Task<BinaryData> ConvertImage(BinaryData binary, ImageConvertOptions? options = null)
{
try
{
using var image = Image.Load<Rgba32>(binary.ToArray());
using var memoryStream = new MemoryStream();
if (options?.ImageType == "png")
{
var colorType = PngColorType.RgbWithAlpha;
switch (options?.ColorType)
{
case "grayscale":
colorType = PngColorType.Grayscale;
break;
case "grayscaleWithAlpha":
colorType = PngColorType.GrayscaleWithAlpha;
break;
case "rgb":
colorType = PngColorType.Rgb;
break;
case "palette":
colorType = PngColorType.Palette;
break;
}
image.SaveAsPng(memoryStream, new PngEncoder
{
ColorType = colorType
});
}
else
{
image.SaveAsPng(memoryStream, new PngEncoder
{
ColorType = PngColorType.RgbWithAlpha
});
}
var convertedBinary = BinaryData.FromBytes(memoryStream.ToArray());
return await Task.FromResult(convertedBinary);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when converting image to RGBA png in {Provider}.");
return binary;
}
}
}

2
src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Plugin.FileHandler.Converters;
using BotSharp.Plugin.FileHandler.Hooks;
using Microsoft.Extensions.Configuration;
@ -20,6 +21,7 @@ public class FileHandlerPlugin : IBotSharpPlugin
});
services.AddScoped<IAgentUtilityHook, FileHandlerUtilityHook>();
services.AddScoped<IImageConverter, FileHandlerImageConverter>();
}
}

130
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -1,4 +1,4 @@
using System.IO;
using BotSharp.Abstraction.Conversations.Settings;
namespace BotSharp.Plugin.FileHandler.Functions;
@ -9,33 +9,42 @@ public class EditImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<EditImageFn> _logger;
private readonly FileHandlerSettings _settings;
private Agent _agent;
private string _conversationId;
private string _messageId;
public EditImageFn(
IServiceProvider services,
ILogger<EditImageFn> logger)
ILogger<EditImageFn> logger,
FileHandlerSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
var descrpition = args?.UserRequest ?? string.Empty;
Init(message);
await Init(message);
SetImageOptions();
var image = await SelectImage(descrpition);
var response = await GetImageEditGeneration(message, descrpition, image);
message.Content = response;
message.StopCompletion = true;
return true;
}
private void Init(RoleDialogModel message)
private async Task Init(RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var convService = _services.GetRequiredService<IConversationService>();
_agent = await agentService.GetAgent(message.CurrentAgentId);
_conversationId = convService.ConversationId;
_messageId = message.MessageId;
}
@ -43,17 +52,26 @@ public class EditImageFn : IFunctionCallback
private void SetImageOptions()
{
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("image_response_format", "bytes");
state.SetState("image_count", "1");
state.SetState("image_response_format", "bytes");
}
private async Task<MessageFileModel?> SelectImage(string? description)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var convSettings = _services.GetRequiredService<ConversationSetting>();
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
{
Description = description,
ContentTypes = new List<string> { MediaTypeNames.Image.Png }
IsIncludeBotFiles = true,
IsAttachFiles = true,
ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg],
MessageLimit = convSettings?.FileSelect?.MessageLimit,
LlmProvider = convSettings?.FileSelect?.LlmProvider,
LlmModel = convSettings?.FileSelect?.LlmModel,
MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens,
ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel
});
return selecteds?.FirstOrDefault();
}
@ -67,24 +85,34 @@ public class EditImageFn : IFunctionCallback
try
{
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2");
var (provider, model) = GetLlmProviderModel();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant"
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = _agent?.Name ?? "Utility Assistant"
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
using var stream = fileBinary.ToStream();
stream.Position = 0;
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty);
stream.Close();
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
var rgbaBinary = await ConvertImageToPngWithRgba(fileBinary);
image.FileExtension = "png";
return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted.";
using var stream = rgbaBinary.ToStream();
stream.Position = 0;
var response = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
stream.Close();
var savedFiles = SaveGeneratedImage(response?.GeneratedImages?.FirstOrDefault());
if (!string.IsNullOrWhiteSpace(response?.Content))
{
return response.Content;
}
return await GetImageEditResponse(description, defaultContent: null);
}
catch (Exception ex)
{
@ -94,9 +122,64 @@ public class EditImageFn : IFunctionCallback
}
}
private void SaveGeneratedImage(ImageGeneration? image)
private async Task<string> GetImageEditResponse(string description, string? defaultContent)
{
if (image == null) return;
if (defaultContent != null)
{
return defaultContent;
}
var llmConfig = _agent.LlmConfig;
var agent = new Agent
{
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = _agent?.Name ?? "Utility Assistant",
LlmConfig = new AgentLlmConfig
{
Provider = llmConfig?.Provider ?? "openai",
Model = llmConfig?.Model ?? "gpt-4o-mini",
MaxOutputTokens = llmConfig?.MaxOutputTokens,
ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel
}
};
return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description);
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("image_edit_llm_provider");
var model = state.GetState("image_edit_llm_provider");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = fileSettings?.Image?.Edit?.LlmProvider;
model = fileSettings?.Image?.Edit?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-image-1";
return (provider, model);
}
private IEnumerable<string> SaveGeneratedImage(ImageGeneration? image)
{
if (image == null)
{
return [];
}
var files = new List<FileDataModel>()
{
@ -109,5 +192,18 @@ public class EditImageFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
return files.Select(x => x.FileName);
}
private async Task<BinaryData> ConvertImageToPngWithRgba(BinaryData binaryFile)
{
var provider = _settings?.ImageConverter?.Provider;
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
if (converter == null)
{
return binaryFile;
}
return await converter.ConvertImage(binaryFile);
}
}

98
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
namespace BotSharp.Plugin.FileHandler.Functions;
public class GenerateImageFn : IFunctionCallback
@ -7,6 +9,8 @@ public class GenerateImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<GenerateImageFn> _logger;
private Agent _agent;
private string _conversationId;
private string _messageId;
@ -25,16 +29,9 @@ public class GenerateImageFn : IFunctionCallback
SetImageOptions();
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var imageAgent = new Agent
{
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = args?.ImageDescription,
TemplateDict = new Dictionary<string, object>()
};
_agent = await agentService.GetAgent(message.CurrentAgentId);
var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription);
var response = await GetImageGeneration(message, args?.ImageDescription);
message.Content = response;
message.StopCompletion = true;
return true;
@ -52,18 +49,33 @@ public class GenerateImageFn : IFunctionCallback
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("image_count", "1");
state.SetState("image_quality", "medium");
state.SetState("image_response_format", "bytes");
}
private async Task<string> GetImageGeneration(Agent agent, RoleDialogModel message, string? description)
private async Task<string> GetImageGeneration(RoleDialogModel message, string? description)
{
try
{
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-3");
var agent = new Agent
{
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = _agent?.Name ?? "Utility Assistant",
Instruction = description
};
var (provider, model) = GetLlmProviderModel();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var result = await completion.GetImageGeneration(agent, dialog);
SaveGeneratedImages(result?.GeneratedImages);
return result?.Content ?? string.Empty;
var savedFiles = SaveGeneratedImages(result?.GeneratedImages);
if (!string.IsNullOrWhiteSpace(result?.Content))
{
return result.Content;
}
return await GetImageGenerationResponse(description, defaultContent: null);
}
catch (Exception ex)
{
@ -73,9 +85,64 @@ public class GenerateImageFn : IFunctionCallback
}
}
private void SaveGeneratedImages(List<ImageGeneration>? images)
private async Task<string> GetImageGenerationResponse(string description, string? defaultContent)
{
if (images.IsNullOrEmpty()) return;
if (defaultContent != null)
{
return defaultContent;
}
var llmConfig = _agent.LlmConfig;
var agent = new Agent
{
Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = _agent?.Name ?? "Utility Assistant",
LlmConfig = new AgentLlmConfig
{
Provider = llmConfig?.Provider ?? "openai",
Model = llmConfig?.Model ?? "gpt-4o-mini",
MaxOutputTokens = llmConfig?.MaxOutputTokens,
ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel
}
};
return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description);
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("image_generate_llm_provider");
var model = state.GetState("image_generate_llm_model");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = fileSettings?.Image?.Generation?.LlmProvider;
model = fileSettings?.Image?.Generation?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-image-1";
return (provider, model);
}
private IEnumerable<string> SaveGeneratedImages(List<ImageGeneration>? images)
{
if (images.IsNullOrEmpty())
{
return [];
}
var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new FileDataModel
{
@ -85,5 +152,6 @@ public class GenerateImageFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
return files.Select(x => x.FileName);
}
}

73
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -33,10 +33,10 @@ public class ReadImageFn : IFunctionCallback
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Agent",
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).",
TemplateDict = new Dictionary<string, object>()
LlmConfig = fromAgent?.LlmConfig ?? new()
};
var wholeDialogs = routingCtx.GetDialogs();
@ -58,13 +58,17 @@ public class ReadImageFn : IFunctionCallback
return new List<RoleDialogModel>();
}
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List<string>
var contentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
});
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes);
var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes);
var images = userImages.Concat(botImages);
foreach (var dialog in dialogs)
{
@ -82,13 +86,12 @@ public class ReadImageFn : IFunctionCallback
if (!imageUrls.IsNullOrEmpty())
{
var lastDialog = dialogs.LastOrDefault(x => x.Role == AgentRole.User) ?? dialogs.Last();
var files = lastDialog.Files ?? [];
lastDialog.Files ??= [];
var addnFiles = imageUrls.Select(x => x?.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => new BotSharpFile { FileUrl = x }).ToList();
files.AddRange(addnFiles);
lastDialog.Files = files;
.Select(x => new BotSharpFile { FileUrl = x }).ToList();
lastDialog.Files.AddRange(addnFiles);
}
return dialogs;
@ -98,8 +101,8 @@ public class ReadImageFn : IFunctionCallback
{
try
{
var provider = "openai";
var model = "gpt-5-mini";
var (provider, model) = GetLlmProviderModel();
SetImageDetailLevel();
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;
@ -111,4 +114,46 @@ public class ReadImageFn : IFunctionCallback
return error;
}
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("image_read_llm_provider");
var model = state.GetState("image_read_llm_model");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = fileSettings?.Image?.Reading?.LlmProvider;
model = fileSettings?.Image?.Reading?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-5-mini";
return (provider, model);
}
private void SetImageDetailLevel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var key = "chat_image_detail_level";
var level = state.GetState(key);
if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel))
{
state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel);
}
}
}

54
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -33,15 +33,15 @@ public class ReadPdfFn : IFunctionCallback
Agent? fromAgent = null;
if (!string.IsNullOrEmpty(message.CurrentAgentId))
{
fromAgent = await agentService.LoadAgent(message.CurrentAgentId);
fromAgent = await agentService.GetAgent(message.CurrentAgentId);
}
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Agent",
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).",
TemplateDict = new Dictionary<string, object>()
LlmConfig = fromAgent?.LlmConfig ?? new()
};
var wholeDialogs = routingCtx.GetDialogs();
@ -89,8 +89,8 @@ public class ReadPdfFn : IFunctionCallback
{
try
{
var provider = "openai";
var model = "gpt-5-mini";
var (provider, model) = GetLlmProviderModel();
SetImageDetailLevel();
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;
@ -102,4 +102,46 @@ public class ReadPdfFn : IFunctionCallback
return error;
}
}
private (string, string) GetLlmProviderModel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("pdf_read_llm_provider");
var model = state.GetState("pdf_read_llm_model");
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = fileSettings?.Image?.Reading?.LlmProvider;
model = fileSettings?.Image?.Reading?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}
provider = "openai";
model = "gpt-5-mini";
return (provider, model);
}
private void SetImageDetailLevel()
{
var state = _services.GetRequiredService<IConversationStateService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var key = "chat_image_detail_level";
var level = state.GetState(key);
if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Pdf?.Reading?.ImageDetailLevel))
{
state.SetState(key, fileSettings.Pdf.Reading.ImageDetailLevel);
}
}
}

33
src/Plugins/BotSharp.Plugin.FileHandler/Helpers/AiResponseHelper.cs Normal file
View file

@ -0,0 +1,33 @@
using BotSharp.Abstraction.Agents.Models;
namespace BotSharp.Plugin.FileHandler.Helpers;
internal static class AiResponseHelper
{
internal static string GetDefaultResponse(IEnumerable<string> files)
{
if (files.IsNullOrEmpty())
{
return $"No image is generated.";
}
if (files.Count() > 1)
{
return $"Here are the images you asked for: {string.Join(", ", files)}";
}
return $"Here is the image you asked for: {string.Join(", ", files)}";
}
internal static async Task<string> GetImageGenerationResponse(IServiceProvider services, Agent agent, string description)
{
var text = $"Please generate a user-friendly response from the following description to " +
$"inform user that you have completed the required image: {description}";
var provider = agent?.LlmConfig?.Provider ?? "openai";
var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini";
var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]);
return response.Content;
}
}

52
src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
View file

@ -2,4 +2,56 @@ namespace BotSharp.Plugin.FileHandler.Settings;
public class FileHandlerSettings
{
public ImageSettings? Image { get; set; }
public PdfSettings? Pdf { get; set; }
public SettingBase? ImageConverter { get; set; }
}
#region Image
public class ImageSettings
{
public ImageReadSettings? Reading { get; set; }
public ImageGenerationSettings? Generation { get; set; }
public ImageEditSettings? Edit { get; set; }
public ImageVariationSettings? Variation { get; set; }
}
public class ImageReadSettings : FileLlmSettingBase
{
public string? ImageDetailLevel { get; set; }
}
public class ImageGenerationSettings : FileLlmSettingBase
{
}
public class ImageEditSettings : FileLlmSettingBase
{
}
public class ImageVariationSettings : FileLlmSettingBase
{
}
#endregion
#region Pdf
public class PdfSettings
{
public PdfReadSettings? Reading { get; set; }
}
public class PdfReadSettings : FileLlmSettingBase
{
public bool ConvertToImage { get; set; }
public string? ImageDetailLevel { get; set; }
}
#endregion
public class FileLlmSettingBase
{
public string? LlmProvider { get; set; }
public string? LlmModel { get; set; }
}

4
src/Plugins/BotSharp.Plugin.FileHandler/Using.cs
View file

@ -16,6 +16,7 @@ global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Converters;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Utilities;
@ -29,4 +30,5 @@ global using BotSharp.Abstraction.Options;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.FileHandler.Enums;
global using BotSharp.Plugin.FileHandler.Settings;
global using BotSharp.Plugin.FileHandler.LlmContexts;
global using BotSharp.Plugin.FileHandler.LlmContexts;
global using BotSharp.Plugin.FileHandler.Helpers;

2
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json
View file

@ -6,7 +6,7 @@
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to editing the requested image."
"description": "The user requirement about editing the requested image."
}
},
"required": [ "user_request" ]

2
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid
View file

@ -1 +1 @@
Please call util-file-edit_image if user wants to edit or change an image in the conversation.
Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation.

2
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid
View file

@ -1,2 +1,2 @@
** Please call util-file-generate_image if user wants you to provide or generate an image or picture.
** If user does not generate image explicitly, please do not call generate_image.
** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image.

95
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -1,9 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Hooks;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Types;
using Google.Ai.Generativelanguage.V1Beta2;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;
@ -272,51 +274,22 @@ public class GeminiChatCompletionProvider : IChatCompletion
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
contentParts.Add(new Part()
{
FileData = new()
{
FileUri = file.FileUrl
}
});
}
}
CollectMessageContentParts(contentParts, message.Files);
}
contents.Add(new Content(contentParts, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)
{
contents.Add(new Content(message.Content, AgentRole.Model));
var text = message.Content;
var contentParts = new List<Part> { new() { Text = text } };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files);
}
contents.Add(new Content(contentParts, AgentRole.Model));
convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
}
}
@ -342,6 +315,50 @@ public class GeminiChatCompletionProvider : IChatCompletion
return (prompt, request);
}
private void CollectMessageContentParts(List<Part> contentParts, List<BotSharpFile> files)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
foreach (var file in files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
contentParts.Add(new Part()
{
FileData = new()
{
FileUri = file.FileUrl
}
});
}
}
}
private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string> funcPrompts, IEnumerable<string> convPrompts)
{
var prompt = string.Empty;

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs
View file

@ -40,7 +40,7 @@ public class AudioSynthesisProvider : IAudioSynthesis
var options = new SpeechGenerationOptions
{
ResponseFormat = responseFormat,
SpeedRatio = speed,
SpeedRatio = speed
};
return (voice, options);

6
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
View file

@ -57,6 +57,12 @@ public class AudioTranscriptionProvider : IAudioTranscription
switch (value)
{
case "json":
format = new AudioTranscriptionFormat("json");
break;
case "text":
format = new AudioTranscriptionFormat("text");
break;
case "simple":
format = AudioTranscriptionFormat.Simple;
break;
case "srt":

108
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -4,7 +4,6 @@ using BotSharp.Abstraction.MessageHub.Models;
using BotSharp.Core.Infrastructures.Streams;
using BotSharp.Core.MessageHub;
using OpenAI.Chat;
using Pipelines.Sockets.Unofficial.Arenas;
namespace BotSharp.Plugin.OpenAI.Providers.Chat;
@ -180,6 +179,19 @@ public class ChatCompletionProvider : IChatCompletion
else
{
// Text response received
msg = new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
RenderedInstruction = string.Join("\r\n", renderedInstructions),
Annotations = value.Annotations?.Select(x => new ChatAnnotation
{
Title = x.WebResourceTitle,
Url = x.WebResourceUri.AbsoluteUri,
StartIndex = x.StartIndex,
EndIndex = x.EndIndex
})?.ToList()
};
await onMessageReceived(msg);
}
@ -320,7 +332,7 @@ public class ChatCompletionProvider : IChatCompletion
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
@ -371,6 +383,12 @@ public class ChatCompletionProvider : IChatCompletion
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}
var imageDetailLevel = ChatImageDetailLevel.Auto;
if (allowMultiModal)
{
imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level"));
}
foreach (var message in filteredMessages)
{
if (message.Role == AgentRole.Function)
@ -390,34 +408,21 @@ public class ChatCompletionProvider : IChatCompletion
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
}
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
}
else if (message.Role == AgentRole.Assistant)
{
messages.Add(new AssistantChatMessage(message.Content));
var text = message.Content;
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
CollectMessageContentParts(contentParts, message.Files, imageDetailLevel);
}
messages.Add(new AssistantChatMessage(contentParts));
}
}
@ -425,6 +430,32 @@ public class ChatCompletionProvider : IChatCompletion
return (prompt, messages, options);
}
private void CollectMessageContentParts(List<ChatMessageContentPart> contentParts, List<BotSharpFile> files, ChatImageDetailLevel imageDetailLevel)
{
foreach (var file in files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel);
contentParts.Add(contentPart);
}
}
}
private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
{
@ -506,8 +537,8 @@ public class ChatCompletionProvider : IChatCompletion
{
temperature = settings.Reasoning.Temperature;
var level = state.GetState("reasoning_effort_level")
.IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel ?? string.Empty)
.IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel ?? string.Empty);
.IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel)
.IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel);
reasoningEffortLevel = ParseReasoningEffortLevel(level);
}
@ -559,6 +590,29 @@ public class ChatCompletionProvider : IChatCompletion
return effortLevel;
}
private ChatImageDetailLevel ParseChatImageDetailLevel(string? level)
{
if (string.IsNullOrWhiteSpace(level))
{
return ChatImageDetailLevel.Auto;
}
var imageLevel = ChatImageDetailLevel.Auto;
switch (level.ToLower())
{
case "low":
imageLevel = ChatImageDetailLevel.Low;
break;
case "high":
imageLevel = ChatImageDetailLevel.High;
break;
default:
break;
}
return imageLevel;
}
public void SetModelName(string model)
{
_model = model;

14
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs
View file

@ -1,3 +1,4 @@
#pragma warning disable OPENAI001
using OpenAI.Images;
namespace BotSharp.Plugin.OpenAI.Providers.Image;
@ -52,14 +53,17 @@ public partial class ImageCompletionProvider
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
var responseFormat = state.GetState("image_response_format");
var background = state.GetState("image_background");
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit;
size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null;
responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null;
background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null;
var options = new ImageEditOptions();
if (!string.IsNullOrEmpty(size))
@ -70,8 +74,12 @@ public partial class ImageCompletionProvider
{
options.ResponseFormat = GetImageResponseFormat(responseFormat);
}
if (!string.IsNullOrEmpty(background))
{
options.Background = GetImageBackground(background);
}
var count = GetImageCount(state.GetState("image_count", "1"));
var count = GetImageCount(state.GetState("image_count"));
return (prompt, count, options);
}
}

14
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs
View file

@ -1,3 +1,4 @@
#pragma warning disable OPENAI001
using OpenAI.Images;
namespace BotSharp.Plugin.OpenAI.Providers.Image;
@ -30,18 +31,21 @@ public partial class ImageCompletionProvider
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
var quality = state.GetState("image_quality");
var style = state.GetState("image_style");
var responseFormat = state.GetState("image_response_format");
var background = state.GetState("image_background");
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation;
size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null;
quality = settings?.Quality != null ? VerifyImageParameter(quality, settings.Quality.Default, settings.Quality.Options) : null;
style = settings?.Style != null ? VerifyImageParameter(style, settings.Style.Default, settings.Style.Options) : null;
responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null;
background = settings?.Background != null ? VerifyImageParameter(background, settings.Background.Default, settings.Background.Options) : null;
var options = new ImageGenerationOptions();
if (!string.IsNullOrEmpty(size))
@ -60,8 +64,12 @@ public partial class ImageCompletionProvider
{
options.ResponseFormat = GetImageResponseFormat(responseFormat);
}
if (!string.IsNullOrEmpty(background))
{
options.Background = GetImageBackground(background);
}
var count = GetImageCount(state.GetState("image_count", "1"));
var count = GetImageCount(state.GetState("image_count"));
return (prompt, count, options);
}
}

5
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs
View file

@ -28,12 +28,13 @@ public partial class ImageCompletionProvider
private (int, ImageVariationOptions) PrepareVariationOptions()
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size");
var responseFormat = state.GetState("image_response_format");
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation;
size = settings?.Size != null ? VerifyImageParameter(size, settings.Size.Default, settings.Size.Options) : null;
responseFormat = settings?.ResponseFormat != null ? VerifyImageParameter(responseFormat, settings.ResponseFormat.Default, settings.ResponseFormat.Options) : null;

21
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs
View file

@ -163,6 +163,27 @@ public partial class ImageCompletionProvider : IImageCompletion
return retFormat;
}
private GeneratedImageBackground GetImageBackground(string? background)
{
var value = !string.IsNullOrEmpty(background) ? background : "auto";
GeneratedImageBackground retBackground;
switch (value)
{
case "transparent":
retBackground = GeneratedImageBackground.Transparent;
break;
case "opaque":
retBackground = GeneratedImageBackground.Opaque;
break;
default:
retBackground = GeneratedImageBackground.Auto;
break;
}
return retBackground;
}
private int GetImageCount(string count)
{
if (!int.TryParse(count, out var retCount))

10
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -62,6 +62,7 @@ public partial class TencentCosService
var fileName = Path.GetFileNameWithoutExtension(file);
var fileExtension = Path.GetExtension(file).Substring(1);
var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty;
var model = new MessageFileModel()
{
MessageId = messageId,
@ -71,7 +72,8 @@ public partial class TencentCosService
FileName = fileName,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
FileSource = source,
FileIndex = fileIndex
};
files.Add(model);
}
@ -241,7 +243,7 @@ public partial class TencentCosService
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
var converters = _services.GetServices<IImageConverter>();
if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
var converter = GetPdf2ImageConverter();
@ -252,10 +254,10 @@ public partial class TencentCosService
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IPdf2ImageConverter? GetPdf2ImageConverter()
private IImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}

43
src/WebStarter/appsettings.json
View file

@ -299,6 +299,13 @@
"MaxConversationPerDay": 100,
"MaxInputLengthPerRequest": 256,
"MinTimeSecondsBetweenMessages": 2
},
"FileSelect": {
"LlmProvider": "openai",
"LlmModel": "gpt-5-mini",
"MaxOutputTokens": 8192,
"ReasoningEffortLevel": "low",
"MessageLimit": 50
}
},
@ -437,6 +444,42 @@
},
"Pdf2ImageConverter": {
"Provider": ""
},
"ImageConverter": {
"Provider": ""
}
},
"FileHandler": {
"Image": {
"Reading": {
"LlmProvider": "openai",
"LlmModel": "gpt-5-mini",
"ImageDetailLevel": "auto"
},
"Generation": {
"LlmProvider": "openai",
"LlmModel": "gpt-image-1"
},
"Edit": {
"LlmProvider": "openai",
"LlmModel": "gpt-image-1"
},
"Variation": {
"LlmProvider": "",
"LlmModel": ""
}
},
"Pdf": {
"Reading": {
"LlmProvider": "openai",
"LlmModel": "gpt-5-mini",
"ConvertToImage": true,
"ImageDetailLevel": "auto"
}
},
"ImageConverter": {
"Provider": "file-handler"
}
},