refine model settings

This commit is contained in:
Jicheng Lu 2025-08-22 17:00:52 -05:00
parent 944869ed79
commit 9108ea6cd3
11 changed files with 102 additions and 38 deletions

View file

@ -42,4 +42,7 @@ public class FileInformation
[JsonPropertyName("file_download_url")] [JsonPropertyName("file_download_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileDownloadUrl { get; set; } = string.Empty; public string? FileDownloadUrl { get; set; } = string.Empty;
public string FileFullName => $"{FileName}.{FileExtension}";
} }

View file

@ -73,23 +73,34 @@ public class LlmModelSetting
} }
} }
#region Embedding model settings
public class EmbeddingSetting public class EmbeddingSetting
{ {
public bool IsDefault { get; set; }
public int Dimension { get; set; } public int Dimension { get; set; }
} }
#endregion
#region Reasoning model settings
public class ReasoningSetting public class ReasoningSetting
{ {
public bool IsDefault { get; set; }
public float Temperature { get; set; } = 1.0f; public float Temperature { get; set; } = 1.0f;
public string? EffortLevel { get; set; } public string? EffortLevel { get; set; }
} }
#endregion
#region Web search model settings
public class WebSearchSetting public class WebSearchSetting
{ {
public bool IsDefault { get; set; } public bool IsDefault { get; set; }
public string? SearchContextSize { get; set; } public string? SearchContextSize { get; set; }
} }
#endregion
#region Image model settings
public class ImageSetting public class ImageSetting
{ {
public ImageGenerationSetting? Generation { get; set; } public ImageGenerationSetting? Generation { get; set; }
@ -99,6 +110,7 @@ public class ImageSetting
public class ImageGenerationSetting public class ImageGenerationSetting
{ {
public bool IsDefault { get; set; }
public ModelSettingBase? Style { get; set; } public ModelSettingBase? Style { get; set; }
public ModelSettingBase? Size { get; set; } public ModelSettingBase? Size { get; set; }
public ModelSettingBase? Quality { get; set; } public ModelSettingBase? Quality { get; set; }
@ -107,16 +119,18 @@ public class ImageGenerationSetting
public class ImageEditSetting public class ImageEditSetting
{ {
public bool IsDefault { get; set; }
public ModelSettingBase? Size { get; set; } public ModelSettingBase? Size { get; set; }
public ModelSettingBase? ResponseFormat { get; set; } public ModelSettingBase? ResponseFormat { get; set; }
} }
public class ImageVariationSetting public class ImageVariationSetting
{ {
public bool IsDefault { get; set; }
public ModelSettingBase? Size { get; set; } public ModelSettingBase? Size { get; set; }
public ModelSettingBase? ResponseFormat { get; set; } public ModelSettingBase? ResponseFormat { get; set; }
} }
#endregion
public class ModelSettingBase public class ModelSettingBase
{ {

View file

@ -93,7 +93,7 @@ public partial class FileInstructService
} }
var providerName = options.Provider ?? "openai"; var providerName = options.Provider ?? "openai";
var model = options?.Model ?? "gpt-4o-mini"; var model = options?.Model ?? "gpt-4.1-mini";
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);

View file

@ -34,7 +34,7 @@ public class WebIntelligentSearchFn : IFunctionCallback
var agent = new Agent var agent = new Agent
{ {
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = fromAgent?.Name ?? "AI Agent", Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = "Please search the websites to handle user's request." Instruction = "Please search the websites to handle user's request."
}; };
@ -53,16 +53,8 @@ public class WebIntelligentSearchFn : IFunctionCallback
{ {
try try
{ {
var provider = "openai"; var (provider, model) = GetLlmProviderModel();
var defaultModel = "gpt-4o-mini-search-preview"; var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var models = llmProviderService.GetProviderModels(provider);
var webSearchModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)?.Name
?? models.FirstOrDefault(x => x.WebSearch != null)?.Name
?? defaultModel;
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: webSearchModel);
var response = await completion.GetChatCompletions(agent, dialogs); var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content; return response.Content;
} }
@ -73,4 +65,18 @@ public class WebIntelligentSearchFn : IFunctionCallback
return error; return error;
} }
} }
private (string, string) GetLlmProviderModel()
{
var provider = "openai";
var model = "gpt-4o-mini-search-preview";
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var models = llmProviderService.GetProviderModels(provider);
var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)
?? models.FirstOrDefault(x => x.WebSearch != null);
model = foundModel?.Name ?? model;
return (provider, model);
}
} }

View file

@ -1,5 +1,3 @@
using System.IO;
namespace BotSharp.Plugin.FileHandler.Functions; namespace BotSharp.Plugin.FileHandler.Functions;
public class EditImageFn : IFunctionCallback public class EditImageFn : IFunctionCallback
@ -30,6 +28,7 @@ public class EditImageFn : IFunctionCallback
var image = await SelectImage(descrpition); var image = await SelectImage(descrpition);
var response = await GetImageEditGeneration(message, descrpition, image); var response = await GetImageEditGeneration(message, descrpition, image);
message.Content = response; message.Content = response;
message.StopCompletion = true;
return true; return true;
} }
@ -43,8 +42,8 @@ public class EditImageFn : IFunctionCallback
private void SetImageOptions() private void SetImageOptions()
{ {
var state = _services.GetRequiredService<IConversationStateService>(); var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("image_response_format", "bytes");
state.SetState("image_count", "1"); state.SetState("image_count", "1");
state.SetState("image_response_format", "bytes");
} }
private async Task<MessageFileModel?> SelectImage(string? description) private async Task<MessageFileModel?> SelectImage(string? description)
@ -53,7 +52,8 @@ public class EditImageFn : IFunctionCallback
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
{ {
Description = description, Description = description,
ContentTypes = new List<string> { MediaTypeNames.Image.Png } IncludeBotFile = true,
ContentTypes = [MediaTypeNames.Image.Png]
}); });
return selecteds?.FirstOrDefault(); return selecteds?.FirstOrDefault();
} }
@ -67,7 +67,8 @@ public class EditImageFn : IFunctionCallback
try try
{ {
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2"); var (provider, model) = GetLlmProviderModel();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text); var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var agent = new Agent var agent = new Agent
@ -78,13 +79,15 @@ public class EditImageFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>(); var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
// To do: convert rgb to rgba image
using var stream = fileBinary.ToStream(); using var stream = fileBinary.ToStream();
stream.Position = 0; stream.Position = 0;
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
stream.Close(); stream.Close();
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; return $"Your image is successfylly editted.";
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -94,6 +97,20 @@ public class EditImageFn : IFunctionCallback
} }
} }
private (string, string) GetLlmProviderModel()
{
var provider = "openai";
var model = "gpt-image-1";
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var models = llmProviderService.GetProviderModels(provider);
var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true)
?? models.FirstOrDefault(x => x.Image?.Edit != null);
model = foundModel?.Name ?? model;
return (provider, model);
}
private void SaveGeneratedImage(ImageGeneration? image) private void SaveGeneratedImage(ImageGeneration? image)
{ {
if (image == null) return; if (image == null) return;

View file

@ -25,16 +25,22 @@ public class GenerateImageFn : IFunctionCallback
SetImageOptions(); SetImageOptions();
var agentService = _services.GetRequiredService<IAgentService>(); var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var imageAgent = new Agent Agent? fromAgent = null;
if (!string.IsNullOrEmpty(message.CurrentAgentId))
{ {
Id = agent?.Id ?? Guid.Empty.ToString(), fromAgent = await agentService.GetAgent(message.CurrentAgentId);
Name = agent?.Name ?? "Unkown", }
var agent = new Agent
{
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = args?.ImageDescription, Instruction = args?.ImageDescription,
TemplateDict = new Dictionary<string, object>() TemplateDict = new Dictionary<string, object>()
}; };
var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription); var response = await GetImageGeneration(agent, message, args?.ImageDescription);
message.Content = response; message.Content = response;
message.StopCompletion = true; message.StopCompletion = true;
return true; return true;
@ -58,7 +64,8 @@ public class GenerateImageFn : IFunctionCallback
{ {
try try
{ {
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-3"); var (provider, model) = GetLlmProviderModel();
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text); var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var result = await completion.GetImageGeneration(agent, dialog); var result = await completion.GetImageGeneration(agent, dialog);
@ -73,6 +80,20 @@ public class GenerateImageFn : IFunctionCallback
} }
} }
private (string, string) GetLlmProviderModel()
{
var provider = "openai";
var model = "gpt-image-1";
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var models = llmProviderService.GetProviderModels(provider);
var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true)
?? models.FirstOrDefault(x => x.Image?.Generation != null);
model = foundModel?.Name ?? model;
return (provider, model);
}
private void SaveGeneratedImages(List<ImageGeneration>? images) private void SaveGeneratedImages(List<ImageGeneration>? images)
{ {
if (images.IsNullOrEmpty()) return; if (images.IsNullOrEmpty()) return;

View file

@ -33,8 +33,8 @@ public class ReadImageFn : IFunctionCallback
var agent = new Agent var agent = new Agent
{ {
Id = BuiltInAgentId.UtilityAssistant, Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = "Utility Agent", Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).",
TemplateDict = new Dictionary<string, object>() TemplateDict = new Dictionary<string, object>()
}; };

View file

@ -33,13 +33,13 @@ public class ReadPdfFn : IFunctionCallback
Agent? fromAgent = null; Agent? fromAgent = null;
if (!string.IsNullOrEmpty(message.CurrentAgentId)) if (!string.IsNullOrEmpty(message.CurrentAgentId))
{ {
fromAgent = await agentService.LoadAgent(message.CurrentAgentId); fromAgent = await agentService.GetAgent(message.CurrentAgentId);
} }
var agent = new Agent var agent = new Agent
{ {
Id = BuiltInAgentId.UtilityAssistant, Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = "Utility Agent", Name = fromAgent?.Name ?? "Utility Assistant",
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).",
TemplateDict = new Dictionary<string, object>() TemplateDict = new Dictionary<string, object>()
}; };

View file

@ -52,9 +52,10 @@ public partial class ImageCompletionProvider
var prompt = message?.Payload ?? message?.Content ?? string.Empty; var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var settingsService = _services.GetRequiredService<ILlmProviderService>(); var settingsService = _services.GetRequiredService<ILlmProviderService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit; var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size"); var size = state.GetState("image_size");
var responseFormat = state.GetState("image_response_format"); var responseFormat = state.GetState("image_response_format");
@ -71,7 +72,7 @@ public partial class ImageCompletionProvider
options.ResponseFormat = GetImageResponseFormat(responseFormat); options.ResponseFormat = GetImageResponseFormat(responseFormat);
} }
var count = GetImageCount(state.GetState("image_count", "1")); var count = GetImageCount(state.GetState("image_count"));
return (prompt, count, options); return (prompt, count, options);
} }
} }

View file

@ -30,9 +30,10 @@ public partial class ImageCompletionProvider
var prompt = message?.Payload ?? message?.Content ?? string.Empty; var prompt = message?.Payload ?? message?.Content ?? string.Empty;
var settingsService = _services.GetRequiredService<ILlmProviderService>(); var settingsService = _services.GetRequiredService<ILlmProviderService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation; var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size"); var size = state.GetState("image_size");
var quality = state.GetState("image_quality"); var quality = state.GetState("image_quality");
var style = state.GetState("image_style"); var style = state.GetState("image_style");
@ -61,7 +62,7 @@ public partial class ImageCompletionProvider
options.ResponseFormat = GetImageResponseFormat(responseFormat); options.ResponseFormat = GetImageResponseFormat(responseFormat);
} }
var count = GetImageCount(state.GetState("image_count", "1")); var count = GetImageCount(state.GetState("image_count"));
return (prompt, count, options); return (prompt, count, options);
} }
} }

View file

@ -28,9 +28,10 @@ public partial class ImageCompletionProvider
private (int, ImageVariationOptions) PrepareVariationOptions() private (int, ImageVariationOptions) PrepareVariationOptions()
{ {
var settingsService = _services.GetRequiredService<ILlmProviderService>(); var settingsService = _services.GetRequiredService<ILlmProviderService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation; var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation;
var state = _services.GetRequiredService<IConversationStateService>();
var size = state.GetState("image_size"); var size = state.GetState("image_size");
var responseFormat = state.GetState("image_response_format"); var responseFormat = state.GetState("image_response_format");