refine model settings
This commit is contained in:
parent
944869ed79
commit
9108ea6cd3
|
|
@ -42,4 +42,7 @@ public class FileInformation
|
||||||
[JsonPropertyName("file_download_url")]
|
[JsonPropertyName("file_download_url")]
|
||||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||||
public string? FileDownloadUrl { get; set; } = string.Empty;
|
public string? FileDownloadUrl { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
|
||||||
|
public string FileFullName => $"{FileName}.{FileExtension}";
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -73,23 +73,34 @@ public class LlmModelSetting
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#region Embedding model settings
|
||||||
public class EmbeddingSetting
|
public class EmbeddingSetting
|
||||||
{
|
{
|
||||||
|
public bool IsDefault { get; set; }
|
||||||
public int Dimension { get; set; }
|
public int Dimension { get; set; }
|
||||||
}
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
|
||||||
|
#region Reasoning model settings
|
||||||
public class ReasoningSetting
|
public class ReasoningSetting
|
||||||
{
|
{
|
||||||
|
public bool IsDefault { get; set; }
|
||||||
public float Temperature { get; set; } = 1.0f;
|
public float Temperature { get; set; } = 1.0f;
|
||||||
public string? EffortLevel { get; set; }
|
public string? EffortLevel { get; set; }
|
||||||
}
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region Web search model settings
|
||||||
public class WebSearchSetting
|
public class WebSearchSetting
|
||||||
{
|
{
|
||||||
public bool IsDefault { get; set; }
|
public bool IsDefault { get; set; }
|
||||||
public string? SearchContextSize { get; set; }
|
public string? SearchContextSize { get; set; }
|
||||||
}
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
|
||||||
|
#region Image model settings
|
||||||
public class ImageSetting
|
public class ImageSetting
|
||||||
{
|
{
|
||||||
public ImageGenerationSetting? Generation { get; set; }
|
public ImageGenerationSetting? Generation { get; set; }
|
||||||
|
|
@ -99,6 +110,7 @@ public class ImageSetting
|
||||||
|
|
||||||
public class ImageGenerationSetting
|
public class ImageGenerationSetting
|
||||||
{
|
{
|
||||||
|
public bool IsDefault { get; set; }
|
||||||
public ModelSettingBase? Style { get; set; }
|
public ModelSettingBase? Style { get; set; }
|
||||||
public ModelSettingBase? Size { get; set; }
|
public ModelSettingBase? Size { get; set; }
|
||||||
public ModelSettingBase? Quality { get; set; }
|
public ModelSettingBase? Quality { get; set; }
|
||||||
|
|
@ -107,16 +119,18 @@ public class ImageGenerationSetting
|
||||||
|
|
||||||
public class ImageEditSetting
|
public class ImageEditSetting
|
||||||
{
|
{
|
||||||
|
public bool IsDefault { get; set; }
|
||||||
public ModelSettingBase? Size { get; set; }
|
public ModelSettingBase? Size { get; set; }
|
||||||
public ModelSettingBase? ResponseFormat { get; set; }
|
public ModelSettingBase? ResponseFormat { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class ImageVariationSetting
|
public class ImageVariationSetting
|
||||||
{
|
{
|
||||||
|
public bool IsDefault { get; set; }
|
||||||
public ModelSettingBase? Size { get; set; }
|
public ModelSettingBase? Size { get; set; }
|
||||||
public ModelSettingBase? ResponseFormat { get; set; }
|
public ModelSettingBase? ResponseFormat { get; set; }
|
||||||
}
|
}
|
||||||
|
#endregion
|
||||||
|
|
||||||
public class ModelSettingBase
|
public class ModelSettingBase
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -93,7 +93,7 @@ public partial class FileInstructService
|
||||||
}
|
}
|
||||||
|
|
||||||
var providerName = options.Provider ?? "openai";
|
var providerName = options.Provider ?? "openai";
|
||||||
var model = options?.Model ?? "gpt-4o-mini";
|
var model = options?.Model ?? "gpt-4.1-mini";
|
||||||
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
|
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
|
||||||
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
|
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ public class WebIntelligentSearchFn : IFunctionCallback
|
||||||
var agent = new Agent
|
var agent = new Agent
|
||||||
{
|
{
|
||||||
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
||||||
Name = fromAgent?.Name ?? "AI Agent",
|
Name = fromAgent?.Name ?? "Utility Assistant",
|
||||||
Instruction = "Please search the websites to handle user's request."
|
Instruction = "Please search the websites to handle user's request."
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -53,16 +53,8 @@ public class WebIntelligentSearchFn : IFunctionCallback
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var provider = "openai";
|
var (provider, model) = GetLlmProviderModel();
|
||||||
var defaultModel = "gpt-4o-mini-search-preview";
|
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
|
||||||
|
|
||||||
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
|
|
||||||
var models = llmProviderService.GetProviderModels(provider);
|
|
||||||
var webSearchModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)?.Name
|
|
||||||
?? models.FirstOrDefault(x => x.WebSearch != null)?.Name
|
|
||||||
?? defaultModel;
|
|
||||||
|
|
||||||
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: webSearchModel);
|
|
||||||
var response = await completion.GetChatCompletions(agent, dialogs);
|
var response = await completion.GetChatCompletions(agent, dialogs);
|
||||||
return response.Content;
|
return response.Content;
|
||||||
}
|
}
|
||||||
|
|
@ -73,4 +65,18 @@ public class WebIntelligentSearchFn : IFunctionCallback
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private (string, string) GetLlmProviderModel()
|
||||||
|
{
|
||||||
|
var provider = "openai";
|
||||||
|
var model = "gpt-4o-mini-search-preview";
|
||||||
|
|
||||||
|
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
|
||||||
|
var models = llmProviderService.GetProviderModels(provider);
|
||||||
|
var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)
|
||||||
|
?? models.FirstOrDefault(x => x.WebSearch != null);
|
||||||
|
|
||||||
|
model = foundModel?.Name ?? model;
|
||||||
|
return (provider, model);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,3 @@
|
||||||
using System.IO;
|
|
||||||
|
|
||||||
namespace BotSharp.Plugin.FileHandler.Functions;
|
namespace BotSharp.Plugin.FileHandler.Functions;
|
||||||
|
|
||||||
public class EditImageFn : IFunctionCallback
|
public class EditImageFn : IFunctionCallback
|
||||||
|
|
@ -30,6 +28,7 @@ public class EditImageFn : IFunctionCallback
|
||||||
var image = await SelectImage(descrpition);
|
var image = await SelectImage(descrpition);
|
||||||
var response = await GetImageEditGeneration(message, descrpition, image);
|
var response = await GetImageEditGeneration(message, descrpition, image);
|
||||||
message.Content = response;
|
message.Content = response;
|
||||||
|
message.StopCompletion = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -43,8 +42,8 @@ public class EditImageFn : IFunctionCallback
|
||||||
private void SetImageOptions()
|
private void SetImageOptions()
|
||||||
{
|
{
|
||||||
var state = _services.GetRequiredService<IConversationStateService>();
|
var state = _services.GetRequiredService<IConversationStateService>();
|
||||||
state.SetState("image_response_format", "bytes");
|
|
||||||
state.SetState("image_count", "1");
|
state.SetState("image_count", "1");
|
||||||
|
state.SetState("image_response_format", "bytes");
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<MessageFileModel?> SelectImage(string? description)
|
private async Task<MessageFileModel?> SelectImage(string? description)
|
||||||
|
|
@ -53,7 +52,8 @@ public class EditImageFn : IFunctionCallback
|
||||||
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
|
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
|
||||||
{
|
{
|
||||||
Description = description,
|
Description = description,
|
||||||
ContentTypes = new List<string> { MediaTypeNames.Image.Png }
|
IncludeBotFile = true,
|
||||||
|
ContentTypes = [MediaTypeNames.Image.Png]
|
||||||
});
|
});
|
||||||
return selecteds?.FirstOrDefault();
|
return selecteds?.FirstOrDefault();
|
||||||
}
|
}
|
||||||
|
|
@ -67,7 +67,8 @@ public class EditImageFn : IFunctionCallback
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-2");
|
var (provider, model) = GetLlmProviderModel();
|
||||||
|
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
|
||||||
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
|
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
|
||||||
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
|
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
|
||||||
var agent = new Agent
|
var agent = new Agent
|
||||||
|
|
@ -78,13 +79,15 @@ public class EditImageFn : IFunctionCallback
|
||||||
|
|
||||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||||
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
|
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
|
||||||
|
|
||||||
|
// To do: convert rgb to rgba image
|
||||||
using var stream = fileBinary.ToStream();
|
using var stream = fileBinary.ToStream();
|
||||||
stream.Position = 0;
|
stream.Position = 0;
|
||||||
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty);
|
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileFullName);
|
||||||
stream.Close();
|
stream.Close();
|
||||||
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
|
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
|
||||||
|
|
||||||
return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted.";
|
return $"Your image is successfylly editted.";
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|
@ -94,6 +97,20 @@ public class EditImageFn : IFunctionCallback
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private (string, string) GetLlmProviderModel()
|
||||||
|
{
|
||||||
|
var provider = "openai";
|
||||||
|
var model = "gpt-image-1";
|
||||||
|
|
||||||
|
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
|
||||||
|
var models = llmProviderService.GetProviderModels(provider);
|
||||||
|
var foundModel = models.FirstOrDefault(x => x.Image?.Edit?.IsDefault == true)
|
||||||
|
?? models.FirstOrDefault(x => x.Image?.Edit != null);
|
||||||
|
|
||||||
|
model = foundModel?.Name ?? model;
|
||||||
|
return (provider, model);
|
||||||
|
}
|
||||||
|
|
||||||
private void SaveGeneratedImage(ImageGeneration? image)
|
private void SaveGeneratedImage(ImageGeneration? image)
|
||||||
{
|
{
|
||||||
if (image == null) return;
|
if (image == null) return;
|
||||||
|
|
|
||||||
|
|
@ -25,16 +25,22 @@ public class GenerateImageFn : IFunctionCallback
|
||||||
SetImageOptions();
|
SetImageOptions();
|
||||||
|
|
||||||
var agentService = _services.GetRequiredService<IAgentService>();
|
var agentService = _services.GetRequiredService<IAgentService>();
|
||||||
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
|
|
||||||
var imageAgent = new Agent
|
Agent? fromAgent = null;
|
||||||
|
if (!string.IsNullOrEmpty(message.CurrentAgentId))
|
||||||
{
|
{
|
||||||
Id = agent?.Id ?? Guid.Empty.ToString(),
|
fromAgent = await agentService.GetAgent(message.CurrentAgentId);
|
||||||
Name = agent?.Name ?? "Unkown",
|
}
|
||||||
|
|
||||||
|
var agent = new Agent
|
||||||
|
{
|
||||||
|
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
||||||
|
Name = fromAgent?.Name ?? "Utility Assistant",
|
||||||
Instruction = args?.ImageDescription,
|
Instruction = args?.ImageDescription,
|
||||||
TemplateDict = new Dictionary<string, object>()
|
TemplateDict = new Dictionary<string, object>()
|
||||||
};
|
};
|
||||||
|
|
||||||
var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription);
|
var response = await GetImageGeneration(agent, message, args?.ImageDescription);
|
||||||
message.Content = response;
|
message.Content = response;
|
||||||
message.StopCompletion = true;
|
message.StopCompletion = true;
|
||||||
return true;
|
return true;
|
||||||
|
|
@ -58,7 +64,8 @@ public class GenerateImageFn : IFunctionCallback
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var completion = CompletionProvider.GetImageCompletion(_services, provider: "openai", model: "dall-e-3");
|
var (provider, model) = GetLlmProviderModel();
|
||||||
|
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model);
|
||||||
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
|
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
|
||||||
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
|
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
|
||||||
var result = await completion.GetImageGeneration(agent, dialog);
|
var result = await completion.GetImageGeneration(agent, dialog);
|
||||||
|
|
@ -73,6 +80,20 @@ public class GenerateImageFn : IFunctionCallback
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private (string, string) GetLlmProviderModel()
|
||||||
|
{
|
||||||
|
var provider = "openai";
|
||||||
|
var model = "gpt-image-1";
|
||||||
|
|
||||||
|
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
|
||||||
|
var models = llmProviderService.GetProviderModels(provider);
|
||||||
|
var foundModel = models.FirstOrDefault(x => x.Image?.Generation?.IsDefault == true)
|
||||||
|
?? models.FirstOrDefault(x => x.Image?.Generation != null);
|
||||||
|
|
||||||
|
model = foundModel?.Name ?? model;
|
||||||
|
return (provider, model);
|
||||||
|
}
|
||||||
|
|
||||||
private void SaveGeneratedImages(List<ImageGeneration>? images)
|
private void SaveGeneratedImages(List<ImageGeneration>? images)
|
||||||
{
|
{
|
||||||
if (images.IsNullOrEmpty()) return;
|
if (images.IsNullOrEmpty()) return;
|
||||||
|
|
|
||||||
|
|
@ -33,8 +33,8 @@ public class ReadImageFn : IFunctionCallback
|
||||||
|
|
||||||
var agent = new Agent
|
var agent = new Agent
|
||||||
{
|
{
|
||||||
Id = BuiltInAgentId.UtilityAssistant,
|
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
||||||
Name = "Utility Agent",
|
Name = fromAgent?.Name ?? "Utility Assistant",
|
||||||
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).",
|
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).",
|
||||||
TemplateDict = new Dictionary<string, object>()
|
TemplateDict = new Dictionary<string, object>()
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -33,13 +33,13 @@ public class ReadPdfFn : IFunctionCallback
|
||||||
Agent? fromAgent = null;
|
Agent? fromAgent = null;
|
||||||
if (!string.IsNullOrEmpty(message.CurrentAgentId))
|
if (!string.IsNullOrEmpty(message.CurrentAgentId))
|
||||||
{
|
{
|
||||||
fromAgent = await agentService.LoadAgent(message.CurrentAgentId);
|
fromAgent = await agentService.GetAgent(message.CurrentAgentId);
|
||||||
}
|
}
|
||||||
|
|
||||||
var agent = new Agent
|
var agent = new Agent
|
||||||
{
|
{
|
||||||
Id = BuiltInAgentId.UtilityAssistant,
|
Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
|
||||||
Name = "Utility Agent",
|
Name = fromAgent?.Name ?? "Utility Assistant",
|
||||||
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).",
|
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).",
|
||||||
TemplateDict = new Dictionary<string, object>()
|
TemplateDict = new Dictionary<string, object>()
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -52,9 +52,10 @@ public partial class ImageCompletionProvider
|
||||||
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
|
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
|
||||||
|
|
||||||
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
||||||
|
var state = _services.GetRequiredService<IConversationStateService>();
|
||||||
|
|
||||||
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit;
|
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Edit;
|
||||||
|
|
||||||
var state = _services.GetRequiredService<IConversationStateService>();
|
|
||||||
var size = state.GetState("image_size");
|
var size = state.GetState("image_size");
|
||||||
var responseFormat = state.GetState("image_response_format");
|
var responseFormat = state.GetState("image_response_format");
|
||||||
|
|
||||||
|
|
@ -71,7 +72,7 @@ public partial class ImageCompletionProvider
|
||||||
options.ResponseFormat = GetImageResponseFormat(responseFormat);
|
options.ResponseFormat = GetImageResponseFormat(responseFormat);
|
||||||
}
|
}
|
||||||
|
|
||||||
var count = GetImageCount(state.GetState("image_count", "1"));
|
var count = GetImageCount(state.GetState("image_count"));
|
||||||
return (prompt, count, options);
|
return (prompt, count, options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -30,9 +30,10 @@ public partial class ImageCompletionProvider
|
||||||
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
|
var prompt = message?.Payload ?? message?.Content ?? string.Empty;
|
||||||
|
|
||||||
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
||||||
|
var state = _services.GetRequiredService<IConversationStateService>();
|
||||||
|
|
||||||
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation;
|
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Generation;
|
||||||
|
|
||||||
var state = _services.GetRequiredService<IConversationStateService>();
|
|
||||||
var size = state.GetState("image_size");
|
var size = state.GetState("image_size");
|
||||||
var quality = state.GetState("image_quality");
|
var quality = state.GetState("image_quality");
|
||||||
var style = state.GetState("image_style");
|
var style = state.GetState("image_style");
|
||||||
|
|
@ -61,7 +62,7 @@ public partial class ImageCompletionProvider
|
||||||
options.ResponseFormat = GetImageResponseFormat(responseFormat);
|
options.ResponseFormat = GetImageResponseFormat(responseFormat);
|
||||||
}
|
}
|
||||||
|
|
||||||
var count = GetImageCount(state.GetState("image_count", "1"));
|
var count = GetImageCount(state.GetState("image_count"));
|
||||||
return (prompt, count, options);
|
return (prompt, count, options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -28,9 +28,10 @@ public partial class ImageCompletionProvider
|
||||||
private (int, ImageVariationOptions) PrepareVariationOptions()
|
private (int, ImageVariationOptions) PrepareVariationOptions()
|
||||||
{
|
{
|
||||||
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
||||||
|
var state = _services.GetRequiredService<IConversationStateService>();
|
||||||
|
|
||||||
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation;
|
var settings = settingsService.GetSetting(Provider, _model)?.Image?.Variation;
|
||||||
|
|
||||||
var state = _services.GetRequiredService<IConversationStateService>();
|
|
||||||
var size = state.GetState("image_size");
|
var size = state.GetState("image_size");
|
||||||
var responseFormat = state.GetState("image_response_format");
|
var responseFormat = state.GetState("image_response_format");
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue