diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index c5332b40..306e72e3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -24,7 +24,4 @@ public interface IChatCompletion Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived); - - Task GetImageGeneration(Agent agent, - List conversations); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs new file mode 100644 index 00000000..b257e114 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.MLTasks; + +public interface IImageGeneration +{ + /// + /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI + /// + string Provider { get; } + + /// + /// Set model name, one provider can consume different model or version(s) + /// + /// deployment name + void SetModelName(string model); + + Task GetImageGeneration(Agent agent, List conversations); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 20762fe0..1e203333 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 0a0bd42a..e8256417 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -32,6 +32,11 @@ public class LlmModelSetting /// public bool MultiModal { get; set; } + /// + /// If true, allow generating images + /// + public bool ImageGeneration { get; set; } + /// /// Prompt cost per 1K token /// diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 4655b1de..3c116c08 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -55,42 +55,6 @@ public class CompletionProvider return completer; } - private static (string, string) GetProviderAndModel(IServiceProvider services, - string? provider = null, - string? model = null, - string? modelId = null, - bool? multiModal = null, - AgentLlmConfig? agentConfig = null) - { - var agentSetting = services.GetRequiredService(); - var state = services.GetRequiredService(); - - if (string.IsNullOrEmpty(provider)) - { - provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider; - provider = state.GetState("provider", provider ?? "azure-openai"); - } - - if (string.IsNullOrEmpty(model)) - { - model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model; - if (state.ContainsState("model")) - { - model = state.GetState("model", model ?? "gpt-35-turbo-4k"); - } - else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) - { - var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; - var llmProviderService = services.GetRequiredService(); - model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name; - } - } - - state.SetState("provider", provider); - state.SetState("model", model); - - return (provider, model); - } public static ITextCompletion GetTextCompletion(IServiceProvider services, string? provider = null, string? model = null, @@ -111,4 +75,66 @@ public class CompletionProvider return completer; } + + public static IImageGeneration GetImageGeneration(IServiceProvider services, + string? provider = null, + string? model = null, + string? modelId = null, + bool imageGenerate = false, + AgentLlmConfig? agentConfig = null) + { + var completions = services.GetServices(); + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, + imageGenerate: imageGenerate, agentConfig: agentConfig); + + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve completion provider by {provider}"); + } + + completer?.SetModelName(model); + + return completer; + } + + private static (string, string) GetProviderAndModel(IServiceProvider services, + string? provider = null, + string? model = null, + string? modelId = null, + bool? multiModal = null, + bool imageGenerate = false, + AgentLlmConfig? agentConfig = null) + { + var agentSetting = services.GetRequiredService(); + var state = services.GetRequiredService(); + + if (string.IsNullOrEmpty(provider)) + { + provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider; + provider = state.GetState("provider", provider ?? "azure-openai"); + } + + if (string.IsNullOrEmpty(model)) + { + model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model; + if (state.ContainsState("model")) + { + model = state.GetState("model", model ?? "dall-e-3"); + } + else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) + { + var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; + var llmProviderService = services.GetRequiredService(); + model = llmProviderService.GetProviderModel(provider, modelIdentity, + multiModal: multiModal, imageGenerate: imageGenerate)?.Name; + } + } + + state.SetState("provider", provider); + state.SetState("model", model); + + return (provider, model); + } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 8320bdb7..b3e86e58 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false) { var models = GetProviderModels(provider) .Where(x => x.Id == id); @@ -54,6 +54,8 @@ public class LlmProviderService : ILlmProviderService models = models.Where(x => x.MultiModal == multiModal); } + models = models.Where(x => x.ImageGeneration == imageGenerate); + var random = new Random(); var index = random.Next(0, models.Count()); var modelSetting = models.ElementAt(index); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 0b013456..bcbd1e8a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -99,8 +99,9 @@ public class InstructModeController : ControllerBase } catch (Exception ex) { - _logger.LogError($"Error in analyzing files. {ex.Message}"); - return $"Error in analyzing files."; + var error = $"Error in analyzing files. {ex.Message}"; + _logger.LogError(error); + return error; } } @@ -113,8 +114,8 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - modelId: input.ModelId ?? "dall-e"); + var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai", + modelId: input.ModelId ?? "dall-e", imageGenerate: true); var message = await completion.GetImageGeneration(new Agent() { Id = Guid.Empty.ToString(), @@ -129,8 +130,8 @@ public class InstructModeController : ControllerBase } catch (Exception ex) { - var error = "Error in image generation."; - _logger.LogError($"{error} {ex.Message}"); + var error = $"Error in image generation. {ex.Message}"; + _logger.LogError(error); imageViewModel.Message = error; return imageViewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 94de179e..5571d9fa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -24,6 +24,7 @@ public class LlmProviderController : ControllerBase [HttpGet("/llm-provider/{provider}/models")] public IEnumerable GetLlmProviderModels([FromRoute] string provider) { - return _llmProvider.GetProviderModels(provider); + var list = _llmProvider.GetProviderModels(provider); + return list.Where(x => !x.ImageGeneration); } } diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 8977e04d..87202109 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -264,9 +264,4 @@ public class ChatCompletionProvider : IChatCompletion { _model = model; } - - public Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index b3decd7b..fb04f961 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -29,5 +29,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index d85b4d65..50fd33ce 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -446,62 +446,4 @@ public class ChatCompletionProvider : IChatCompletion functionResultData = "31 celsius"; return new ChatRequestToolMessage(functionResultData.ToString(), toolCall.Id); } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - var contentHooks = _services.GetServices().ToList(); - foreach (var hook in contentHooks) - { - await hook.BeforeGenerating(agent, conversations); - } - - var client = ProviderHelper.GetClient(Provider, _model, _services); - var options = BuildImageGenerationOptions(conversations); - var response = await client.GetImageGenerationsAsync(options); - var image = response.Value.Data.First(); - - var content = string.Empty; - if (!string.IsNullOrEmpty(image.RevisedPrompt)) - { - content = image.RevisedPrompt; - } - - var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) - { - CurrentAgentId = agent.Id, - MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - Data = image.Url.AbsoluteUri ?? image.Base64Data - }; - - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(responseMessage, new TokenStatsModel - { - Prompt = options.Prompt, - Provider = Provider, - Model = _model, - PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(), - CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count() - }); - } - - return responseMessage; - } - - private ImageGenerationOptions BuildImageGenerationOptions(List conversations) - { - var state = _services.GetRequiredService(); - - var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024"; - var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard"; - - var options = new ImageGenerationOptions - { - DeploymentName = _model, - Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty, - Size = new ImageSize(sizeValue), - Quality = new ImageGenerationQuality(qualityValue) - }; - return options; - } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs new file mode 100644 index 00000000..c0211f11 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs @@ -0,0 +1,104 @@ +using Azure.AI.OpenAI; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class ImageGenerationProvider : IImageGeneration +{ + protected readonly AzureOpenAiSettings _settings; + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + + protected string _model; + + public virtual string Provider => "azure-openai"; + + public ImageGenerationProvider( + AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) + { + _settings = settings; + _services = services; + _logger = logger; + } + + + public async Task GetImageGeneration(Agent agent, List conversations) + { + var contentHooks = _services.GetServices().ToList(); + + // Before + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var options = PrepareOptions(conversations); + var response = await client.GetImageGenerationsAsync(options); + var image = response.Value.Data.First(); + + var content = string.Empty; + if (!string.IsNullOrEmpty(image.RevisedPrompt)) + { + content = image.RevisedPrompt; + } + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + Data = image.Url.AbsoluteUri ?? image.Base64Data + }; + + // After + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = options.Prompt, + Provider = Provider, + Model = _model, + PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(), + CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count() + }); + } + + return responseMessage; + } + + private ImageGenerationOptions PrepareOptions(List conversations) + { + var state = _services.GetRequiredService(); + + var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024"; + var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard"; + + var options = new ImageGenerationOptions + { + DeploymentName = _model, + Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty, + Size = new ImageSize(sizeValue), + Quality = new ImageGenerationQuality(qualityValue) + }; + return options; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs new file mode 100644 index 00000000..5a02b6ae --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs @@ -0,0 +1,16 @@ +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.Logging; +using System; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class OpenAiImageGenerationProvider : ImageGenerationProvider +{ + public override string Provider => "openai"; + + public OpenAiImageGenerationProvider(AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) : base(settings, logger, services) + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index c4628c6d..d278b110 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -149,9 +149,4 @@ public class ChatCompletionProvider : IChatCompletion { _model = model; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 80c950c8..88b37dbb 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -139,9 +139,4 @@ public class ChatCompletionProvider : IChatCompletion return msg; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index cdd96a0b..0db444ce 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -191,9 +191,4 @@ public class ChatCompletionProvider : IChatCompletion { _model = model; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index 7010842d..c702a7ec 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -231,11 +231,6 @@ public class ChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } - public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index cb70b189..156f238c 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -102,10 +102,5 @@ namespace BotSharp.Plugin.SemanticKernel { _model = model; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 25e588c3..c4556c36 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -268,9 +268,4 @@ public class ChatCompletionProvider : IChatCompletion FunctionDef functionDef = new FunctionDef(def.Name, def.Description, fundef.ToArray()); return functionDef; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } }