add image generation

src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs

@@ -24,4 +24,7 @@ public interface IChatCompletion
     Task<bool> GetChatCompletionsStreamingAsync(Agent agent, 
         List<RoleDialogModel> conversations, 
         Func<RoleDialogModel, Task> onMessageReceived);
+
+    Task<RoleDialogModel> GetImageGeneration(Agent agent,
+        List<RoleDialogModel> conversations);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -51,5 +51,6 @@ public class LlmModelSetting
 public enum LlmModelType
 {
     Text = 1,
-    Chat = 2
+    Chat = 2,
+    Image = 3
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -103,4 +103,36 @@ public class InstructModeController : ControllerBase
             return $"Error in analyzing files.";
         }
     }
+
+    [HttpPost("/instruct/image-generation")]
+    public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
+    {
+        var state = _services.GetRequiredService<IConversationStateService>();
+        input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
+        var imageViewModel = new ImageGenerationViewModel();
+
+        try
+        {
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",
+                modelId: input.ModelId ?? "dall-e");
+            var message = await completion.GetImageGeneration(new Agent()
+            {
+                Id = Guid.Empty.ToString(),
+            }, new List<RoleDialogModel>
+            {
+                new RoleDialogModel(AgentRole.User, input.Text)
+            });
+            
+            imageViewModel.Content = message.Content;
+            imageViewModel.Data = message.Data;
+            return imageViewModel;
+        }
+        catch (Exception ex)
+        {
+            var error = "Error in image generation.";
+            _logger.LogError($"{error} {ex.Message}");
+            imageViewModel.Message = error;
+            return imageViewModel;
+        }
+    }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs

@@ -0,0 +1,18 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Instructs;
+
+public class ImageGenerationViewModel
+{
+    [JsonPropertyName("content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Content { get; set; }
+
+    [JsonPropertyName("data")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public object? Data { get; set; }
+
+    [JsonPropertyName("message")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Message { get; set; }
+}

src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs

@@ -264,4 +264,9 @@ public class ChatCompletionProvider : IChatCompletion
     {
         _model = model;
     }
+
+    public Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -446,4 +446,62 @@ public class ChatCompletionProvider : IChatCompletion
         functionResultData = "31 celsius";
         return new ChatRequestToolMessage(functionResultData.ToString(), toolCall.Id);
     }
+
+    public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
+        foreach (var hook in contentHooks)
+        {
+            await hook.BeforeGenerating(agent, conversations);
+        }
+
+        var client = ProviderHelper.GetClient(Provider, _model, _services);
+        var options = BuildImageGenerationOptions(conversations);
+        var response = await client.GetImageGenerationsAsync(options);
+        var image = response.Value.Data.First();
+
+        var content = string.Empty;
+        if (!string.IsNullOrEmpty(image.RevisedPrompt))
+        {
+            content = image.RevisedPrompt;
+        }
+
+        var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
+        {
+            CurrentAgentId = agent.Id,
+            MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
+            Data = image.Url.AbsoluteUri ?? image.Base64Data
+        };
+
+        foreach (var hook in contentHooks)
+        {
+            await hook.AfterGenerated(responseMessage, new TokenStatsModel
+            {
+                Prompt = options.Prompt,
+                Provider = Provider,
+                Model = _model,
+                PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(),
+                CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count()
+            });
+        }
+
+        return responseMessage;
+    }
+
+    private ImageGenerationOptions BuildImageGenerationOptions(List<RoleDialogModel> conversations)
+    {
+        var state = _services.GetRequiredService<IConversationStateService>();
+
+        var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024";
+        var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard";
+
+        var options = new ImageGenerationOptions
+        {
+            DeploymentName = _model,
+            Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty,
+            Size = new ImageSize(sizeValue),
+            Quality = new ImageGenerationQuality(qualityValue)
+        };
+        return options;
+    }
 }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs

@@ -149,4 +149,9 @@ public class ChatCompletionProvider : IChatCompletion
     {
         _model = model;
     }
+
+    public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs

@@ -139,4 +139,9 @@ public class ChatCompletionProvider : IChatCompletion
 
         return msg;
     }
+
+    public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs

@@ -191,4 +191,9 @@ public class ChatCompletionProvider : IChatCompletion
     {
         _model = model;
     }
+
+    public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs

@@ -231,6 +231,11 @@ public class ChatCompletionProvider : IChatCompletion
         throw new NotImplementedException();
     }
 
+    public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        throw new NotImplementedException();
+    }
+
     public void SetModelName(string model)
     {
         _model = model;

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs

@@ -102,5 +102,10 @@ namespace BotSharp.Plugin.SemanticKernel
         {
             _model = model;
         }
+
+        public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs

@@ -1,8 +1,6 @@
 using BotSharp.Abstraction.Agents;
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.Loggers;
-using BotSharp.Abstraction.Routing;
-using Sdcb.SparkDesk.ResponseInternals;
 
 namespace BotSharp.Plugin.SparkDesk.Providers;
 
@@ -270,4 +268,9 @@ public class ChatCompletionProvider : IChatCompletion
         FunctionDef functionDef = new FunctionDef(def.Name, def.Description, fundef.ToArray());
         return functionDef;
     }
+
+    public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
+    {
+        throw new NotImplementedException();
+    }
 }