add image generator

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -6,7 +6,6 @@ public interface IBotSharpFileService
     Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source, IEnumerable<string> fileTypes, List<RoleDialogModel> conversations, int? offset = null);
     IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
     string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
-    bool HasConversationUserFiles(string conversationId);
     bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
 
     string GetUserAvatar();

src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs

@@ -3,8 +3,14 @@ namespace BotSharp.Abstraction.Files.Models;
 public class LlmFileContext
 {
     [JsonPropertyName("user_request")]
-    public string UserRequest { get; set; }
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? UserRequest { get; set; }
 
     [JsonPropertyName("file_types")]
-    public string FileTypes { get; set; }
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileTypes { get; set; }
+
+    [JsonPropertyName("image_description")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ImageDescription { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs

@@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Utilities;
 
 public static class ListExtenstions
 {
-    public static bool IsNullOrEmpty<T>(this IEnumerable<T> list)
+    public static bool IsNullOrEmpty<T>(this IEnumerable<T>? list)
     {
         return list == null || !list.Any();
     }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -47,8 +47,10 @@
 
   <ItemGroup>
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\agent.json" />
+    <None Remove="data\agents\00000000-0000-0000-0000-000000000000\functions\generate_image.json" />
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\instruction.liquid" />
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\functions\load_attachment.json" />
+    <None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\generate_image.fn.liquid" />
     <None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment.fn.liquid" />
     <None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
     <None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions.json" />
@@ -163,6 +165,12 @@
     <Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\00000000-0000-0000-0000-000000000000\functions\generate_image.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\generate_image.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\plugins\config.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs

@@ -19,5 +19,7 @@ public class FilePlugin : IBotSharpPlugin
 
         services.AddScoped<IAgentHook, FileAnalyzerHook>();
         services.AddScoped<IAgentUtilityHook, FileAnalyzerUtilityHook>();
+        services.AddScoped<IAgentHook, ImageGeneratorHook>();
+        services.AddScoped<IAgentUtilityHook, ImageGeneratorUtilityHook>();
     }
 }

src/Infrastructure/BotSharp.Core/Files/Functions/GenerateImageFn.cs

@@ -0,0 +1,125 @@
+using BotSharp.Abstraction.Functions;
+using System.Net.Http;
+
+namespace BotSharp.Core.Files.Functions;
+
+public class GenerateImageFn : IFunctionCallback
+{
+    public string Name => "generate_image";
+    public string Indication => "Generating image";
+
+    private readonly IServiceProvider _services;
+    private readonly ILogger<GenerateImageFn> _logger;
+    private static string UTILITY_ASSISTANT = Guid.Empty.ToString();
+    private string _conversationId;
+    private string _messageId;
+
+    public GenerateImageFn(
+        IServiceProvider services,
+        ILogger<GenerateImageFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<LlmFileContext>(message.FunctionArgs);
+        Init(message);
+        SetImageOptions();
+        
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(UTILITY_ASSISTANT);
+        var imageAgent = new Agent
+        {
+            Id = agent?.Id ?? Guid.Empty.ToString(),
+            Name = agent?.Name ?? "Unkown",
+            Instruction = args?.ImageDescription,
+            TemplateDict = new Dictionary<string, object>()
+        };
+
+        var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription);
+        message.Content = response;
+        message.StopCompletion = true;
+        return true;
+    }
+
+    private void Init(RoleDialogModel message)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        _conversationId = convService.ConversationId;
+        _messageId = message.MessageId;
+    }
+
+    private void SetImageOptions()
+    {
+        var state = _services.GetRequiredService<IConversationStateService>();
+        var size = state.SetState("image_size", "1024x1024");
+        var quality = state.SetState("image_quality", "standard");
+        var style = state.SetState("image_style", "natural");
+        var format = state.SetState("image_format", "bytes");
+        var count = state.SetState("image_count", "1");
+    }
+
+    private async Task<string> GetImageGeneration(Agent agent, RoleDialogModel message, string? description)
+    {
+        try
+        {
+            var completion = CompletionProvider.GetImageGeneration(_services, provider: "openai", model: "dall-e-3", imageGenerate: true);
+            var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
+            var dialog = RoleDialogModel.From(message, AgentRole.User, text);
+            var result = await completion.GetImageGeneration(agent, new List<RoleDialogModel> { dialog });
+            await SaveGeneratedImages(result?.GeneratedImages);
+            return result?.Content ?? string.Empty;
+        }
+        catch (Exception ex)
+        {
+            var error = $"Error when generating image.";
+            _logger.LogWarning($"{error} {ex.Message}");
+            return error;
+        }
+    }
+
+    private async Task SaveGeneratedImages(List<ImageGeneration>? images)
+    {
+        if (images.IsNullOrEmpty()) return;
+
+        var files = new List<BotSharpFile>();
+        foreach (var image in images)
+        {
+            if (string.IsNullOrEmpty(image?.ImageUrl)
+                && string.IsNullOrEmpty(image?.ImageData))
+            {
+                continue;
+            }
+
+            try
+            {
+                var data = image.ImageData;
+                if (!string.IsNullOrEmpty(image.ImageUrl))
+                {
+                    var http = _services.GetRequiredService<IHttpClientFactory>();
+                    using var client = http.CreateClient();
+                    var bytes = await client.GetByteArrayAsync(image.ImageUrl);
+                    data = Convert.ToBase64String(bytes);
+                }
+
+                if (!string.IsNullOrEmpty(data))
+                {
+                    var imageName = $"{Guid.NewGuid().ToString()}.png";
+                    var imageData = $"data:image/png;base64,{data}";
+                    files.Add(new BotSharpFile { FileName = imageName, FileData = imageData });
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning($"Error when saving generated image: {image.ImageUrl ?? image.ImageData}\r\n{ex.Message}");
+                continue;
+            }
+        }
+
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
+    }
+}

src/Infrastructure/BotSharp.Core/Files/Hooks/FileAnalyzerHook.cs

@@ -3,6 +3,7 @@ namespace BotSharp.Core.Files.Hooks;
 public class FileAnalyzerHook : AgentHookBase
 {
     private static string UTILITY_ASSISTANT = Guid.Empty.ToString();
+    private static string FUNCTION_NAME = "load_attachment";
 
     public override string SelfId => string.Empty;
 
@@ -43,11 +44,10 @@ public class FileAnalyzerHook : AgentHookBase
 
     private (string, FunctionDef?) GetPromptAndFunction()
     {
-        var fn = "load_attachment";
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var agent = db.GetAgent(UTILITY_ASSISTANT);
-        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fn}.fn"))?.Content ?? string.Empty;
-        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fn));
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty;
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME));
         return (prompt, loadAttachmentFn);
     }
 }

src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorHook.cs

@@ -0,0 +1,53 @@
+namespace BotSharp.Core.Files.Hooks;
+
+public class ImageGeneratorHook : AgentHookBase
+{
+    private static string UTILITY_ASSISTANT = Guid.Empty.ToString();
+    private static string FUNCTION_NAME = "generate_image";
+
+    public override string SelfId => string.Empty;
+
+    public ImageGeneratorHook(IServiceProvider services, AgentSettings settings)
+        : base(services, settings)
+    {
+    }
+
+    public override void OnAgentLoaded(Agent agent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var isConvMode = conv.IsConversationMode();
+        var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(AgentUtility.ImageGenerator);
+
+        if (isConvMode && isEnabled)
+        {
+            var (prompt, fn) = GetPromptAndFunction();
+            if (fn != null)
+            {
+                if (!string.IsNullOrWhiteSpace(prompt))
+                {
+                    agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+                }
+
+                if (agent.Functions == null)
+                {
+                    agent.Functions = new List<FunctionDef> { fn };
+                }
+                else
+                {
+                    agent.Functions.Add(fn);
+                }
+            }
+        }
+
+        base.OnAgentLoaded(agent);
+    }
+
+    private (string, FunctionDef?) GetPromptAndFunction()
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agent = db.GetAgent(UTILITY_ASSISTANT);
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty;
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME));
+        return (prompt, loadAttachmentFn);
+    }
+}

src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorUtilityHook.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Core.Files.Hooks;
+
+internal class ImageGeneratorUtilityHook : IAgentUtilityHook
+{
+    public void AddUtilities(List<string> utilities)
+    {
+        utilities.Add(AgentUtility.ImageGenerator);
+    }
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs

@@ -181,16 +181,6 @@ public partial class BotSharpFileService
         return found;
     }
 
-    public bool HasConversationUserFiles(string conversationId)
-    {
-        if (string.IsNullOrEmpty(conversationId)) return false;
-
-        var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
-        if (!ExistDirectory(dir)) return false;
-
-        return Directory.GetDirectories(dir).Any();
-    }
-
     public bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files)
     {
         if (files.IsNullOrEmpty()) return false;

src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions/generate_image.json

@@ -0,0 +1,14 @@
+{
+  "name": "generate_image",
+  "description": "If the user requests you providing or generating image or picture, you can call this function to generate image.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "image_description": {
+        "type": "string",
+        "description": "The image description that user requests."
+      }
+    },
+    "required": [ "image_description" ]
+  }
+}

src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/generate_image.fn.liquid

@@ -0,0 +1 @@
+Please call generate_image if user wants you to provide or generate an image or picture.

src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs

@@ -9,6 +9,7 @@ namespace BotSharp.Plugin.HttpHandler.Hooks;
 public class HttpHandlerHook : AgentHookBase
 {
     private static string UTILITY_ASSISTANT = Guid.Empty.ToString();
+    private static string FUNCTION_NAME = "handle_http_request";
 
     public override string SelfId => string.Empty;
 
@@ -49,11 +50,10 @@ public class HttpHandlerHook : AgentHookBase
 
     private (string, FunctionDef?) GetPromptAndFunction()
     {
-        var fn = "handle_http_request";
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var agent = db.GetAgent(UTILITY_ASSISTANT);
-        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fn}.fn"))?.Content ?? string.Empty;
-        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fn));
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty;
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME));
         return (prompt, loadAttachmentFn);
     }
 }