diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index eaac3b59..baaeff2e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -6,7 +6,6 @@ public interface IBotSharpFileService Task> GetChatImages(string conversationId, string source, IEnumerable fileTypes, List conversations, int? offset = null); IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); - bool HasConversationUserFiles(string conversationId); bool SaveMessageFiles(string conversationId, string messageId, string source, List files); string GetUserAvatar(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs index 538f45e8..533a8389 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs @@ -3,8 +3,14 @@ namespace BotSharp.Abstraction.Files.Models; public class LlmFileContext { [JsonPropertyName("user_request")] - public string UserRequest { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserRequest { get; set; } [JsonPropertyName("file_types")] - public string FileTypes { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileTypes { get; set; } + + [JsonPropertyName("image_description")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageDescription { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs index 6853f055..6c330cc9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/ListExtenstions.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Utilities; public static class ListExtenstions { - public static bool IsNullOrEmpty(this IEnumerable list) + public static bool IsNullOrEmpty(this IEnumerable? list) { return list == null || !list.Any(); } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 558663ef..2ca27817 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -47,8 +47,10 @@ + + @@ -163,6 +165,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index 6bed7e2a..4ddb3ab5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -19,5 +19,7 @@ public class FilePlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/GenerateImageFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/GenerateImageFn.cs new file mode 100644 index 00000000..b7d89b36 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/GenerateImageFn.cs @@ -0,0 +1,125 @@ +using BotSharp.Abstraction.Functions; +using System.Net.Http; + +namespace BotSharp.Core.Files.Functions; + +public class GenerateImageFn : IFunctionCallback +{ + public string Name => "generate_image"; + public string Indication => "Generating image"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); + private string _conversationId; + private string _messageId; + + public GenerateImageFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + Init(message); + SetImageOptions(); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(UTILITY_ASSISTANT); + var imageAgent = new Agent + { + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unkown", + Instruction = args?.ImageDescription, + TemplateDict = new Dictionary() + }; + + var response = await GetImageGeneration(imageAgent, message, args?.ImageDescription); + message.Content = response; + message.StopCompletion = true; + return true; + } + + private void Init(RoleDialogModel message) + { + var convService = _services.GetRequiredService(); + _conversationId = convService.ConversationId; + _messageId = message.MessageId; + } + + private void SetImageOptions() + { + var state = _services.GetRequiredService(); + var size = state.SetState("image_size", "1024x1024"); + var quality = state.SetState("image_quality", "standard"); + var style = state.SetState("image_style", "natural"); + var format = state.SetState("image_format", "bytes"); + var count = state.SetState("image_count", "1"); + } + + private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) + { + try + { + var completion = CompletionProvider.GetImageGeneration(_services, provider: "openai", model: "dall-e-3", imageGenerate: true); + var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; + var dialog = RoleDialogModel.From(message, AgentRole.User, text); + var result = await completion.GetImageGeneration(agent, new List { dialog }); + await SaveGeneratedImages(result?.GeneratedImages); + return result?.Content ?? string.Empty; + } + catch (Exception ex) + { + var error = $"Error when generating image."; + _logger.LogWarning($"{error} {ex.Message}"); + return error; + } + } + + private async Task SaveGeneratedImages(List? images) + { + if (images.IsNullOrEmpty()) return; + + var files = new List(); + foreach (var image in images) + { + if (string.IsNullOrEmpty(image?.ImageUrl) + && string.IsNullOrEmpty(image?.ImageData)) + { + continue; + } + + try + { + var data = image.ImageData; + if (!string.IsNullOrEmpty(image.ImageUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + var bytes = await client.GetByteArrayAsync(image.ImageUrl); + data = Convert.ToBase64String(bytes); + } + + if (!string.IsNullOrEmpty(data)) + { + var imageName = $"{Guid.NewGuid().ToString()}.png"; + var imageData = $"data:image/png;base64,{data}"; + files.Add(new BotSharpFile { FileName = imageName, FileData = imageData }); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving generated image: {image.ImageUrl ?? image.ImageData}\r\n{ex.Message}"); + continue; + } + } + + var fileService = _services.GetRequiredService(); + fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/FileAnalyzerHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/FileAnalyzerHook.cs index 87c90cf6..cded3e62 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/FileAnalyzerHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/FileAnalyzerHook.cs @@ -3,6 +3,7 @@ namespace BotSharp.Core.Files.Hooks; public class FileAnalyzerHook : AgentHookBase { private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); + private static string FUNCTION_NAME = "load_attachment"; public override string SelfId => string.Empty; @@ -43,11 +44,10 @@ public class FileAnalyzerHook : AgentHookBase private (string, FunctionDef?) GetPromptAndFunction() { - var fn = "load_attachment"; var db = _services.GetRequiredService(); var agent = db.GetAgent(UTILITY_ASSISTANT); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fn}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fn)); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); return (prompt, loadAttachmentFn); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorHook.cs new file mode 100644 index 00000000..f3702b3e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorHook.cs @@ -0,0 +1,53 @@ +namespace BotSharp.Core.Files.Hooks; + +public class ImageGeneratorHook : AgentHookBase +{ + private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); + private static string FUNCTION_NAME = "generate_image"; + + public override string SelfId => string.Empty; + + public ImageGeneratorHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(AgentUtility.ImageGenerator); + + if (isConvMode && isEnabled) + { + var (prompt, fn) = GetPromptAndFunction(); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, FunctionDef?) GetPromptAndFunction() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(UTILITY_ASSISTANT); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); + return (prompt, loadAttachmentFn); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorUtilityHook.cs new file mode 100644 index 00000000..a7fdbb51 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/ImageGeneratorUtilityHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Core.Files.Hooks; + +internal class ImageGeneratorUtilityHook : IAgentUtilityHook +{ + public void AddUtilities(List utilities) + { + utilities.Add(AgentUtility.ImageGenerator); + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs index ee7476bc..34f6b74a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs @@ -181,16 +181,6 @@ public partial class BotSharpFileService return found; } - public bool HasConversationUserFiles(string conversationId) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); - if (!ExistDirectory(dir)) return false; - - return Directory.GetDirectories(dir).Any(); - } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions/generate_image.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions/generate_image.json new file mode 100644 index 00000000..d9c8b70e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions/generate_image.json @@ -0,0 +1,14 @@ +{ + "name": "generate_image", + "description": "If the user requests you providing or generating image or picture, you can call this function to generate image.", + "parameters": { + "type": "object", + "properties": { + "image_description": { + "type": "string", + "description": "The image description that user requests." + } + }, + "required": [ "image_description" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/generate_image.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/generate_image.fn.liquid new file mode 100644 index 00000000..4ab76c60 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/generate_image.fn.liquid @@ -0,0 +1 @@ +Please call generate_image if user wants you to provide or generate an image or picture. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs index ae7adea0..891ebc19 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs @@ -9,6 +9,7 @@ namespace BotSharp.Plugin.HttpHandler.Hooks; public class HttpHandlerHook : AgentHookBase { private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); + private static string FUNCTION_NAME = "handle_http_request"; public override string SelfId => string.Empty; @@ -49,11 +50,10 @@ public class HttpHandlerHook : AgentHookBase private (string, FunctionDef?) GetPromptAndFunction() { - var fn = "handle_http_request"; var db = _services.GetRequiredService(); var agent = db.GetAgent(UTILITY_ASSISTANT); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fn}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fn)); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); return (prompt, loadAttachmentFn); } }