diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs index 61cb0f4d..bbcc51ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs @@ -16,7 +16,8 @@ public enum AgentField Template, Response, Sample, - LlmConfig + LlmConfig, + Tool } public enum AgentTaskField diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs new file mode 100644 index 00000000..875062f8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Agents.Enums; + +public class AgentTool +{ + public const string FileAnalyzer = "file-analyzer"; + public const string ImageGenerator = "image-generator"; + public const string HttpHandler = "http-handler"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index b435a4d1..c2368cd2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -51,4 +51,6 @@ public interface IAgentService List GetAgentsByUser(string userId); PluginDef GetPlugin(string agentId); + + IEnumerable GetAgentTools(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 7b48f452..c1f82e7b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -90,6 +90,12 @@ public class Agent public List Profiles { get; set; } = new List(); + /// + /// Useful tools + /// + public List Tools { get; set; } + = new List(); + /// /// Inherit from agent /// @@ -121,6 +127,7 @@ public class Agent Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, + Tools = agent.Tools, Knowledges = agent.Knowledges, IsPublic = agent.IsPublic, Disabled = agent.Disabled, @@ -162,6 +169,12 @@ public class Agent return this; } + public Agent SetTools(List tools) + { + Tools = tools ?? new List(); + return this; + } + public Agent SetResponses(List responses) { Responses = responses ?? new List(); ; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 8434359e..9fd40c09 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -58,4 +58,6 @@ public interface IConversationService Task GetConversationSummary(IEnumerable conversationId); Task GetConversationRecordOrCreateNew(string agentId); + + bool IsConversationMode(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs new file mode 100644 index 00000000..b257e114 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.MLTasks; + +public interface IImageGeneration +{ + /// + /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI + /// + string Provider { get; } + + /// + /// Set model name, one provider can consume different model or version(s) + /// + /// deployment name + void SetModelName(string model); + + Task GetImageGeneration(Agent agent, List conversations); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 20762fe0..1e203333 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index b86578fe..e8256417 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -32,6 +32,11 @@ public class LlmModelSetting /// public bool MultiModal { get; set; } + /// + /// If true, allow generating images + /// + public bool ImageGeneration { get; set; } + /// /// Prompt cost per 1K token /// @@ -51,5 +56,6 @@ public class LlmModelSetting public enum LlmModelType { Text = 1, - Chat = 2 + Chat = 2, + Image = 3 } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 6c2d7ea2..e3f1774e 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -7,7 +7,7 @@ public partial class AgentService [MemoryCache(10 * 60, perInstanceCache: true)] public async Task LoadAgent(string id) { - if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) + if (string.IsNullOrEmpty(id)) { return null; } @@ -28,7 +28,7 @@ public partial class AgentService var agent = await GetAgent(id); if (agent == null) { - throw new Exception($"Can't load agent by id: {id}"); + return null; } if (agent.InheritAgentId != null) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index daab8916..ecf2ecfa 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -1,8 +1,6 @@ -using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Repositories.Enums; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Users.Enums; -using Microsoft.EntityFrameworkCore.Metadata; using System.IO; namespace BotSharp.Core.Agents.Services; @@ -34,6 +32,7 @@ public partial class AgentService record.Templates = agent.Templates ?? new List(); record.Responses = agent.Responses ?? new List(); record.Samples = agent.Samples ?? new List(); + record.Tools = agent.Tools ?? new List(); if (agent.LlmConfig != null && !agent.LlmConfig.IsInherit) { record.LlmConfig = agent.LlmConfig; @@ -95,6 +94,7 @@ public partial class AgentService .SetFunctions(foundAgent.Functions) .SetResponses(foundAgent.Responses) .SetSamples(foundAgent.Samples) + .SetTools(foundAgent.Tools) .SetLlmConfig(foundAgent.LlmConfig); _db.UpdateAgent(clonedAgent, AgentField.All); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs index da1b8cf1..69e67c30 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Reflection; namespace BotSharp.Core.Agents.Services; @@ -53,4 +54,14 @@ public partial class AgentService : IAgentService var agents = _db.GetAgentsByUser(userId); return agents; } + + public IEnumerable GetAgentTools() + { + var tools = typeof(AgentTool).GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(f => f.IsLiteral && f.FieldType == typeof(string)) + .Select(x => x.GetRawConstantValue()?.ToString()) + .ToList(); + + return tools; + } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index eb01f6a0..ca24ca97 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -46,6 +46,10 @@ + + + + @@ -146,6 +150,18 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 5142133a..c810790e 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -161,4 +161,9 @@ public partial class ConversationService : IConversationService return converation; } + + public bool IsConversationMode() + { + return !string.IsNullOrWhiteSpace(_conversationId); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs index a4f04149..ca0e24bf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs @@ -10,9 +10,9 @@ public class LoadAttachmentFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a"; private readonly IEnumerable _imageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; private readonly IEnumerable _pdfTypes = new List { "pdf" }; + private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public LoadAttachmentFn( IServiceProvider services, @@ -29,13 +29,13 @@ public class LoadAttachmentFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List(); + var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List(); var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); - var agent = await agentService.LoadAgent(!string.IsNullOrEmpty(message.CurrentAgentId) ? message.CurrentAgentId : AIAssistant); + var agent = await agentService.LoadAgent(TOOL_ASSISTANT); var fileAgent = new Agent { - Id = agent.Id, - Name = agent.Name, + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unkown", Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.", TemplateDict = new Dictionary() }; diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index 558657b8..e114ceb6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -1,63 +1,53 @@ - -using Microsoft.EntityFrameworkCore; - namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase { - private readonly IServiceProvider _services; + private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public override string SelfId => string.Empty; public AttachmentProcessingHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - _services = services; } public override void OnAgentLoaded(Agent agent) { - var fileService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); - var hasConvFiles = fileService.HasConversationUserFiles(conv.ConversationId); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Tools.IsNullOrEmpty() && agent.Tools.Contains(AgentTool.FileAnalyzer); - if (hasConvFiles) + if (isConvMode && isEnabled) { - agent.Instruction += "\r\n\r\nPlease call load_attachment if user wants to describe files, such as images, pdf.\r\n\r\n"; - - if (agent.Functions != null) + var (prompt, loadAttachmentFn) = GetLoadAttachmentFn(); + if (loadAttachmentFn != null) { - var json = JsonSerializer.Serialize(new + if (!string.IsNullOrWhiteSpace(prompt)) { - user_request = new - { - type = "string", - description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." - }, - file_types = new - { - type = "string", - description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'" - } - }); + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } - agent.Functions.Add(new FunctionDef + if (agent.Functions == null) { - Name = "load_attachment", - Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", - Parameters = - { - Properties = JsonSerializer.Deserialize(json), - Required = new List - { - "user_request", - "file_types" - } - } - }); + agent.Functions = new List { loadAttachmentFn }; + } + else + { + agent.Functions.Add(loadAttachmentFn); + } } } base.OnAgentLoaded(agent); } + + private (string, FunctionDef?) GetLoadAttachmentFn() + { + var fnName = "load_attachment"; + var db = _services.GetRequiredService(); + var agent = db.GetAgent(TOOL_ASSISTANT); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fnName}_prompt"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fnName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 4655b1de..3c116c08 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -55,42 +55,6 @@ public class CompletionProvider return completer; } - private static (string, string) GetProviderAndModel(IServiceProvider services, - string? provider = null, - string? model = null, - string? modelId = null, - bool? multiModal = null, - AgentLlmConfig? agentConfig = null) - { - var agentSetting = services.GetRequiredService(); - var state = services.GetRequiredService(); - - if (string.IsNullOrEmpty(provider)) - { - provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider; - provider = state.GetState("provider", provider ?? "azure-openai"); - } - - if (string.IsNullOrEmpty(model)) - { - model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model; - if (state.ContainsState("model")) - { - model = state.GetState("model", model ?? "gpt-35-turbo-4k"); - } - else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) - { - var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; - var llmProviderService = services.GetRequiredService(); - model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name; - } - } - - state.SetState("provider", provider); - state.SetState("model", model); - - return (provider, model); - } public static ITextCompletion GetTextCompletion(IServiceProvider services, string? provider = null, string? model = null, @@ -111,4 +75,66 @@ public class CompletionProvider return completer; } + + public static IImageGeneration GetImageGeneration(IServiceProvider services, + string? provider = null, + string? model = null, + string? modelId = null, + bool imageGenerate = false, + AgentLlmConfig? agentConfig = null) + { + var completions = services.GetServices(); + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, + imageGenerate: imageGenerate, agentConfig: agentConfig); + + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve completion provider by {provider}"); + } + + completer?.SetModelName(model); + + return completer; + } + + private static (string, string) GetProviderAndModel(IServiceProvider services, + string? provider = null, + string? model = null, + string? modelId = null, + bool? multiModal = null, + bool imageGenerate = false, + AgentLlmConfig? agentConfig = null) + { + var agentSetting = services.GetRequiredService(); + var state = services.GetRequiredService(); + + if (string.IsNullOrEmpty(provider)) + { + provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider; + provider = state.GetState("provider", provider ?? "azure-openai"); + } + + if (string.IsNullOrEmpty(model)) + { + model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model; + if (state.ContainsState("model")) + { + model = state.GetState("model", model ?? "dall-e-3"); + } + else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) + { + var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; + var llmProviderService = services.GetRequiredService(); + model = llmProviderService.GetProviderModel(provider, modelIdentity, + multiModal: multiModal, imageGenerate: imageGenerate)?.Name; + } + } + + state.SetState("provider", provider); + state.SetState("model", model); + + return (provider, model); + } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 8320bdb7..b3e86e58 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false) { var models = GetProviderModels(provider) .Where(x => x.Id == id); @@ -54,6 +54,8 @@ public class LlmProviderService : ILlmProviderService models = models.Where(x => x.MultiModal == multiModal); } + models = models.Where(x => x.ImageGeneration == imageGenerate); + var random = new Random(); var index = random.Next(0, models.Count()); var modelSetting = models.ElementAt(index); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index 3488e0e7..89d5081d 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -53,6 +53,9 @@ namespace BotSharp.Core.Repository case AgentField.LlmConfig: UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); break; + case AgentField.Tool: + UpdateAgentTools(agent.Id, agent.Tools); + break; case AgentField.All: UpdateAgentAllFields(agent); break; @@ -145,6 +148,19 @@ namespace BotSharp.Core.Repository File.WriteAllText(agentFile, json); } + private void UpdateAgentTools(string agentId, List tools) + { + if (tools == null) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.Tools = tools; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + private void UpdateAgentRoutingRules(string agentId, List rules) { if (rules == null) return; @@ -271,6 +287,7 @@ namespace BotSharp.Core.Repository agent.Disabled = inputAgent.Disabled; agent.Type = inputAgent.Type; agent.Profiles = inputAgent.Profiles; + agent.Tools = inputAgent.Tools; agent.RoutingRules = inputAgent.RoutingRules; agent.LlmConfig = inputAgent.LlmConfig; agent.UpdatedDateTime = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json new file mode 100644 index 00000000..ef0ce222 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json @@ -0,0 +1,13 @@ +{ + "id": "00000000-0000-0000-0000-000000000000", + "name": "Tool Assistant", + "description": "Tool assistant that can be used to complete many different tasks", + "type": "static", + "createdDateTime": "2023-06-24T10:39:32.2349685Z", + "updatedDateTime": "2023-06-24T14:39:32.2349686Z", + "iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png", + "disabled": false, + "isPublic": false, + "profiles": [ "tool" ], + "routingRules": [] +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json new file mode 100644 index 00000000..75b0b53e --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json @@ -0,0 +1,20 @@ +[ + { + "name": "load_attachment", + "description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." + }, + "file_types": { + "type": "string", + "description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'." + } + }, + "required": [ "user_request", "file_types" ] + } + } +] \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid new file mode 100644 index 00000000..941957e3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid @@ -0,0 +1 @@ +You are a tool agent. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid new file mode 100644 index 00000000..1e1f0ab8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid @@ -0,0 +1 @@ +Please call load_attachment if user wants to describe files, such as images, pdf. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index a7b5652d..5d34ec77 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -140,4 +140,10 @@ public class AgentController : ControllerBase { return await _agentService.DeleteAgent(agentId); } + + [HttpGet("/agent/tools")] + public IEnumerable GetAgentTools() + { + return _agentService.GetAgentTools(); + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 45120a26..700d4015 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -99,8 +99,41 @@ public class InstructModeController : ControllerBase } catch (Exception ex) { - _logger.LogError($"Error in analyzing files. {ex.Message}"); - return $"Error in analyzing files."; + var error = $"Error in analyzing files. {ex.Message}"; + _logger.LogError(error); + return error; + } + } + + [HttpPost("/instruct/image-generation")] + public async Task ImageGeneration([FromBody] IncomingMessageModel input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai", + modelId: input.ModelId ?? "dall-e", imageGenerate: true); + var message = await completion.GetImageGeneration(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, input.Text) + }); + + imageViewModel.RevisedPrompt = message.Content; + imageViewModel.Data = message.Data; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image generation. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; } } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 94de179e..5571d9fa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -24,6 +24,7 @@ public class LlmProviderController : ControllerBase [HttpGet("/llm-provider/{provider}/models")] public IEnumerable GetLlmProviderModels([FromRoute] string provider) { - return _llmProvider.GetProviderModels(provider); + var list = _llmProvider.GetProviderModels(provider); + return list.Where(x => !x.ImageGeneration); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs index b5f8ea82..4078fc03 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs @@ -43,6 +43,7 @@ public class AgentCreationModel /// Combine different Agents together to form a Profile. /// public List Profiles { get; set; } = new List(); + public List Tools { get; set; } = new List(); public List RoutingRules { get; set; } = new List(); public AgentLlmConfig? LlmConfig { get; set; } @@ -57,6 +58,7 @@ public class AgentCreationModel Functions = Functions, Responses = Responses, Samples = Samples, + Tools = Tools, IsPublic = IsPublic, Type = Type, Disabled = Disabled, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs index 0a461ea6..ba331f85 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs @@ -25,6 +25,11 @@ public class AgentUpdateModel /// public List? Samples { get; set; } + /// + /// Tools + /// + public List? Tools { get; set; } + /// /// Functions /// @@ -71,6 +76,7 @@ public class AgentUpdateModel Templates = Templates ?? new List(), Functions = Functions ?? new List(), Responses = Responses ?? new List(), + Tools = Tools ?? new List(), LlmConfig = LlmConfig }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs index 9d87a3e2..5faa1b98 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs @@ -17,6 +17,7 @@ public class AgentViewModel public List Functions { get; set; } public List Responses { get; set; } public List Samples { get; set; } + public List Tools { get; set; } [JsonPropertyName("is_public")] public bool IsPublic { get; set; } @@ -63,6 +64,7 @@ public class AgentViewModel Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, + Tools = agent.Tools, IsPublic= agent.IsPublic, Disabled = agent.Disabled, IconUrl = agent.IconUrl, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs new file mode 100644 index 00000000..1dc3872d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class ImageGenerationViewModel +{ + [JsonPropertyName("revised_prompt")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? RevisedPrompt { get; set; } + + [JsonPropertyName("data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public object? Data { get; set; } + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index b3decd7b..fb04f961 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -29,5 +29,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs new file mode 100644 index 00000000..c0211f11 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs @@ -0,0 +1,104 @@ +using Azure.AI.OpenAI; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class ImageGenerationProvider : IImageGeneration +{ + protected readonly AzureOpenAiSettings _settings; + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + + protected string _model; + + public virtual string Provider => "azure-openai"; + + public ImageGenerationProvider( + AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) + { + _settings = settings; + _services = services; + _logger = logger; + } + + + public async Task GetImageGeneration(Agent agent, List conversations) + { + var contentHooks = _services.GetServices().ToList(); + + // Before + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var options = PrepareOptions(conversations); + var response = await client.GetImageGenerationsAsync(options); + var image = response.Value.Data.First(); + + var content = string.Empty; + if (!string.IsNullOrEmpty(image.RevisedPrompt)) + { + content = image.RevisedPrompt; + } + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + Data = image.Url.AbsoluteUri ?? image.Base64Data + }; + + // After + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = options.Prompt, + Provider = Provider, + Model = _model, + PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(), + CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count() + }); + } + + return responseMessage; + } + + private ImageGenerationOptions PrepareOptions(List conversations) + { + var state = _services.GetRequiredService(); + + var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024"; + var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard"; + + var options = new ImageGenerationOptions + { + DeploymentName = _model, + Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty, + Size = new ImageSize(sizeValue), + Quality = new ImageGenerationQuality(qualityValue) + }; + return options; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs new file mode 100644 index 00000000..5a02b6ae --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs @@ -0,0 +1,16 @@ +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.Logging; +using System; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class OpenAiImageGenerationProvider : ImageGenerationProvider +{ + public override string Provider => "openai"; + + public OpenAiImageGenerationProvider(AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) : base(settings, logger, services) + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs index 77370893..fb65f150 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs @@ -36,7 +36,7 @@ public class WebSocketsMiddleware { var regexes = new List { - new Regex(@"/conversation/[a-z0-9-]+/message/[a-z0-9-]+/[a-z]+/file/[a-z0-9-]+/[a-z0-9-]+", RegexOptions.IgnoreCase), + new Regex(@"/conversation/(.*?)/message/(.*?)/(.*?)/file/(.*?)/(.*?)", RegexOptions.IgnoreCase), new Regex(@"/user/avatar", RegexOptions.IgnoreCase) }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 8e2f4dd4..8e4395a1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -14,6 +14,7 @@ public class AgentDocument : MongoBase public List Functions { get; set; } public List Responses { get; set; } public List Samples { get; set; } + public List Tools { get; set; } public bool IsPublic { get; set; } public bool Disabled { get; set; } public List Profiles { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 167f4b79..2fe77b39 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -55,6 +55,9 @@ public partial class MongoRepository case AgentField.LlmConfig: UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); break; + case AgentField.Tool: + UpdateAgentTools(agent.Id, agent.Tools); + break; case AgentField.All: UpdateAgentAllFields(agent); break; @@ -216,6 +219,18 @@ public partial class MongoRepository _dc.Agents.UpdateOne(filter, update); } + private void UpdateAgentTools(string agentId, List tools) + { + if (tools == null) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Tools, tools) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) { var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config); @@ -242,6 +257,7 @@ public partial class MongoRepository .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) .Set(x => x.Samples, agent.Samples) + .Set(x => x.Tools, agent.Tools) .Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig)) .Set(x => x.IsPublic, agent.IsPublic) .Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -367,6 +383,7 @@ public partial class MongoRepository .Select(r => AgentResponseMongoElement.ToMongoElement(r))? .ToList() ?? new List(), Samples = x.Samples ?? new List(), + Tools = x.Tools ?? new List(), IsPublic = x.IsPublic, Type = x.Type, InheritAgentId = x.InheritAgentId, @@ -456,6 +473,7 @@ public partial class MongoRepository .Select(r => AgentResponseMongoElement.ToDomainElement(r)) .ToList() : new List(), Samples = agentDoc.Samples ?? new List(), + Tools = agentDoc.Tools ?? new List(), IsPublic = agentDoc.IsPublic, Disabled = agentDoc.Disabled, Type = agentDoc.Type, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs index 44320bd9..eb05768d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs @@ -50,6 +50,7 @@ public partial class MongoRepository .Select(r => AgentResponseMongoElement.ToMongoElement(r))? .ToList() ?? new List(), Samples = x.Samples ?? new List(), + Tools = x.Tools ?? new List(), IsPublic = x.IsPublic, Type = x.Type, InheritAgentId = x.InheritAgentId, @@ -74,6 +75,7 @@ public partial class MongoRepository .Set(x => x.Functions, agent.Functions) .Set(x => x.Responses, agent.Responses) .Set(x => x.Samples, agent.Samples) + .Set(x => x.Tools, agent.Tools) .Set(x => x.IsPublic, agent.IsPublic) .Set(x => x.Type, agent.Type) .Set(x => x.InheritAgentId, agent.InheritAgentId) diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 7d529df1..c4556c36 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Loggers; -using BotSharp.Abstraction.Routing; -using Sdcb.SparkDesk.ResponseInternals; namespace BotSharp.Plugin.SparkDesk.Providers;