diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 900bc4a7..0e69d7b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs index d5dabb3c..e2a56817 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs @@ -2,6 +2,9 @@ namespace BotSharp.Abstraction.Functions.Models; public class FunctionDef { + [JsonPropertyName("type")] + public string Type { get; set; } = "function"; + [JsonPropertyName("name")] public string Name { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 1e203333..201f1342 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs new file mode 100644 index 00000000..5eebb5b3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -0,0 +1,12 @@ +using BotSharp.Abstraction.Realtime.Models; + +namespace BotSharp.Abstraction.MLTasks; + +public interface IRealTimeCompletion +{ + string Provider { get; } + + void SetModelName(string model); + + Task CreateSession(Agent agent, List conversations); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 008f528b..a40415c7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -37,6 +37,11 @@ public class LlmModelSetting /// public bool MultiModal { get; set; } + /// + /// If true, allow real-time interaction + /// + public bool RealTime { get; set; } + /// /// If true, allow generating images /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSession.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSession.cs new file mode 100644 index 00000000..59f191c9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSession.cs @@ -0,0 +1,22 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeSession +{ + public string Id { get; set; } = null!; + + public string Object { get; set;} = null!; + public string Model { get; set; } = null!; + public string Voice { get; set; } = null!; + + [JsonPropertyName("client_secret")] + public RealtimeSessionClientSecret Secret { get; set; } = null!; +} + +public class RealtimeSessionClientSecret +{ + [JsonPropertyName("value")] + public string Value { get; set; } = null!; + + [JsonPropertyName("expires_at")] + public long Expires { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs new file mode 100644 index 00000000..0d0afa1a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeSessionUpdate +{ + /// + /// Optional client-generated ID used to identify this event. + /// + public string EventId { get; set; } = null!; + public string Type { get; set; } = "session.update"; + public RealtimeSession Session { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index d6746e09..f399c50e 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -43,11 +43,13 @@ public class CompletionProvider string? model = null, string? modelId = null, bool? multiModal = null, + bool? realTime = null, AgentLlmConfig? agentConfig = null) { var completions = services.GetServices(); (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, - multiModal: multiModal, agentConfig: agentConfig); + multiModal: multiModal, + agentConfig: agentConfig); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) @@ -141,11 +143,36 @@ public class CompletionProvider return completer; } + public static IRealTimeCompletion GetRealTimeCompletion(IServiceProvider services, + string? provider = null, + string? model = null, + string? modelId = null, + bool? multiModal = null, + AgentLlmConfig? agentConfig = null) + { + var completions = services.GetServices(); + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, + multiModal: multiModal, + realTime: true, + agentConfig: agentConfig); + + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve completion provider by {provider}"); + } + + completer?.SetModelName(model); + return completer; + } + private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, string? modelId = null, bool? multiModal = null, + bool? realTime = null, bool imageGenerate = false, AgentLlmConfig? agentConfig = null) { @@ -170,7 +197,9 @@ public class CompletionProvider var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; var llmProviderService = services.GetRequiredService(); model = llmProviderService.GetProviderModel(provider, modelIdentity, - multiModal: multiModal, imageGenerate: imageGenerate)?.Name; + multiModal: multiModal, + realTime: realTime, + imageGenerate: imageGenerate)?.Name; } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index b3e86e58..e1934f60 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false) { var models = GetProviderModels(provider) .Where(x => x.Id == id); @@ -54,6 +54,11 @@ public class LlmProviderService : ILlmProviderService models = models.Where(x => x.MultiModal == multiModal); } + if (realTime.HasValue) + { + models = models.Where(x => x.RealTime == realTime); + } + models = models.Where(x => x.ImageGeneration == imageGenerate); var random = new Random(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs new file mode 100644 index 00000000..7d6fc056 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs @@ -0,0 +1,48 @@ +using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.OpenAPI.Controllers; + +[Authorize] +[ApiController] +public class RealtimeController : ControllerBase +{ + private readonly IServiceProvider _services; + + public RealtimeController(IServiceProvider services) + { + _services = services; + } + + /// + /// Create an ephemeral API token for use in client-side applications with the Realtime API. + /// + /// + [HttpGet("/agent/{agentId}/realtime/session")] + public async Task CreateSession(string agentId) + { + var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4"); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(agentId); + + return await completion.CreateSession(agent, []); + } + + [HttpPost("/agent/{agentId}/function/{functionName}/execute")] + public async Task ExecuteFunction(string agentId, string functionName, [FromBody] JsonDocument args) + { + // var agentService = _services.GetRequiredService(); + // var agent = await agentService.LoadAgent(agentId); + var routing = _services.GetRequiredService(); + // Call functions + var message = new RoleDialogModel(AgentRole.Function, "") + { + FunctionName = functionName, + FunctionArgs = JsonSerializer.Serialize(args) + }; + await routing.InvokeFunction(functionName, message); + return message.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs index 1833a783..7666e525 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs @@ -67,9 +67,9 @@ public class HandleEmailReaderFn : IFunctionCallback var llmProviderService = _services.GetRequiredService(); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4"); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var convService = _services.GetService(); + var convService = _services.GetRequiredService(); var conversationId = convService.ConversationId; var dialogs = convService.GetDialogHistory(fromBreakpoint: false); var response = await completion.GetChatCompletions(agent, dialogs); diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index 68ce651c..d0bea79a 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 0db444ce..32f7220b 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -41,7 +41,6 @@ public class ChatCompletionProvider : IChatCompletion var inferenceParams = new InferenceParams() { - Temperature = 0.1f, AntiPrompts = new List { $"{AgentRole.User}:", "[/INST]" }, MaxTokens = 128 }; @@ -120,7 +119,6 @@ public class ChatCompletionProvider : IChatCompletion var inferenceParams = new InferenceParams() { - Temperature = 0.1f, AntiPrompts = new List { $"{AgentRole.User}:", "[/INST]" }, MaxTokens = 64 }; @@ -170,7 +168,7 @@ public class ChatCompletionProvider : IChatCompletion llama.LoadModel(model); var executor = new StatelessExecutor(llama.Model, llama.Params); - var inferenceParams = new InferenceParams() { Temperature = 1.0f, AntiPrompts = new List { $"{AgentRole.User}:" }, MaxTokens = 64 }; + var inferenceParams = new InferenceParams() { AntiPrompts = new List { $"{AgentRole.User}:" }, MaxTokens = 64 }; var convSetting = _services.GetRequiredService(); if (convSetting.ShowVerboseLog) diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs index 0fe0251d..2a98becc 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs @@ -42,7 +42,7 @@ public class TextCompletionProvider : ITextCompletion llama.LoadModel(_model); var executor = new InstructExecutor(llama.Model.CreateContext(llama.Params)); - var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 }; + var inferenceParams = new InferenceParams() { MaxTokens = 128 }; _tokenStatistics.StartTimer(); string completion = ""; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 4b7792d2..9a9c57fb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -12,7 +12,8 @@ - + + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs new file mode 100644 index 00000000..ce249e70 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs @@ -0,0 +1,49 @@ +using BotSharp.Abstraction.Functions.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.OpenAI.Models; + +public class RealtimeSessionRequest +{ + [JsonPropertyName("model")] + public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17"; + + [JsonPropertyName("temperature")] + public float temperature { get; set; } = 0.8f; + + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = ["audio", "text"]; + + [JsonPropertyName("instructions")] + public string Instructions { get; set; } = "You are a friendly assistant."; + + [JsonPropertyName("max_response_output_tokens")] + public int MaxResponseOutputTokens { get; set; } = 512; + + [JsonPropertyName("tool_choice")] + public string ToolChoice { get; set; } = "auto"; + + [JsonPropertyName("tools")] + public FunctionDef[] Tools { get; set; } = []; + + [JsonPropertyName("turn_detection")] + public RealtimeSessionTurnDetection TurnDetection { get; set; } = new(); +} + +public class RealtimeSessionTurnDetection +{ + /// + /// Milliseconds + /// + [JsonPropertyName("prefix_padding_ms")] + public int PrefixPadding { get; set; } = 300; + + [JsonPropertyName("silence_duration_ms")] + public int SilenceDuration { get; set; } = 500; + + [JsonPropertyName("threshold")] + public float Threshold { get; set; } = 0.5f; + + [JsonPropertyName("type")] + public string Type { get; set; } = "server_vad"; +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index e6d0637c..c1adbbe7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -6,6 +6,8 @@ using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; using BotSharp.Plugin.OpenAI.Providers.Audio; using Microsoft.Extensions.Configuration; +using Refit; +using BotSharp.Plugin.OpenAI.Providers.Realtime; namespace BotSharp.Plugin.OpenAI; @@ -32,5 +34,9 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + + services.AddRefitClient() + .ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com")); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs new file mode 100644 index 00000000..c26ce46d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Realtime.Models; +using Refit; + +namespace BotSharp.Plugin.OpenAI.Providers.Realtime; + +public interface IOpenAiRealtimeApi +{ + [Post("/v1/realtime/sessions")] + Task GetSessionAsync(RealtimeSessionRequest model, [Authorize("Bearer")] string token); +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs new file mode 100644 index 00000000..47389499 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -0,0 +1,244 @@ +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Realtime.Models; +using OpenAI.Chat; +using System.Text.Json; + +namespace BotSharp.Plugin.OpenAI.Providers.Realtime; + +public class RealTimeCompletionProvider : IRealTimeCompletion +{ + public string Provider => "openai"; + + protected readonly OpenAiSettings _settings; + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + + protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17"; + + public RealTimeCompletionProvider( + OpenAiSettings settings, + ILogger logger, + IServiceProvider services) + { + _settings = settings; + _logger = logger; + _services = services; + } + + public async Task CreateSession(Agent agent, List conversations) + { + var contentHooks = _services.GetServices().ToList(); + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var (prompt, messages, options) = PrepareOptions(agent, conversations); + + var args = new RealtimeSessionRequest + { + Instructions = prompt, + ToolChoice = "auto", + Tools = options.Tools.Select(x => + { + var fn = new FunctionDef + { + Name = x.FunctionName, + Description = x.FunctionDescription + }; + fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); + return fn; + }).ToArray(), + }; + + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, args.Model); + + var api = _services.GetRequiredService(); + var session = await api.GetSessionAsync(args, settings.ApiKey); + return session; + } + + protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) + { + var agentService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + var allowMultiModal = settings != null && settings.MultiModal; + + var messages = new List(); + + var temperature = float.Parse(state.GetState("temperature", "0.0")); + var maxTokens = int.Parse(state.GetState("max_tokens", "1024")); + var options = new ChatCompletionOptions() + { + ToolChoice = ChatToolChoice.CreateAutoChoice(), + Temperature = temperature, + MaxOutputTokenCount = maxTokens + }; + + var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []); + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; + + var property = agentService.RenderFunctionProperty(agent, function); + + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } + + if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) + { + var text = agentService.RenderedInstruction(agent); + messages.Add(new SystemChatMessage(text)); + } + + if (!string.IsNullOrEmpty(agent.Knowledges)) + { + messages.Add(new SystemChatMessage(agent.Knowledges)); + } + + var samples = ProviderHelper.GetChatSamples(agent.Samples); + foreach (var sample in samples) + { + messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content)); + } + + var filteredMessages = conversations.Select(x => x).ToList(); + var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User); + if (firstUserMsgIdx > 0) + { + filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); + } + + foreach (var message in filteredMessages) + { + if (message.Role == AgentRole.Function) + { + messages.Add(new AssistantChatMessage(new List + { + ChatToolCall.CreateFunctionToolCall(message.ToolCallId, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + })); + + messages.Add(new ToolChatMessage(message.ToolCallId, message.Content)); + } + else if (message.Role == AgentRole.User) + { + var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + foreach (var file in message.Files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); + contentParts.Add(contentPart); + } + } + } + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); + } + else if (message.Role == AgentRole.Assistant) + { + messages.Add(new AssistantChatMessage(message.Content)); + } + } + + var prompt = GetPrompt(messages, options); + return (prompt, messages, options); + } + + + private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) + { + var prompt = string.Empty; + + if (!messages.IsNullOrEmpty()) + { + // System instruction + var verbose = string.Join("\r\n", messages + .Select(x => x as SystemChatMessage) + .Where(x => x != null) + .Select(x => + { + if (!string.IsNullOrEmpty(x.ParticipantName)) + { + // To display Agent name in log + return $"[{x.ParticipantName}]: {x.Content.FirstOrDefault()?.Text ?? string.Empty}"; + } + return $"{AgentRole.System}: {x.Content.FirstOrDefault()?.Text ?? string.Empty}"; + })); + prompt += $"{verbose}\r\n"; + + prompt += "\r\n[CONVERSATION]"; + verbose = string.Join("\r\n", messages + .Where(x => x as SystemChatMessage == null) + .Select(x => + { + var fnMessage = x as ToolChatMessage; + if (fnMessage != null) + { + return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; + } + + var userMessage = x as UserChatMessage; + if (userMessage != null) + { + var content = x.Content.FirstOrDefault()?.Text ?? string.Empty; + return !string.IsNullOrEmpty(userMessage.ParticipantName) && userMessage.ParticipantName != "route_to_agent" ? + $"{userMessage.ParticipantName}: {content}" : + $"{AgentRole.User}: {content}"; + } + + var assistMessage = x as AssistantChatMessage; + if (assistMessage != null) + { + var toolCall = assistMessage.ToolCalls?.FirstOrDefault(); + return toolCall != null ? + $"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" : + $"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; + } + + return string.Empty; + })); + prompt += $"\r\n{verbose}\r\n"; + } + + if (!options.Tools.IsNullOrEmpty()) + { + var functions = string.Join("\r\n", options.Tools.Select(fn => + { + return $"\r\n{fn.FunctionName}: {fn.FunctionDescription}\r\n{fn.FunctionParameters}"; + })); + prompt += $"\r\n[FUNCTIONS]{functions}\r\n"; + } + + return prompt; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index a85a88a7..655fa954 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -12,6 +12,7 @@ "https://botsharp.scisharpstack.org", "https://chat.scisharpstack.org" ], + "Jwt": { "Issuer": "botsharp", "Audience": "botsharp", @@ -108,6 +109,51 @@ "CompletionCost": 0.002 } ] + }, + { + "Provider": "openai", + "Models": [ + { + "Id": "gpt-4", + "Name": "gpt-4o-mini", + "Version": "2024-07-18", + "ApiKey": "", + "Type": "chat", + "MultiModal": true, + "PromptCost": 0.00015, + "CompletionCost": 0.0006 + }, + { + "Id": "gpt-4", + "Name": "gpt-4o-2024-11-20", + "Version": "2024-11-20", + "ApiKey": "", + "Type": "chat", + "MultiModal": true, + "PromptCost": 0.0025, + "CompletionCost": 0.01 + }, + { + "Id": "gpt-4", + "Name": "gpt-4o-mini-realtime-preview-2024-12-17", + "Version": "2024-12-17", + "ApiKey": "", + "Type": "chat", + "MultiModal": true, + "RealTime": true, + "PromptCost": 0.0025, + "CompletionCost": 0.01 + }, + { + "Id": "text-embedding-3", + "Name": "text-embedding-3-small", + "Version": "3-small", + "ApiKey": "", + "Type": "embedding", + "Dimension": 1536, + "PromptCost": 0.00002 + } + ] } ], @@ -124,8 +170,8 @@ "HostAgentId": "01e2fc5c-2c89-4ec7-8470-7688608b496c", "EnableTranslator": false, "LlmConfig": { - "Provider": "azure-openai", - "Model": "gpt-35-turbo" + "Provider": "openai", + "Model": "gpt-4o-mini" } }, @@ -245,6 +291,7 @@ "Default": "FileRepository", "TablePrefix": "BotSharp", "BotSharpMongoDb": "", + "Redis": "botsharp.redis.cache.windows.net:6380,password=,ssl=True,abortConnect=False", "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] },