#pragma warning disable OPENAI001 using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.MessageHub; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; public class ChatCompletionProvider : IChatCompletion { protected readonly OpenAiSettings _settings; protected readonly IServiceProvider _services; protected readonly ILogger _logger; protected string _model; private List renderedInstructions = []; public virtual string Provider => "openai"; public string Model => _model; public ChatCompletionProvider( OpenAiSettings settings, ILogger logger, IServiceProvider services) { _settings = settings; _logger = logger; _services = services; } public async Task GetChatCompletions(Agent agent, List conversations) { var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) { await hook.BeforeGenerating(agent, conversations); } var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); var response = chatClient.CompleteChat(messages, options); var value = response.Value; var reason = value.FinishReason; var content = value.Content; var text = content.FirstOrDefault()?.Text ?? string.Empty; RoleDialogModel responseMessage; if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) { var toolCall = value.ToolCalls.FirstOrDefault(); responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments?.ToString(), RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. if (!string.IsNullOrEmpty(responseMessage.FunctionName)) { responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last(); } } else { responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } var tokenUsage = response.Value?.Usage; var inputTokenDetails = response.Value?.Usage?.InputTokenDetails; // After chat completion hook foreach (var hook in contentHooks) { await hook.AfterGenerated(responseMessage, new TokenStatsModel { Prompt = prompt, Provider = Provider, Model = _model, TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } return responseMessage; } public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) { await hook.BeforeGenerating(agent, conversations); } var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); var response = await chatClient.CompleteChatAsync(messages, options); var value = response.Value; var reason = value.FinishReason; var content = value.Content; var text = content.FirstOrDefault()?.Text ?? string.Empty; var msg = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, RenderedInstruction = string.Join("\r\n", renderedInstructions) }; var tokenUsage = response?.Value?.Usage; var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; // After chat completion hook foreach (var hook in hooks) { await hook.AfterGenerated(msg, new TokenStatsModel { Prompt = prompt, Provider = Provider, Model = _model, TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) { var toolCall = value.ToolCalls?.FirstOrDefault(); _logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})"); var funcContextIn = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolCall?.Id, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments?.ToString(), RenderedInstruction = string.Join("\r\n", renderedInstructions) }; // Somethings LLM will generate a function name with agent name. if (!string.IsNullOrEmpty(funcContextIn.FunctionName)) { funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last(); } // Execute functions await onFunctionExecuting(funcContextIn); } else { // Text response received await onMessageReceived(msg); } return true; } public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); var hub = _services.GetRequiredService>>(); var conv = _services.GetRequiredService(); var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) { await hook.BeforeGenerating(agent, conversations); } hub.Push(new() { EventName = ChatEvent.BeforeReceiveLlmStreamMessage, RefId = conv.ConversationId, Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) { CurrentAgentId = agent.Id, MessageId = messageId } }); using var textStream = new RealtimeTextStream(); var toolCalls = new List(); ChatTokenUsage? tokenUsage = null; var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) { CurrentAgentId = agent.Id, MessageId = messageId }; await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) { tokenUsage = choice.Usage; if (!choice.ToolCallUpdates.IsNullOrEmpty()) { toolCalls.AddRange(choice.ToolCallUpdates); } if (!choice.ContentUpdate.IsNullOrEmpty()) { var text = choice.ContentUpdate[0]?.Text ?? string.Empty; textStream.Collect(text); #if DEBUG _logger.LogCritical($"Stream Content update: {text}"); #endif var content = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, MessageId = messageId }; hub.Push(new() { EventName = ChatEvent.OnReceiveLlmStreamMessage, RefId = conv.ConversationId, Data = content }); } if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) { var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); var functionName = meta?.FunctionName; var toolCallId = meta?.ToolCallId; var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); var functionArguments = string.Join(string.Empty, args); #if DEBUG _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArguments})"); #endif responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) { CurrentAgentId = agent.Id, MessageId = messageId, ToolCallId = toolCallId, FunctionName = functionName, FunctionArgs = functionArguments }; } else if (choice.FinishReason.HasValue) { var allText = textStream.GetText(); _logger.LogInformation($"Stream text Content: {allText}"); responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) { CurrentAgentId = agent.Id, MessageId = messageId, IsStreaming = true }; } } hub.Push(new() { EventName = ChatEvent.AfterReceiveLlmStreamMessage, RefId = conv.ConversationId, Data = responseMessage }); var inputTokenDetails = tokenUsage?.InputTokenDetails; // After chat completion hook foreach (var hook in contentHooks) { await hook.AfterGenerated(responseMessage, new TokenStatsModel { Prompt = prompt, Provider = Provider, Model = _model, TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } return responseMessage; } protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; renderedInstructions = []; var messages = new List(); var options = InitChatCompletionOption(agent); // Render instructions if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var text = agentService.RenderInstruction(agent); renderedInstructions.Add(text); messages.Add(new SystemChatMessage(text)); } // Filter functions var functions = agentService.FilterFunctions(renderedInstructions.FirstOrDefault(), agent); // Render functions foreach (var function in functions) { if (!agentService.RenderFunction(agent, function)) continue; var property = agentService.RenderFunctionProperty(agent, function); options.Tools.Add(ChatTool.CreateFunctionTool( functionName: function.Name, functionDescription: function.Description, functionParameters: BinaryData.FromObjectAsJson(property))); } if (!string.IsNullOrEmpty(agent.Knowledges)) { messages.Add(new SystemChatMessage(agent.Knowledges)); } var samples = ProviderHelper.GetChatSamples(agent.Samples); foreach (var sample in samples) { messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content)); } var filteredMessages = conversations.Select(x => x).ToList(); var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User); if (firstUserMsgIdx > 0) { filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); } foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) { messages.Add(new AssistantChatMessage(new List { ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content)); } else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) { foreach (var file in message.Files) { if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var binary = fileStorage.GetFileBytes(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } } } messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); } else if (message.Role == AgentRole.Assistant) { messages.Add(new AssistantChatMessage(message.Content)); } } var prompt = GetPrompt(messages, options); return (prompt, messages, options); } private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; if (!messages.IsNullOrEmpty()) { // System instruction var verbose = string.Join("\r\n", messages .Select(x => x as SystemChatMessage) .Where(x => x != null) .Select(x => { if (!string.IsNullOrEmpty(x.ParticipantName)) { // To display Agent name in log return $"[{x.ParticipantName}]: {x.Content.FirstOrDefault()?.Text ?? string.Empty}"; } return $"{AgentRole.System}: {x.Content.FirstOrDefault()?.Text ?? string.Empty}"; })); prompt += $"{verbose}\r\n"; prompt += "\r\n[CONVERSATION]"; verbose = string.Join("\r\n", messages .Where(x => x as SystemChatMessage == null) .Select(x => { var fnMessage = x as ToolChatMessage; if (fnMessage != null) { return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; } var userMessage = x as UserChatMessage; if (userMessage != null) { var content = x.Content.FirstOrDefault()?.Text ?? string.Empty; return !string.IsNullOrEmpty(userMessage.ParticipantName) && userMessage.ParticipantName != "route_to_agent" ? $"{userMessage.ParticipantName}: {content}" : $"{AgentRole.User}: {content}"; } var assistMessage = x as AssistantChatMessage; if (assistMessage != null) { var toolCall = assistMessage.ToolCalls?.FirstOrDefault(); return toolCall != null ? $"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" : $"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; } return string.Empty; })); prompt += $"\r\n{verbose}\r\n"; } if (!options.Tools.IsNullOrEmpty()) { var functions = string.Join("\r\n", options.Tools.Select(fn => { return $"\r\n{fn.FunctionName}: {fn.FunctionDescription}\r\n{fn.FunctionParameters}"; })); prompt += $"\r\n[FUNCTIONS]{functions}\r\n"; } return prompt; } private ChatCompletionOptions InitChatCompletionOption(Agent agent) { var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); ChatReasoningEffortLevel? reasoningEffortLevel = null; var temperature = float.Parse(state.GetState("temperature", "0.0")); if (settings?.Reasoning != null) { temperature = settings.Reasoning.Temperature; var level = state.GetState("reasoning_effort_level") .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel ?? string.Empty) .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel ?? string.Empty); reasoningEffortLevel = ParseReasoningEffortLevel(level); } var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) ? tokens : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; return new ChatCompletionOptions() { Temperature = temperature, MaxOutputTokenCount = maxTokens, ReasoningEffortLevel = reasoningEffortLevel }; } private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) { if (string.IsNullOrWhiteSpace(level)) { return null; } var effortLevel = new ChatReasoningEffortLevel("minimal"); switch (level.ToLower()) { case "low": effortLevel = ChatReasoningEffortLevel.Low; break; case "medium": effortLevel = ChatReasoningEffortLevel.Medium; break; case "high": effortLevel = ChatReasoningEffortLevel.High; break; default: break; } return effortLevel; } public void SetModelName(string model) { _model = model; } } class ToolCallData { public ChatFinishReason? Reason { get; set; } public List ToolCalls { get; set; } = []; }