using Azure; using Azure.AI.OpenAI; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Conversations.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Plugin.AzureOpenAI.Settings; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; using System.Threading.Tasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; public class ChatCompletionProvider : IChatCompletion { private readonly AzureOpenAiSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly ITokenStatistics _tokenStatistics; private string _model; public virtual string Provider => "azure-openai"; public ChatCompletionProvider(AzureOpenAiSettings settings, ILogger logger, IServiceProvider services, ITokenStatistics tokenStatistics) { _settings = settings; _logger = logger; _services = services; _tokenStatistics = tokenStatistics; } protected virtual (OpenAIClient, string) GetClient() { if (_model == "gpt-4") { var client = new OpenAIClient(new Uri(_settings.GPT4.Endpoint), new AzureKeyCredential(_settings.GPT4.ApiKey)); return (client, _settings.GPT4.DeploymentModel); } else { var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey)); return (client, _settings.DeploymentModel.ChatCompletionModel); } } public List GetChatSamples(string sampleText) { var samples = new List(); if (string.IsNullOrEmpty(sampleText)) { return samples; } var lines = sampleText.Split('\n'); for (int i = 0; i < lines.Length; i++) { var line = lines[i]; if (string.IsNullOrEmpty(line.Trim())) { continue; } var role = line.Substring(0, line.IndexOf(' ') - 1).Trim(); var content = line.Substring(line.IndexOf(' ') + 1).Trim(); // comments if (role == "##") { continue; } samples.Add(new RoleDialogModel(role, content)); } return samples; } public RoleDialogModel GetChatCompletions(Agent agent, List conversations) { var (client, deploymentModel) = GetClient(); var chatCompletionsOptions = PrepareOptions(agent, conversations); var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions); var choice = response.Value.Choices[0]; var message = choice.Message; _tokenStatistics.AddToken(new TokenStatsModel { Model = _model, PromptCount = response.Value.Usage.PromptTokens, CompletionCount = response.Value.Usage.CompletionTokens, PromptCost = 0.0015f, CompletionCost = 0.002f }); if (choice.FinishReason == CompletionsFinishReason.FunctionCall) { _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name} => {message.FunctionCall.Arguments}"); var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content) { CurrentAgentId = agent.Id, FunctionName = message.FunctionCall.Name, FunctionArgs = message.FunctionCall.Arguments }; // Somethings LLM will generate a function name with agent name. if (!string.IsNullOrEmpty(funcContextIn.FunctionName)) { funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last(); } return funcContextIn; } else { var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) { CurrentAgentId = agent.Id }; return msg; } } public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { var (client, deploymentModel) = GetClient(); var chatCompletionsOptions = PrepareOptions(agent, conversations); var response = await client.GetChatCompletionsAsync(deploymentModel, chatCompletionsOptions); var choice = response.Value.Choices[0]; var message = choice.Message; _tokenStatistics.AddToken(new TokenStatsModel { Model = _model, PromptCount = response.Value.Usage.PromptTokens, CompletionCount = response.Value.Usage.CompletionTokens, PromptCost = 0.0015f, CompletionCost = 0.002f }); if (choice.FinishReason == CompletionsFinishReason.FunctionCall) { _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name} => {message.FunctionCall.Arguments}"); var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content) { CurrentAgentId = agent.Id, FunctionName = message.FunctionCall.Name, FunctionArgs = message.FunctionCall.Arguments }; // Somethings LLM will generate a function name with agent name. if (!string.IsNullOrEmpty(funcContextIn.FunctionName)) { funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last(); } // Execute functions await onFunctionExecuting(funcContextIn); } else { var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) { CurrentAgentId= agent.Id }; // Text response received await onMessageReceived(msg); } return true; } public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey)); var chatCompletionsOptions = PrepareOptions(agent, conversations); var response = await client.GetChatCompletionsStreamingAsync(_settings.DeploymentModel.ChatCompletionModel, chatCompletionsOptions); using StreamingChatCompletions streaming = response.Value; string output = ""; await foreach (var choice in streaming.GetChoicesStreaming()) { if (choice.FinishReason == CompletionsFinishReason.FunctionCall) { var args = ""; await foreach (var message in choice.GetMessageStreaming()) { if (message.FunctionCall == null || message.FunctionCall.Arguments == null) continue; Console.Write(message.FunctionCall.Arguments); args += message.FunctionCall.Arguments; } await onMessageReceived(new RoleDialogModel(ChatRole.Assistant.ToString(), args)); continue; } await foreach (var message in choice.GetMessageStreaming()) { if (message.Content == null) continue; Console.Write(message.Content); output += message.Content; _logger.LogInformation(message.Content); await onMessageReceived(new RoleDialogModel(message.Role.ToString(), message.Content)); } output = ""; } return true; } protected ChatCompletionsOptions PrepareOptions(Agent agent, List conversations) { var chatCompletionsOptions = new ChatCompletionsOptions(); if (!string.IsNullOrEmpty(agent.Instruction)) { chatCompletionsOptions.Messages.Add(new ChatMessage(ChatRole.System, agent.Instruction)); } if (!string.IsNullOrEmpty(agent.Knowledges)) { chatCompletionsOptions.Messages.Add(new ChatMessage(ChatRole.System, agent.Knowledges)); } var samples = GetChatSamples(agent.Samples); foreach (var message in samples) { chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content)); } foreach (var function in agent.Functions) { chatCompletionsOptions.Functions.Add(new FunctionDefinition { Name = function.Name, Description = function.Description, Parameters = BinaryData.FromObjectAsJson(function.Parameters) }); } foreach (var message in conversations) { if (message.Role == ChatRole.Function) { chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content) { Name = message.FunctionName }); } else { chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content)); } } // https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683 var state = _services.GetRequiredService(); var temperature = float.Parse(state.GetState("temperature", "0.5")); var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.5")); chatCompletionsOptions.Temperature = temperature; chatCompletionsOptions.NucleusSamplingFactor = samplingFactor; // chatCompletionsOptions.FrequencyPenalty = 0; // chatCompletionsOptions.PresencePenalty = 0; var convSetting = _services.GetRequiredService(); if (convSetting.ShowVerboseLog) { _logger.LogInformation("VERBOSE COMPLETION MESSAGES"); var verbose = string.Join("\n", chatCompletionsOptions.Messages.Select(x => { return x.Role == ChatRole.Function ? $"{x.Role}: {x.Name} => {x.Content}" : $"{x.Role}: {x.Content}"; })); _logger.LogInformation(verbose); } return chatCompletionsOptions; } public void SetModelName(string model) { _model = model; } }