554 lines
22 KiB
C#
554 lines
22 KiB
C#
#pragma warning disable OPENAI001
|
|
using BotSharp.Abstraction.Hooks;
|
|
using BotSharp.Abstraction.MessageHub.Models;
|
|
using BotSharp.Core.Infrastructures.Streams;
|
|
using BotSharp.Core.MessageHub;
|
|
using OpenAI.Chat;
|
|
|
|
namespace BotSharp.Plugin.OpenAI.Providers.Chat;
|
|
|
|
public class ChatCompletionProvider : IChatCompletion
|
|
{
|
|
protected readonly OpenAiSettings _settings;
|
|
protected readonly IServiceProvider _services;
|
|
protected readonly ILogger<ChatCompletionProvider> _logger;
|
|
|
|
protected string _model;
|
|
private List<string> renderedInstructions = [];
|
|
|
|
public virtual string Provider => "openai";
|
|
public string Model => _model;
|
|
|
|
public ChatCompletionProvider(
|
|
OpenAiSettings settings,
|
|
ILogger<ChatCompletionProvider> logger,
|
|
IServiceProvider services)
|
|
{
|
|
_settings = settings;
|
|
_logger = logger;
|
|
_services = services;
|
|
}
|
|
|
|
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
|
|
{
|
|
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
|
|
|
|
// Before chat completion hook
|
|
foreach (var hook in contentHooks)
|
|
{
|
|
await hook.BeforeGenerating(agent, conversations);
|
|
}
|
|
|
|
var client = ProviderHelper.GetClient(Provider, _model, _services);
|
|
var chatClient = client.GetChatClient(_model);
|
|
var (prompt, messages, options) = PrepareOptions(agent, conversations);
|
|
|
|
var response = chatClient.CompleteChat(messages, options);
|
|
var value = response.Value;
|
|
var reason = value.FinishReason;
|
|
var content = value.Content;
|
|
var text = content.FirstOrDefault()?.Text ?? string.Empty;
|
|
|
|
RoleDialogModel responseMessage;
|
|
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
|
|
{
|
|
var toolCall = value.ToolCalls.FirstOrDefault();
|
|
responseMessage = new RoleDialogModel(AgentRole.Function, text)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
|
|
ToolCallId = toolCall?.Id,
|
|
FunctionName = toolCall?.FunctionName,
|
|
FunctionArgs = toolCall?.FunctionArguments?.ToString(),
|
|
RenderedInstruction = string.Join("\r\n", renderedInstructions)
|
|
};
|
|
|
|
// Somethings LLM will generate a function name with agent name.
|
|
if (!string.IsNullOrEmpty(responseMessage.FunctionName))
|
|
{
|
|
responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
|
|
RenderedInstruction = string.Join("\r\n", renderedInstructions)
|
|
};
|
|
}
|
|
|
|
var tokenUsage = response.Value?.Usage;
|
|
var inputTokenDetails = response.Value?.Usage?.InputTokenDetails;
|
|
|
|
// After chat completion hook
|
|
foreach (var hook in contentHooks)
|
|
{
|
|
await hook.AfterGenerated(responseMessage, new TokenStatsModel
|
|
{
|
|
Prompt = prompt,
|
|
Provider = Provider,
|
|
Model = _model,
|
|
TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0),
|
|
CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0,
|
|
TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0
|
|
});
|
|
}
|
|
|
|
return responseMessage;
|
|
}
|
|
|
|
public async Task<bool> GetChatCompletionsAsync(Agent agent,
|
|
List<RoleDialogModel> conversations,
|
|
Func<RoleDialogModel, Task> onMessageReceived,
|
|
Func<RoleDialogModel, Task> onFunctionExecuting)
|
|
{
|
|
var hooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
|
|
|
|
// Before chat completion hook
|
|
foreach (var hook in hooks)
|
|
{
|
|
await hook.BeforeGenerating(agent, conversations);
|
|
}
|
|
|
|
var client = ProviderHelper.GetClient(Provider, _model, _services);
|
|
var chatClient = client.GetChatClient(_model);
|
|
var (prompt, messages, options) = PrepareOptions(agent, conversations);
|
|
|
|
var response = await chatClient.CompleteChatAsync(messages, options);
|
|
var value = response.Value;
|
|
var reason = value.FinishReason;
|
|
var content = value.Content;
|
|
var text = content.FirstOrDefault()?.Text ?? string.Empty;
|
|
|
|
var msg = new RoleDialogModel(AgentRole.Assistant, text)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
RenderedInstruction = string.Join("\r\n", renderedInstructions)
|
|
};
|
|
|
|
var tokenUsage = response?.Value?.Usage;
|
|
var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails;
|
|
|
|
// After chat completion hook
|
|
foreach (var hook in hooks)
|
|
{
|
|
await hook.AfterGenerated(msg, new TokenStatsModel
|
|
{
|
|
Prompt = prompt,
|
|
Provider = Provider,
|
|
Model = _model,
|
|
TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0),
|
|
CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0,
|
|
TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0
|
|
});
|
|
}
|
|
|
|
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
|
|
{
|
|
var toolCall = value.ToolCalls?.FirstOrDefault();
|
|
_logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})");
|
|
|
|
var funcContextIn = new RoleDialogModel(AgentRole.Function, text)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
|
|
ToolCallId = toolCall?.Id,
|
|
FunctionName = toolCall?.FunctionName,
|
|
FunctionArgs = toolCall?.FunctionArguments?.ToString(),
|
|
RenderedInstruction = string.Join("\r\n", renderedInstructions)
|
|
};
|
|
|
|
// Somethings LLM will generate a function name with agent name.
|
|
if (!string.IsNullOrEmpty(funcContextIn.FunctionName))
|
|
{
|
|
funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last();
|
|
}
|
|
|
|
// Execute functions
|
|
await onFunctionExecuting(funcContextIn);
|
|
}
|
|
else
|
|
{
|
|
// Text response received
|
|
await onMessageReceived(msg);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public async Task<RoleDialogModel> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations)
|
|
{
|
|
var client = ProviderHelper.GetClient(Provider, _model, _services);
|
|
var chatClient = client.GetChatClient(_model);
|
|
var (prompt, messages, options) = PrepareOptions(agent, conversations);
|
|
|
|
var hub = _services.GetRequiredService<MessageHub<HubObserveData<RoleDialogModel>>>();
|
|
var conv = _services.GetRequiredService<IConversationService>();
|
|
var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty;
|
|
|
|
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
|
|
// Before chat completion hook
|
|
foreach (var hook in contentHooks)
|
|
{
|
|
await hook.BeforeGenerating(agent, conversations);
|
|
}
|
|
|
|
hub.Push(new()
|
|
{
|
|
EventName = ChatEvent.BeforeReceiveLlmStreamMessage,
|
|
RefId = conv.ConversationId,
|
|
Data = new RoleDialogModel(AgentRole.Assistant, string.Empty)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = messageId
|
|
}
|
|
});
|
|
|
|
using var textStream = new RealtimeTextStream();
|
|
var toolCalls = new List<StreamingChatToolCallUpdate>();
|
|
ChatTokenUsage? tokenUsage = null;
|
|
|
|
var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = messageId
|
|
};
|
|
|
|
await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options))
|
|
{
|
|
tokenUsage = choice.Usage;
|
|
|
|
if (!choice.ToolCallUpdates.IsNullOrEmpty())
|
|
{
|
|
toolCalls.AddRange(choice.ToolCallUpdates);
|
|
}
|
|
|
|
if (!choice.ContentUpdate.IsNullOrEmpty())
|
|
{
|
|
var text = choice.ContentUpdate[0]?.Text ?? string.Empty;
|
|
textStream.Collect(text);
|
|
|
|
#if DEBUG
|
|
_logger.LogCritical($"Stream Content update: {text}");
|
|
#endif
|
|
|
|
var content = new RoleDialogModel(AgentRole.Assistant, text)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = messageId
|
|
};
|
|
hub.Push(new()
|
|
{
|
|
EventName = ChatEvent.OnReceiveLlmStreamMessage,
|
|
RefId = conv.ConversationId,
|
|
Data = content
|
|
});
|
|
}
|
|
|
|
if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall)
|
|
{
|
|
var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName));
|
|
var functionName = meta?.FunctionName;
|
|
var toolCallId = meta?.ToolCallId;
|
|
var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList();
|
|
var functionArguments = string.Join(string.Empty, args);
|
|
|
|
#if DEBUG
|
|
_logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArguments})");
|
|
#endif
|
|
|
|
responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = messageId,
|
|
ToolCallId = toolCallId,
|
|
FunctionName = functionName,
|
|
FunctionArgs = functionArguments
|
|
};
|
|
}
|
|
else if (choice.FinishReason.HasValue)
|
|
{
|
|
var allText = textStream.GetText();
|
|
_logger.LogInformation($"Stream text Content: {allText}");
|
|
|
|
responseMessage = new RoleDialogModel(AgentRole.Assistant, allText)
|
|
{
|
|
CurrentAgentId = agent.Id,
|
|
MessageId = messageId,
|
|
IsStreaming = true
|
|
};
|
|
}
|
|
}
|
|
|
|
hub.Push(new()
|
|
{
|
|
EventName = ChatEvent.AfterReceiveLlmStreamMessage,
|
|
RefId = conv.ConversationId,
|
|
Data = responseMessage
|
|
});
|
|
|
|
|
|
var inputTokenDetails = tokenUsage?.InputTokenDetails;
|
|
// After chat completion hook
|
|
foreach (var hook in contentHooks)
|
|
{
|
|
await hook.AfterGenerated(responseMessage, new TokenStatsModel
|
|
{
|
|
Prompt = prompt,
|
|
Provider = Provider,
|
|
Model = _model,
|
|
TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0),
|
|
CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0,
|
|
TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0
|
|
});
|
|
}
|
|
|
|
return responseMessage;
|
|
}
|
|
|
|
|
|
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
|
|
{
|
|
var agentService = _services.GetRequiredService<IAgentService>();
|
|
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
|
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
|
var settings = settingsService.GetSetting(Provider, _model);
|
|
var allowMultiModal = settings != null && settings.MultiModal;
|
|
renderedInstructions = [];
|
|
|
|
var messages = new List<ChatMessage>();
|
|
var options = InitChatCompletionOption(agent);
|
|
|
|
// Render instructions
|
|
if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
|
|
{
|
|
var text = agentService.RenderInstruction(agent);
|
|
renderedInstructions.Add(text);
|
|
messages.Add(new SystemChatMessage(text));
|
|
}
|
|
|
|
// Filter functions
|
|
var functions = agentService.FilterFunctions(renderedInstructions.FirstOrDefault(), agent);
|
|
|
|
// Render functions
|
|
foreach (var function in functions)
|
|
{
|
|
if (!agentService.RenderFunction(agent, function)) continue;
|
|
|
|
var property = agentService.RenderFunctionProperty(agent, function);
|
|
|
|
options.Tools.Add(ChatTool.CreateFunctionTool(
|
|
functionName: function.Name,
|
|
functionDescription: function.Description,
|
|
functionParameters: BinaryData.FromObjectAsJson(property)));
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(agent.Knowledges))
|
|
{
|
|
messages.Add(new SystemChatMessage(agent.Knowledges));
|
|
}
|
|
|
|
var samples = ProviderHelper.GetChatSamples(agent.Samples);
|
|
foreach (var sample in samples)
|
|
{
|
|
messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content));
|
|
}
|
|
|
|
var filteredMessages = conversations.Select(x => x).ToList();
|
|
var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
|
|
if (firstUserMsgIdx > 0)
|
|
{
|
|
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
|
|
}
|
|
|
|
foreach (var message in filteredMessages)
|
|
{
|
|
if (message.Role == AgentRole.Function)
|
|
{
|
|
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
|
|
{
|
|
ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
|
|
}));
|
|
|
|
messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
|
|
}
|
|
else if (message.Role == AgentRole.User)
|
|
{
|
|
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
|
|
var textPart = ChatMessageContentPart.CreateTextPart(text);
|
|
var contentParts = new List<ChatMessageContentPart> { textPart };
|
|
|
|
if (allowMultiModal && !message.Files.IsNullOrEmpty())
|
|
{
|
|
foreach (var file in message.Files)
|
|
{
|
|
if (!string.IsNullOrEmpty(file.FileData))
|
|
{
|
|
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
|
|
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
|
|
contentParts.Add(contentPart);
|
|
}
|
|
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
|
|
{
|
|
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
|
|
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
|
|
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
|
|
contentParts.Add(contentPart);
|
|
}
|
|
else if (!string.IsNullOrEmpty(file.FileUrl))
|
|
{
|
|
var uri = new Uri(file.FileUrl);
|
|
var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto);
|
|
contentParts.Add(contentPart);
|
|
}
|
|
}
|
|
}
|
|
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
|
|
}
|
|
else if (message.Role == AgentRole.Assistant)
|
|
{
|
|
messages.Add(new AssistantChatMessage(message.Content));
|
|
}
|
|
}
|
|
|
|
var prompt = GetPrompt(messages, options);
|
|
return (prompt, messages, options);
|
|
}
|
|
|
|
|
|
private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
|
|
{
|
|
var prompt = string.Empty;
|
|
|
|
if (!messages.IsNullOrEmpty())
|
|
{
|
|
// System instruction
|
|
var verbose = string.Join("\r\n", messages
|
|
.Select(x => x as SystemChatMessage)
|
|
.Where(x => x != null)
|
|
.Select(x =>
|
|
{
|
|
if (!string.IsNullOrEmpty(x.ParticipantName))
|
|
{
|
|
// To display Agent name in log
|
|
return $"[{x.ParticipantName}]: {x.Content.FirstOrDefault()?.Text ?? string.Empty}";
|
|
}
|
|
return $"{AgentRole.System}: {x.Content.FirstOrDefault()?.Text ?? string.Empty}";
|
|
}));
|
|
prompt += $"{verbose}\r\n";
|
|
|
|
prompt += "\r\n[CONVERSATION]";
|
|
verbose = string.Join("\r\n", messages
|
|
.Where(x => x as SystemChatMessage == null)
|
|
.Select(x =>
|
|
{
|
|
var fnMessage = x as ToolChatMessage;
|
|
if (fnMessage != null)
|
|
{
|
|
return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
|
|
}
|
|
|
|
var userMessage = x as UserChatMessage;
|
|
if (userMessage != null)
|
|
{
|
|
var content = x.Content.FirstOrDefault()?.Text ?? string.Empty;
|
|
return !string.IsNullOrEmpty(userMessage.ParticipantName) && userMessage.ParticipantName != "route_to_agent" ?
|
|
$"{userMessage.ParticipantName}: {content}" :
|
|
$"{AgentRole.User}: {content}";
|
|
}
|
|
|
|
var assistMessage = x as AssistantChatMessage;
|
|
if (assistMessage != null)
|
|
{
|
|
var toolCall = assistMessage.ToolCalls?.FirstOrDefault();
|
|
return toolCall != null ?
|
|
$"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" :
|
|
$"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
|
|
}
|
|
|
|
return string.Empty;
|
|
}));
|
|
prompt += $"\r\n{verbose}\r\n";
|
|
}
|
|
|
|
if (!options.Tools.IsNullOrEmpty())
|
|
{
|
|
var functions = string.Join("\r\n", options.Tools.Select(fn =>
|
|
{
|
|
return $"\r\n{fn.FunctionName}: {fn.FunctionDescription}\r\n{fn.FunctionParameters}";
|
|
}));
|
|
prompt += $"\r\n[FUNCTIONS]{functions}\r\n";
|
|
}
|
|
|
|
return prompt;
|
|
}
|
|
|
|
private ChatCompletionOptions InitChatCompletionOption(Agent agent)
|
|
{
|
|
var state = _services.GetRequiredService<IConversationStateService>();
|
|
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
|
var settings = settingsService.GetSetting(Provider, _model);
|
|
|
|
ChatReasoningEffortLevel? reasoningEffortLevel = null;
|
|
var temperature = float.Parse(state.GetState("temperature", "0.0"));
|
|
if (settings?.Reasoning != null)
|
|
{
|
|
temperature = settings.Reasoning.Temperature;
|
|
var level = state.GetState("reasoning_effort_level")
|
|
.IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel ?? string.Empty)
|
|
.IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel ?? string.Empty);
|
|
reasoningEffortLevel = ParseReasoningEffortLevel(level);
|
|
}
|
|
|
|
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
|
|
? tokens
|
|
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
|
|
|
|
return new ChatCompletionOptions()
|
|
{
|
|
Temperature = temperature,
|
|
MaxOutputTokenCount = maxTokens,
|
|
ReasoningEffortLevel = reasoningEffortLevel
|
|
};
|
|
}
|
|
|
|
private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(level))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var effortLevel = new ChatReasoningEffortLevel("minimal");
|
|
switch (level.ToLower())
|
|
{
|
|
case "low":
|
|
effortLevel = ChatReasoningEffortLevel.Low;
|
|
break;
|
|
case "medium":
|
|
effortLevel = ChatReasoningEffortLevel.Medium;
|
|
break;
|
|
case "high":
|
|
effortLevel = ChatReasoningEffortLevel.High;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return effortLevel;
|
|
}
|
|
|
|
public void SetModelName(string model)
|
|
{
|
|
_model = model;
|
|
}
|
|
}
|
|
|
|
|
|
class ToolCallData
|
|
{
|
|
public ChatFinishReason? Reason { get; set; }
|
|
public List<StreamingChatToolCallUpdate> ToolCalls { get; set; } = [];
|
|
} |