diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 1a3a829b..dc9df32a 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -50,10 +50,10 @@ public class ChatCompletionProvider : IChatCompletion { var toolResult = response.Content.OfType().First(); - responseMessage = new RoleDialogModel(AgentRole.Function, response.FirstMessage?.Text) + responseMessage = new RoleDialogModel(AgentRole.Function, response.FirstMessage?.Text ?? string.Empty) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = toolResult.Id, FunctionName = toolResult.Name, FunctionArgs = JsonSerializer.Serialize(toolResult.Input) @@ -62,10 +62,10 @@ public class ChatCompletionProvider : IChatCompletion else { var message = response.FirstMessage; - responseMessage = new RoleDialogModel(AgentRole.Assistant, message.Text) + responseMessage = new RoleDialogModel(AgentRole.Assistant, message?.Text ?? string.Empty) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } @@ -77,8 +77,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Usage.InputTokens, - CompletionCount = response.Usage.OutputTokens + PromptCount = response.Usage?.InputTokens ?? 0, + CompletionCount = response.Usage?.OutputTokens ?? 0 }); } @@ -120,7 +120,14 @@ public class ChatCompletionProvider : IChatCompletion prompt += "\r\n\r\n" + response_with_function;*/ var messages = new List(); - foreach (var conv in conversations) + var filteredMessages = conversations.Select(x => x).ToList(); + var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User); + if (firstUserMsgIdx > 0) + { + filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); + } + + foreach (var conv in filteredMessages) { if (conv.Role == AgentRole.User) { diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index f14c0d85..06fa6a3c 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index 5a436fef..36f0c3ee 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -24,7 +24,7 @@ public partial class AudioCompletionProvider var options = new AudioTranscriptionOptions { ResponseFormat = format, - Granularities = granularity, + TimestampGranularities = granularity, Temperature = temperature, Prompt = text }; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index 4e5fc3fa..44d49f3c 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -10,7 +10,7 @@ public partial class AudioCompletionProvider .GetAudioClient(_model); var (voice, options) = PrepareGenerationOptions(); - var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + var result = await audioClient.GenerateSpeechAsync(text, voice, options); return result.Value; } @@ -24,7 +24,7 @@ public partial class AudioCompletionProvider var options = new SpeechGenerationOptions { ResponseFormat = format, - Speed = speed + SpeedRatio = speed }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index d4a8a713..afb76a96 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -50,14 +50,15 @@ public class ChatCompletionProvider : IChatCompletion var content = value.Content; var text = content.FirstOrDefault()?.Text ?? string.Empty; - if (reason == ChatFinishReason.FunctionCall) + if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) { + var toolCall = value.ToolCalls.FirstOrDefault(); responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - FunctionName = value.FunctionCall.FunctionName, - FunctionArgs = value.FunctionCall.FunctionArguments + FunctionName = toolCall?.FunctionName, + FunctionArgs = toolCall?.FunctionArguments?.ToString() }; // Somethings LLM will generate a function name with agent name. @@ -66,17 +67,17 @@ public class ChatCompletionProvider : IChatCompletion responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last(); } } - else if (reason == ChatFinishReason.ToolCalls) - { - var toolCall = value.ToolCalls.FirstOrDefault(); - responseMessage = new RoleDialogModel(AgentRole.Function, text) - { - CurrentAgentId = agent.Id, - MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments - }; - } + //else if (reason == ChatFinishReason.ToolCalls) + //{ + // var toolCall = value.ToolCalls.FirstOrDefault(); + // responseMessage = new RoleDialogModel(AgentRole.Function, text) + // { + // CurrentAgentId = agent.Id, + // MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + // FunctionName = toolCall?.FunctionName, + // FunctionArgs = toolCall?.FunctionArguments + // }; + //} else { responseMessage = new RoleDialogModel(AgentRole.Assistant, text) @@ -113,8 +114,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = value?.Usage?.InputTokens ?? 0, - CompletionCount = value?.Usage?.OutputTokens ?? 0 + PromptCount = value?.Usage?.InputTokenCount ?? 0, + CompletionCount = value?.Usage?.OutputTokenCount ?? 0 }); } @@ -157,20 +158,21 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value.Usage.InputTokens, - CompletionCount = response.Value.Usage.OutputTokens + PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, + CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 }); } - if (reason == ChatFinishReason.FunctionCall) + if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) { - _logger.LogInformation($"[{agent.Name}]: {value.FunctionCall.FunctionName}({value.FunctionCall.FunctionArguments})"); + var toolCall = value.ToolCalls?.FirstOrDefault(); + _logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})"); var funcContextIn = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - FunctionName = value.FunctionCall?.FunctionName, - FunctionArgs = value.FunctionCall?.FunctionArguments + FunctionName = toolCall?.FunctionName, + FunctionArgs = toolCall?.FunctionArguments?.ToString() }; // Somethings LLM will generate a function name with agent name. @@ -201,11 +203,12 @@ public class ChatCompletionProvider : IChatCompletion await foreach (var choice in response) { - if (choice.FinishReason == ChatFinishReason.FunctionCall) + if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) { - Console.Write(choice.FunctionCallUpdate?.FunctionArgumentsUpdate); + var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; + Console.Write(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, choice.FunctionCallUpdate?.FunctionArgumentsUpdate)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); continue; } @@ -213,7 +216,7 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); } return true; @@ -235,7 +238,7 @@ public class ChatCompletionProvider : IChatCompletion var options = new ChatCompletionOptions() { Temperature = temperature, - MaxTokens = maxTokens + MaxOutputTokenCount = maxTokens }; foreach (var function in agent.Functions) @@ -267,21 +270,35 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content)); } - foreach (var message in conversations) + var filteredMessages = conversations.Select(x => x).ToList(); + var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User); + if (firstUserMsgIdx > 0) + { + filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); + } + + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) { - messages.Add(new AssistantChatMessage(string.Empty) - { - FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty) - }); + //messages.Add(new AssistantChatMessage(string.Empty) + //{ + // FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty) + //}); - messages.Add(new FunctionChatMessage(message.FunctionName, message.Content)); + //messages.Add(new FunctionChatMessage(message.FunctionName, message.Content)); + + messages.Add(new AssistantChatMessage(new List + { + ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + })); + + messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); } else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text); + var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) @@ -291,20 +308,20 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low); contentParts.Add(contentPart); } } @@ -347,7 +364,7 @@ public class ChatCompletionProvider : IChatCompletion .Where(x => x as SystemChatMessage == null) .Select(x => { - var fnMessage = x as FunctionChatMessage; + var fnMessage = x as ToolChatMessage; if (fnMessage != null) { return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; @@ -365,8 +382,9 @@ public class ChatCompletionProvider : IChatCompletion var assistMessage = x as AssistantChatMessage; if (assistMessage != null) { - return assistMessage.FunctionCall != null ? - $"{AgentRole.Assistant}: Call function {assistMessage.FunctionCall.FunctionName}({assistMessage.FunctionCall.FunctionArguments})" : + var toolCall = assistMessage.ToolCalls?.FirstOrDefault(); + return toolCall != null ? + $"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" : $"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 762e870f..6878ca3c 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -31,7 +31,7 @@ public class TextEmbeddingProvider : ITextEmbedding var options = PrepareOptions(); var response = await embeddingClient.GenerateEmbeddingAsync(text, options); var value = response.Value; - return value.Vector.ToArray(); + return value.ToFloats().ToArray(); } public async Task> GetVectorsAsync(List texts) @@ -41,7 +41,7 @@ public class TextEmbeddingProvider : ITextEmbedding var options = PrepareOptions(); var response = await embeddingClient.GenerateEmbeddingsAsync(texts, options); var value = response.Value; - return value.Select(x => x.Vector.ToArray()).ToList(); + return value.Select(x => x.ToFloats().ToArray()).ToList(); } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs index 7dbd4588..259a8dd2 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs @@ -1,5 +1,5 @@ using Azure.AI.OpenAI; -using Azure; +using System.ClientModel; namespace BotSharp.Plugin.AzureOpenAI.Providers; @@ -9,7 +9,7 @@ public class ProviderHelper { var settingsService = services.GetRequiredService(); var settings = settingsService.GetSetting(provider, model); - var client = new AzureOpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey)); + var client = new AzureOpenAIClient(new Uri(settings.Endpoint), new ApiKeyCredential(settings.ApiKey)); return client; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 70ba3a15..e1138d92 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index 9f248086..f6213eb7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -24,7 +24,7 @@ public partial class AudioCompletionProvider var options = new AudioTranscriptionOptions { ResponseFormat = format, - Granularities = granularity, + TimestampGranularities = granularity, Temperature = temperature, Prompt = text }; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index 09acf389..a0c0225a 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -10,7 +10,7 @@ public partial class AudioCompletionProvider .GetAudioClient(_model); var (voice, options) = PrepareGenerationOptions(); - var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + var result = await audioClient.GenerateSpeechAsync(text, voice, options); return result.Value; } @@ -24,7 +24,7 @@ public partial class AudioCompletionProvider var options = new SpeechGenerationOptions { ResponseFormat = format, - Speed = speed + SpeedRatio = speed }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 57f7ac13..50151513 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -44,14 +44,16 @@ public class ChatCompletionProvider : IChatCompletion var text = content.FirstOrDefault()?.Text ?? string.Empty; RoleDialogModel responseMessage; - if (reason == ChatFinishReason.FunctionCall) + if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) { + var toolCall = value.ToolCalls.FirstOrDefault(); responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - FunctionName = value.FunctionCall.FunctionName, - FunctionArgs = value.FunctionCall.FunctionArguments + ToolCallId = toolCall?.Id, + FunctionName = toolCall?.FunctionName, + FunctionArgs = toolCall?.FunctionArguments?.ToString() }; // Somethings LLM will generate a function name with agent name. @@ -60,17 +62,6 @@ public class ChatCompletionProvider : IChatCompletion responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last(); } } - else if (reason == ChatFinishReason.ToolCalls) - { - var toolCall = value.ToolCalls.FirstOrDefault(); - responseMessage = new RoleDialogModel(AgentRole.Function, text) - { - CurrentAgentId = agent.Id, - MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - FunctionName = toolCall?.FunctionName, - FunctionArgs = toolCall?.FunctionArguments - }; - } else { responseMessage = new RoleDialogModel(AgentRole.Assistant, text) @@ -88,8 +79,8 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value.Usage.InputTokens, - CompletionCount = response.Value.Usage.OutputTokens + PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, + CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 }); } @@ -132,20 +123,22 @@ public class ChatCompletionProvider : IChatCompletion Prompt = prompt, Provider = Provider, Model = _model, - PromptCount = response.Value.Usage.InputTokens, - CompletionCount = response.Value.Usage.OutputTokens + PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, + CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 }); } - if (reason == ChatFinishReason.FunctionCall) + if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) { - _logger.LogInformation($"[{agent.Name}]: {value.FunctionCall.FunctionName}({value.FunctionCall.FunctionArguments})"); + var toolCall = value.ToolCalls?.FirstOrDefault(); + _logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})"); var funcContextIn = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - FunctionName = value.FunctionCall?.FunctionName, - FunctionArgs = value.FunctionCall?.FunctionArguments + ToolCallId = toolCall?.Id, + FunctionName = toolCall?.FunctionName, + FunctionArgs = toolCall?.FunctionArguments?.ToString() }; // Somethings LLM will generate a function name with agent name. @@ -176,11 +169,12 @@ public class ChatCompletionProvider : IChatCompletion await foreach (var choice in response) { - if (choice.FinishReason == ChatFinishReason.FunctionCall) + if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) { - Console.Write(choice.FunctionCallUpdate?.FunctionArgumentsUpdate); + var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; + _logger.LogInformation(update); - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, choice.FunctionCallUpdate?.FunctionArgumentsUpdate)); + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update)); continue; } @@ -188,7 +182,7 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(choice.ContentUpdate[0]?.Text); - await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); + await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty)); } return true; @@ -211,7 +205,7 @@ public class ChatCompletionProvider : IChatCompletion var options = new ChatCompletionOptions() { Temperature = temperature, - MaxTokens = maxTokens + MaxOutputTokenCount = maxTokens }; foreach (var function in agent.Functions) @@ -243,21 +237,28 @@ public class ChatCompletionProvider : IChatCompletion messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content)); } - foreach (var message in conversations) + var filteredMessages = conversations.Select(x => x).ToList(); + var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User); + if (firstUserMsgIdx > 0) + { + filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); + } + + foreach (var message in filteredMessages) { if (message.Role == AgentRole.Function) { - messages.Add(new AssistantChatMessage(string.Empty) + messages.Add(new AssistantChatMessage(new List { - FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty) - }); + ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + })); - messages.Add(new FunctionChatMessage(message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); } else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text); + var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) @@ -267,20 +268,20 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low); contentParts.Add(contentPart); } } @@ -324,7 +325,7 @@ public class ChatCompletionProvider : IChatCompletion .Where(x => x as SystemChatMessage == null) .Select(x => { - var fnMessage = x as FunctionChatMessage; + var fnMessage = x as ToolChatMessage; if (fnMessage != null) { return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; @@ -342,8 +343,9 @@ public class ChatCompletionProvider : IChatCompletion var assistMessage = x as AssistantChatMessage; if (assistMessage != null) { - return assistMessage.FunctionCall != null ? - $"{AgentRole.Assistant}: Call function {assistMessage.FunctionCall.FunctionName}({assistMessage.FunctionCall.FunctionArguments})" : + var toolCall = assistMessage.ToolCalls?.FirstOrDefault(); + return toolCall != null ? + $"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" : $"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index cb9e7d15..5b92d896 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -31,7 +31,7 @@ public class TextEmbeddingProvider : ITextEmbedding var options = PrepareOptions(); var response = await embeddingClient.GenerateEmbeddingAsync(text, options); var value = response.Value; - return value.Vector.ToArray(); + return value.ToFloats().ToArray(); } public async Task> GetVectorsAsync(List texts) @@ -41,7 +41,7 @@ public class TextEmbeddingProvider : ITextEmbedding var options = PrepareOptions(); var response = await embeddingClient.GenerateEmbeddingsAsync(texts, options); var value = response.Value; - return value.Select(x => x.Vector.ToArray()).ToList(); + return value.Select(x => x.ToFloats().ToArray()).ToList(); } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs index 2c985c9e..ed1d9d4b 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs @@ -40,14 +40,18 @@ public class LookupDictionaryFn : IFunctionCallback }; var response = await GetAiResponse(agent); - args = JsonSerializer.Deserialize(response.Content); + args = response.Content.JsonContent(); // check if need to instantely - var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); - var result = connection.Query(args.SqlStatement); + IEnumerable? result = null; + if (!string.IsNullOrWhiteSpace(args.SqlStatement)) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + result = connection.Query(args.SqlStatement); + } - if (result == null) + if (result.IsNullOrEmpty()) { message.Content = "Record not found"; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs index 810aa44f..0f8f615b 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs @@ -5,11 +5,16 @@ namespace BotSharp.Plugin.SqlDriver.Models; public class LookupDictionary { [JsonPropertyName("sql_statement")] - public string? SqlStatement { get; set; } + public string SqlStatement { get; set; } = string.Empty; [JsonPropertyName("reason")] - public string? Reason { get; set; } + public string Reason { get; set; } = string.Empty; [JsonPropertyName("table")] - public string? Table { get; set; } + public string Table { get; set; } = string.Empty; + + public override string ToString() + { + return $"Sql: {SqlStatement}, Table: {Table}, Reason: {Reason}"; + } }