diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 4d93f307..51f89375 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -24,5 +24,5 @@ public interface IChatCompletion Func onFunctionExecuting); Task GetChatCompletionsStreamingAsync(Agent agent, - List conversations) => Task.FromResult(new RoleDialogModel(AgentRole.Assistant, string.Empty)); + List conversations); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index e8e18eb8..a559877f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -12,6 +12,7 @@ using BotSharp.Core.Templating; using BotSharp.Core.Translation; using BotSharp.Core.Observables.Queues; using Microsoft.Extensions.Configuration; +using BotSharp.Abstraction.Observables.Models; namespace BotSharp.Core.Conversations; @@ -42,7 +43,7 @@ public class ConversationPlugin : IBotSharpPlugin return settingService.Bind("GoogleApi"); }); - services.AddSingleton(); + services.AddSingleton>(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs index 9950a613..affb142d 100644 --- a/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs +++ b/src/Infrastructure/BotSharp.Core/Observables/Queues/MessageHub.cs @@ -1,15 +1,14 @@ -using BotSharp.Abstraction.Observables.Models; using System.Reactive.Subjects; namespace BotSharp.Core.Observables.Queues; -public class MessageHub +public class MessageHub where T : class { - private readonly ILogger _logger; - private readonly ISubject _observable = new Subject(); - public IObservable Events => _observable; + private readonly ILogger> _logger; + private readonly ISubject _observable = new Subject(); + public IObservable Events => _observable; - public MessageHub(ILogger logger) + public MessageHub(ILogger> logger) { _logger = logger; } @@ -18,7 +17,7 @@ public class MessageHub /// Push an item to the observers. /// /// - public void Push(HubObserveData item) + public void Push(T item) { _observable.OnNext(item); } diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 285e8abd..50d67a1b 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -96,8 +96,7 @@ public class ChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, - Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 494326fa..372fb3de 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index ab7135ef..605bceb8 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,6 +1,9 @@ using Azure; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; using OpenAI.Chat; using System.ClientModel; @@ -203,39 +206,133 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var response = chatClient.CompleteChatStreamingAsync(messages, options); + var hub = _services.GetRequiredService>(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; - await foreach (var choice in response) + var contentHooks = _services.GetHooks(agent.Id); + // Before chat completion hook + foreach (var hook in contentHooks) { - if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) - { - var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; - Console.Write(update); + await hook.BeforeGenerating(agent, conversations); + } - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var toolCalls = new List(); + ChatTokenUsage? tokenUsage = null; + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) + { + tokenUsage = choice.Usage; + + if (!choice.ToolCallUpdates.IsNullOrEmpty()) + { + toolCalls.AddRange(choice.ToolCallUpdates); } - if (choice.ContentUpdate.IsNullOrEmpty()) continue; - - _logger.LogInformation(choice.ContentUpdate[0]?.Text); - - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + if (!choice.ContentUpdate.IsNullOrEmpty()) { - RenderedInstruction = string.Join("\r\n", renderedInstructions) + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + textStream.Collect(text); + +#if DEBUG + _logger.LogCritical($"Content update: {text}"); +#endif + + var content = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); + } + + if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) + { + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); + +#if DEBUG + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); +#endif + + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = toolCallId, + FunctionName = functionName, + FunctionArgs = functionArgument + }; + } + else if (choice.FinishReason.HasValue) + { + var allText = textStream.GetText(); + _logger.LogCritical($"Text Content: {allText}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + } + } + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + + var inputTokenDetails = tokenUsage?.InputTokenDetails; + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } - return true; + return responseMessage; } protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index b1fb144f..d3c3f2ac 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Crontab; +using BotSharp.Abstraction.Observables.Models; using BotSharp.Core.Observables.Queues; using BotSharp.Plugin.ChatHub.Hooks; using BotSharp.Plugin.ChatHub.Observers; @@ -35,8 +36,8 @@ public class ChatHubPlugin : IBotSharpPlugin, IBotSharpAppPlugin public void Configure(IApplicationBuilder app) { var services = app.ApplicationServices; - var queue = services.GetRequiredService(); - var logger = services.GetRequiredService>(); + var queue = services.GetRequiredService>(); + var logger = services.GetRequiredService>>(); queue.Events.Subscribe(new ChatHubObserver(logger)); } } diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 2f7e326a..3f9a26ce 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 42ce1ac9..bcbcc2a4 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; +using BotSharp.Plugin.DeepSeek.Providers; using Microsoft.Extensions.Logging; using OpenAI.Chat; -using BotSharp.Abstraction.Files; -using BotSharp.Plugin.DeepSeek.Providers; -using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.DeepSeekAI.Providers.Chat; @@ -170,39 +173,133 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var response = chatClient.CompleteChatStreamingAsync(messages, options); + var hub = _services.GetRequiredService>(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; - await foreach (var choice in response) + var contentHooks = _services.GetHooks(agent.Id); + // Before chat completion hook + foreach (var hook in contentHooks) { - if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls) - { - var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty; - _logger.LogInformation(update); + await hook.BeforeGenerating(agent, conversations); + } - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var toolCalls = new List(); + ChatTokenUsage? tokenUsage = null; + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options)) + { + tokenUsage = choice.Usage; + + if (!choice.ToolCallUpdates.IsNullOrEmpty()) + { + toolCalls.AddRange(choice.ToolCallUpdates); } - if (choice.ContentUpdate.IsNullOrEmpty()) continue; - - _logger.LogInformation(choice.ContentUpdate[0]?.Text); - - await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty) + if (!choice.ContentUpdate.IsNullOrEmpty()) { - RenderedInstruction = string.Join("\r\n", renderedInstructions) + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + textStream.Collect(text); + +#if DEBUG + _logger.LogCritical($"Content update: {text}"); +#endif + + var content = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); + } + + if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) + { + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); + +#if DEBUG + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); +#endif + + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = toolCallId, + FunctionName = functionName, + FunctionArgs = functionArgument + }; + } + else if (choice.FinishReason.HasValue) + { + var allText = textStream.GetText(); + _logger.LogCritical($"Text Content: {allText}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + } + } + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + + var inputTokenDetails = tokenUsage?.InputTokenDetails; + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 }); } - return true; + return responseMessage; } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 1ead28e6..1d67eac8 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -159,40 +159,9 @@ public class GeminiChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { - var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - var chatClient = client.CreateGenerativeModel(_model.ToModelId()); - var (prompt, messages) = PrepareOptions(chatClient,agent, conversations); - - var asyncEnumerable = chatClient.StreamContentAsync(messages); - - await foreach (var response in asyncEnumerable) - { - if (response.GetFunction() != null) - { - var func = response.GetFunction(); - var update = func?.Args?.ToJsonString().ToString() ?? string.Empty; - _logger.LogInformation(update); - - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; - } - - if (response.Text().IsNullOrEmpty()) continue; - - _logger.LogInformation(response.Text()); - - await onMessageReceived(new RoleDialogModel(response.Candidates?.LastOrDefault()?.Content?.Role?.ToString() ?? AgentRole.Assistant.ToString(), response.Text() ?? string.Empty) - { - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - } - - return true; + throw new NotImplementedException(); } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index 72a47adc..e992fd60 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -145,7 +145,7 @@ public class PalmChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 460677ba..5a38b0a9 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -76,9 +76,9 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { - return true; + throw new NotImplementedException(); } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index af7a6237..80807a26 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 45f41e95..321b9aee 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -1,5 +1,12 @@ using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; +using Microsoft.AspNetCore.SignalR; +using static LLama.Common.ChatHistory; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.LLamaSharp.Providers; @@ -159,12 +166,8 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { - string totalResponse = ""; - var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); - content += $"\r\n{AgentRole.Assistant}: "; - var state = _services.GetRequiredService(); var model = state.GetState("model", "llama-2-7b-chat.Q8_0"); @@ -180,13 +183,60 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(agent.Instruction); } + var hub = _services.GetRequiredService>(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + await foreach (var response in executor.InferAsync(agent.Instruction, inferenceParams)) { Console.Write(response); - totalResponse += response; + textStream.Collect(response); + + var content = new RoleDialogModel(AgentRole.Assistant, response) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = content + }); } - return true; + responseMessage = new RoleDialogModel(AgentRole.Assistant, textStream.GetText()) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + return responseMessage; } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs index aa6b1175..9e2aa1d4 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs @@ -65,7 +65,7 @@ namespace BotSharp.Plugin.VertexAI.Providers throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index c1d4bef1..8b3ecea3 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -235,7 +235,7 @@ public class ChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index d95a25ae..e510a9a2 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -169,8 +169,10 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio throw new NotImplementedException(); /// - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) => + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) + { throw new NotImplementedException(); + } private sealed class NopAIFunction(string name, string description, JsonElement schema) : AIFunction { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 16e56b25..314697c8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using Azure; using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.Observables.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.Observables.Queues; using BotSharp.Plugin.OpenAI.Models.Realtime; @@ -190,7 +191,7 @@ public class ChatCompletionProvider : IChatCompletion var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var hub = _services.GetRequiredService(); + var hub = _services.GetRequiredService>(); var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; var contentHooks = _services.GetHooks(agent.Id); @@ -210,7 +211,6 @@ public class ChatCompletionProvider : IChatCompletion MessageId = messageId } }); - using var textStream = new RealtimeTextStream(); var toolCalls = new List(); @@ -273,7 +273,6 @@ public class ChatCompletionProvider : IChatCompletion FunctionName = functionName, FunctionArgs = functionArgument }; - } else if (choice.FinishReason.HasValue) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index 3f742777..57277399 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -94,7 +94,7 @@ namespace BotSharp.Plugin.SemanticKernel throw new NotImplementedException(); } /// - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj index 51f26f87..d49d41df 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/BotSharp.Plugin.SparkDesk.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index ba0aa220..035e03e0 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -1,6 +1,10 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.Observables.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.Observables.Queues; +using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.SparkDesk.Providers; @@ -143,34 +147,77 @@ public class ChatCompletionProvider : IChatCompletion return true; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) { var client = new SparkDeskClient(appId: _settings.AppId, apiKey: _settings.ApiKey, apiSecret: _settings.ApiSecret); var (prompt, messages, funcall) = PrepareOptions(agent, conversations); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + var hub = _services.GetRequiredService>(); + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "BeforeReceiveLlmStreamMessage", + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + using var textStream = new RealtimeTextStream(); await foreach (StreamedChatResponse response in client.ChatAsStreamAsync(modelVersion: _settings.ModelVersion, messages, functions: funcall.Length == 0 ? null : funcall)) { - if (response.FunctionCall !=null) + if (response.FunctionCall != null) { - await onMessageReceived(new RoleDialogModel(AgentRole.Function, response.Text) - { + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = response.FunctionCall.Name, FunctionName = response.FunctionCall.Name, - FunctionArgs = response.FunctionCall.Arguments, - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - continue; + FunctionArgs = response.FunctionCall.Arguments + }; } - - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, response.Text) + else { - CurrentAgentId = agent.Id, - RenderedInstruction = string.Join("\r\n", renderedInstructions) - }); - - } + textStream.Collect(response.Text); + responseMessage = new RoleDialogModel(AgentRole.Assistant, response.Text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; - return true; + hub.Push(new() + { + ServiceProvider = _services, + EventName = "OnReceiveLlmStreamMessage", + Data = responseMessage + }); + } + } + + if (responseMessage.Role == AgentRole.Assistant) + { + responseMessage.Content = textStream.GetText(); + responseMessage.IsStreaming = true; + } + + hub.Push(new() + { + ServiceProvider = _services, + EventName = "AfterReceiveLlmStreamMessage", + Data = responseMessage + }); + + return responseMessage; } public void SetModelName(string model)