diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationCompletionHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationCompletionHookBase.cs index 6d832fe6..d885e7f1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationCompletionHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationCompletionHookBase.cs @@ -46,8 +46,8 @@ public abstract class ConversationCompletionHookBase : IConversationCompletionHo return Task.CompletedTask; } - public virtual Task AfterCompletion(string response) + public virtual Task AfterCompletion(RoleDialogModel message) { - return Task.FromResult(response); + return Task.CompletedTask; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationCompletionHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationCompletionHook.cs index fadc055f..ece8d785 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationCompletionHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationCompletionHook.cs @@ -18,5 +18,5 @@ public interface IConversationCompletionHook IConversationCompletionHook SetChatCompletion(IChatCompletion chatCompletion); Task BeforeCompletion(); - Task AfterCompletion(string response); + Task AfterCompletion(RoleDialogModel message); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 8d87c51e..e3a1684d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -8,8 +8,8 @@ public interface IConversationService Task GetConversation(string id); Task> GetConversations(); Task DeleteConversation(string id); - Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog); - Task SendMessage(string agentId, string conversationId, List wholeDialogs); + Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog, Func onMessageReceived); + Task SendMessage(string agentId, string conversationId, List wholeDialogs, Func onMessageReceived); List GetDialogHistory(string agentId, string conversationId); Task CleanHistory(string agentId); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 04f3e2e4..3f1d6530 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -3,11 +3,16 @@ namespace BotSharp.Abstraction.Conversations.Models; public class RoleDialogModel { /// - /// user, system, assistant + /// user, system, assistant, function /// public string Role { get; set; } public string Content { get; set; } + /// + /// Function name if LLM response function call + /// + public string? Name { get; set; } + public RoleDialogModel(string role, string text) { Role = role; diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 91dea670..8c997ef8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -5,5 +5,5 @@ namespace BotSharp.Abstraction.MLTasks; public interface IChatCompletion { string GetChatCompletions(Agent agent, List conversations); - Task GetChatCompletionsStreamingAsync(Agent agent, List conversations); + Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs index debb5db3..7d8ae3f9 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs @@ -46,11 +46,13 @@ public class ConversationController : ControllerBase, IApiAdapter { var conv = _services.GetRequiredService(); - var result = await conv.SendMessage(agentId, conversationId, new RoleDialogModel("user", input.Text)); + var response = new MessageResponseModel(); - return new MessageResponseModel + await conv.SendMessage(agentId, conversationId, new RoleDialogModel("user", input.Text), async msg => { - Text = result - }; + response.Text += msg.Content; + }); + + return response; } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index d07dc31f..d608524b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -67,20 +67,22 @@ public class ConversationService : IConversationService return record.ToConversation(); } - public async Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog) + public async Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog, Func onMessageReceived) { _storage.Append(agentId, conversationId, lastDalog); var wholeDialogs = GetDialogHistory(agentId, conversationId); - var response = await SendMessage(agentId, conversationId, wholeDialogs); - - _storage.Append(agentId, conversationId, new RoleDialogModel("assistant", response)); + var response = await SendMessage(agentId, conversationId, wholeDialogs, async msg => + { + await onMessageReceived(msg); + _storage.Append(agentId, conversationId, new RoleDialogModel(msg.Role, msg.Content)); + }); return response; } - public async Task SendMessage(string agentId, string conversationId, List wholeDialogs) + public async Task SendMessage(string agentId, string conversationId, List wholeDialogs, Func onMessageReceived) { var agent = await _services.GetRequiredService().GetAgent(agentId); var converation = await GetConversation(conversationId); @@ -110,15 +112,14 @@ public class ConversationService : IConversationService .BeforeCompletion(); }); - var response = await chatCompletion.GetChatCompletionsStreamingAsync(agent, wholeDialogs); - - // After chat completion hook - hooks.ForEach(async hook => + var result = await chatCompletion.GetChatCompletionsStreamingAsync(agent, wholeDialogs, async msg => { - response = await hook.AfterCompletion(response); + // After chat completion hook + hooks.ForEach(async hook => await hook.AfterCompletion(msg)); + await onMessageReceived(msg); }); - return response; + return result; } public IChatCompletion GetChatCompletion() diff --git a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs index 35cdc80f..01636aff 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs @@ -19,7 +19,7 @@ public class ChatCompletionProvider : IChatCompletion throw new NotImplementedException(); } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { string totalResponse = ""; var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Content.Replace("user:", "")}")).Trim(); @@ -36,6 +36,6 @@ public class ChatCompletionProvider : IChatCompletion totalResponse += response; } - return Task.FromResult(totalResponse.Trim()); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 1de10c9a..cf1c6de7 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -86,7 +86,7 @@ public class ChatCompletionProvider : IChatCompletion return functions; } - public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey)); var chatCompletionsOptions = PrepareOptions(agent, conversations); @@ -97,16 +97,23 @@ public class ChatCompletionProvider : IChatCompletion string output = ""; await foreach (var choice in streaming.GetChoicesStreaming()) { + if (choice.FinishReason == CompletionsFinishReason.FunctionCall) + { + } + await foreach (var message in choice.GetMessageStreaming()) { if (message.Content == null) continue; Console.Write(message.Content); output += message.Content; + await onMessageReceived(new RoleDialogModel(message.Role.ToString(), message.Content)); } + + output = ""; } - return output.Trim(); + return true; } private ChatCompletionsOptions PrepareOptions(Agent agent, List conversations) diff --git a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs index 6e7ad937..8b763392 100644 --- a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs +++ b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs @@ -78,22 +78,24 @@ public class ChatbotUiController : ControllerBase, IApiAdapter converation = await conversationService.NewConversation(sess); } - var result = await conversationService.SendMessage(input.AgentId, input.ConversationId, conversations); + var result = await conversationService.SendMessage(input.AgentId, + input.ConversationId, + conversations, + async msg => + await OnChunkReceived(outputStream, msg)); - await OnChunkReceived(outputStream, result); await OnEventCompleted(outputStream); } - private async Task OnChunkReceived(Stream outputStream, string content) + private async Task OnChunkReceived(Stream outputStream, RoleDialogModel message) { - var response = new OpenAiChatOutput { Choices = new List { new OpenAiChoice { - Delta = new RoleDialogModel("assistant", content) + Delta = new RoleDialogModel(message.Role, message.Content) } } }; @@ -105,7 +107,7 @@ public class ChatbotUiController : ControllerBase, IApiAdapter var buffer = Encoding.UTF8.GetBytes($"data:{json}\n"); await outputStream.WriteAsync(buffer, 0, buffer.Length); - await Task.Delay(100); + await Task.Delay(10); buffer = Encoding.UTF8.GetBytes("\n"); await outputStream.WriteAsync(buffer, 0, buffer.Length);