From b6c0d47b02c8875bd31d6cbe98e9448eb3b68f0f Mon Sep 17 00:00:00 2001 From: hchen Date: Mon, 7 Aug 2023 12:48:09 -0500 Subject: [PATCH 1/3] Add onFunctionExecuting to SendMessage. --- .../Conversations/IConversationService.cs | 2 +- .../Conversations/ConversationController.cs | 3 ++ .../Services/ConversationService.cs | 6 ++- .../Providers/ChatCompletionProvider.cs | 13 ++++- .../Controllers/WebhookController.cs | 50 +++++++++++++++++-- .../MessagingModels/SenderActionEnum.cs | 13 +++++ .../MessagingModels/SendingMessageRequest.cs | 3 ++ .../WeChatBackgroundService.cs | 3 ++ 8 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 409c5e8e..42de95f7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -8,7 +8,7 @@ public interface IConversationService Task GetConversation(string id); Task> GetConversations(); Task DeleteConversation(string id); - Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog, Func onMessageReceived); + Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog, Func onMessageReceived, Func onFunctionExecuting); Task SendMessage(string agentId, string conversationId, List wholeDialogs, Func onMessageReceived); List GetDialogHistory(string agentId, string conversationId, int lastCount = 20); Task CleanHistory(string agentId); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs index 7d8ae3f9..4c1090f6 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationController.cs @@ -51,6 +51,9 @@ public class ConversationController : ControllerBase, IApiAdapter await conv.SendMessage(agentId, conversationId, new RoleDialogModel("user", input.Text), async msg => { response.Text += msg.Content; + }, async fn => + { + }); return response; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index e09a33f5..7607a634 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -67,7 +67,9 @@ public class ConversationService : IConversationService return record.ToConversation(); } - public async Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog, Func onMessageReceived) + public async Task SendMessage(string agentId, string conversationId, RoleDialogModel lastDalog, + Func onMessageReceived, + Func onFunctionExecuting) { _storage.Append(agentId, conversationId, lastDalog); @@ -81,6 +83,8 @@ public class ConversationService : IConversationService var functions = _services.GetServices().Where(x => x.Name == msg.FunctionName); foreach (var fn in functions) { + await onFunctionExecuting(msg); + msg.ExecutionResult = await fn.Execute(msg.Content); var result = msg.ExecutionResult.Replace("\r", " ").Replace("\n", " "); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 49cae228..3b172314 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -5,6 +5,7 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Text.Json; @@ -15,10 +16,12 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers; public class ChatCompletionProvider : IChatCompletion { private readonly AzureOpenAiSettings _settings; + private readonly ILogger _logger; - public ChatCompletionProvider(AzureOpenAiSettings settings) + public ChatCompletionProvider(AzureOpenAiSettings settings, ILogger logger) { _settings = settings; + _logger = logger; } public string GetChatCompletions(Agent agent, List conversations) @@ -38,6 +41,8 @@ public class ChatCompletionProvider : IChatCompletion output += message.Content; } + _logger.LogInformation(output); + return output.Trim(); } @@ -124,6 +129,9 @@ public class ChatCompletionProvider : IChatCompletion choice = response.Value.Choices[0]; message = choice.Message; + + _logger.LogInformation(message.Content); + await onMessageReceived(new RoleDialogModel(ChatRole.Assistant.ToString(), message.Content)); return true; @@ -161,6 +169,9 @@ public class ChatCompletionProvider : IChatCompletion continue; Console.Write(message.Content); output += message.Content; + + _logger.LogInformation(message.Content); + await onMessageReceived(new RoleDialogModel(message.Role.ToString(), message.Content)); } diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs index 8b07cc42..b666bf07 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs @@ -14,6 +14,7 @@ using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Refit; namespace BotSharp.Plugin.MetaMessenger.Controllers; @@ -58,7 +59,6 @@ public class WebhookController : ControllerBase // TODO validate request // https://developers.facebook.com/docs/messenger-platform/webhooks#verification-requests - try { // received message @@ -69,10 +69,6 @@ public class WebhookController : ControllerBase string content = ""; var sessionId = req.Entry[0].Messaging[0].Sender.Id; var input = req.Entry[0].Messaging[0].Message.Text; - var result = await conv.SendMessage(agentId, sessionId, new RoleDialogModel("user", input), async msg => - { - content = msg.Content; - }); var setting = _services.GetRequiredService(); var messenger = _services.GetRequiredService(); @@ -80,14 +76,58 @@ public class WebhookController : ControllerBase { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }; + + // Marking seen + /*await messenger.SendMessage(setting.ApiVersion, setting.PageId, new SendingMessageRequest + { + AccessToken = setting.PageAccessToken, + Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), + SenderAction = SenderActionEnum.MarkSeen + });*/ + + // Typing on + await messenger.SendMessage(setting.ApiVersion, setting.PageId, new SendingMessageRequest + { + AccessToken = setting.PageAccessToken, + Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), + SenderAction = SenderActionEnum.TypingOn + }); + + // Go to LLM + var result = await conv.SendMessage(agentId, sessionId, new RoleDialogModel("user", input), async msg => + { + content = msg.Content; + }, async fn => + { + await messenger.SendMessage(setting.ApiVersion, setting.PageId, new SendingMessageRequest + { + AccessToken = setting.PageAccessToken, + Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), + Message = JsonSerializer.Serialize(new { Text = "I'm pulling the relevent information, please wait a second ..." }, jsonOpt) + }); + }); + + // Response to user await messenger.SendMessage(setting.ApiVersion, setting.PageId, new SendingMessageRequest { AccessToken = setting.PageAccessToken, Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), Message = JsonSerializer.Serialize(new { Text = content }, jsonOpt) }); + + // Typing off + await messenger.SendMessage(setting.ApiVersion, setting.PageId, new SendingMessageRequest + { + AccessToken = setting.PageAccessToken, + Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), + SenderAction = SenderActionEnum.TypingOff + }); } } + catch (ApiException ex) + { + Console.WriteLine(ex.Content); + } catch (Exception ex) { Console.WriteLine(ex.ToString()); diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs new file mode 100644 index 00000000..ad24e33d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SenderActionEnum.cs @@ -0,0 +1,13 @@ +using System.Runtime.Serialization; + +namespace BotSharp.Plugin.MetaMessenger.MessagingModels; + +public enum SenderActionEnum +{ + [EnumMember(Value = "typing_on")] + TypingOn, + [EnumMember(Value = "typing_off")] + TypingOff, + [EnumMember(Value = "mark_seen")] + MarkSeen +} diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SendingMessageRequest.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SendingMessageRequest.cs index 1eb33e66..1419856f 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SendingMessageRequest.cs +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/MessagingModels/SendingMessageRequest.cs @@ -24,4 +24,7 @@ public class SendingMessageRequest [AliasAs("messaging_type")] public string MessagingType { get; set; } = "RESPONSE"; + + [AliasAs("sender_action")] + public SenderActionEnum? SenderAction { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WeChat/WeChatBackgroundService.cs b/src/Plugins/BotSharp.Plugin.WeChat/WeChatBackgroundService.cs index 1b72e2f9..5823e813 100644 --- a/src/Plugins/BotSharp.Plugin.WeChat/WeChatBackgroundService.cs +++ b/src/Plugins/BotSharp.Plugin.WeChat/WeChatBackgroundService.cs @@ -59,6 +59,9 @@ namespace BotSharp.Plugin.WeChat var result = await conversationService.SendMessage(AgentId, latestConversationId, new RoleDialogModel("user", message), async msg => { await ReplyTextMessageAsync(openid, msg.Content); + }, async fn => + { + }); } From 3f49d11eff1f373b82b1d81199f6ee7770e0194a Mon Sep 17 00:00:00 2001 From: hchen Date: Mon, 7 Aug 2023 15:17:54 -0500 Subject: [PATCH 2/3] Re-format dialog history. --- .../Conversations/Models/RoleDialogModel.cs | 1 + .../FunctionExecutionValidationResult.cs | 1 + .../Services/ConversationStorage.cs | 27 ++++++++++++++----- .../Controllers/WebhookController.cs | 7 +++++ 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 0556b8ad..fa0a1938 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -6,6 +6,7 @@ public class RoleDialogModel /// user, system, assistant, function /// public string Role { get; set; } + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public string Content { get; set; } /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionValidationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionValidationResult.cs index aa1db6f3..423e6563 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionValidationResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionValidationResult.cs @@ -19,5 +19,6 @@ public class FunctionExecutionValidationResult public string ValidationStatus { get; set; } [JsonPropertyName("validation_message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string ValidationMessage { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index fc9564f9..d30fe413 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Conversations.Models; using System.IO; +using Tensorflow; namespace BotSharp.Core.Conversations.Services; @@ -14,20 +15,32 @@ public class ConversationStorage : IConversationStorage public void Append(string agentId, string conversationId, RoleDialogModel dialog) { var conversationFile = GetStorageFile(agentId, conversationId); - File.AppendAllText(conversationFile, $"{dialog.Role}: {dialog.Content}\n"); + var sb = new StringBuilder(); + sb.AppendLine($"{dialog.Role}|{dialog.CreatedAt}"); + sb.AppendLine($" - {dialog.Content}"); + var conversation = sb.ToString(); + File.AppendAllText(conversationFile, conversation); } public List GetDialogs(string agentId, string conversationId) { var conversationFile = GetStorageFile(agentId, conversationId); var dialogs = File.ReadAllLines(conversationFile); - return dialogs.Select(x => + + var results = new List(); + for (int i = 0; i < dialogs.Length; i += 2) { - var pos = x.IndexOf(':'); - var role = x.Substring(0, pos); - var text = x.Substring(pos + 1); - return new RoleDialogModel(role, text); - }).ToList(); + var meta = dialogs[i]; + var dialog = dialogs[i + 1]; + var role = meta.Split('|')[0]; + var createdAt = DateTime.Parse(meta.Split('|')[1]); + var text = dialog.Substring(4); + results.Add(new RoleDialogModel(role, text) + { + CreatedAt = createdAt + }); + } + return results; } public void InitStorage(string agentId, string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs b/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs index b666bf07..6c15c19b 100644 --- a/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs +++ b/src/Plugins/BotSharp.Plugin.MetaMessenger/Controllers/WebhookController.cs @@ -105,6 +105,13 @@ public class WebhookController : ControllerBase Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), Message = JsonSerializer.Serialize(new { Text = "I'm pulling the relevent information, please wait a second ..." }, jsonOpt) }); + + await messenger.SendMessage(setting.ApiVersion, setting.PageId, new SendingMessageRequest + { + AccessToken = setting.PageAccessToken, + Recipient = JsonSerializer.Serialize(new { Id = sessionId }, jsonOpt), + SenderAction = SenderActionEnum.TypingOn + }); }); // Response to user From 19a31510c0673860e6082dd5beef58ccc33fc71d Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Mon, 7 Aug 2023 17:37:32 -0500 Subject: [PATCH 3/3] Adjust conversation history format. --- .../Models/FunctionExecutionResult.cs | 19 ------------------- .../Services/ConversationStorage.cs | 5 +++-- .../Providers/ChatCompletionProvider.cs | 9 ++++----- 3 files changed, 7 insertions(+), 26 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionResult.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionResult.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionResult.cs deleted file mode 100644 index 5cf77f75..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionExecutionResult.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Abstraction.Functions.Models; - -public class FunctionExecutionResult where T : new() -{ - private readonly string _name; - - public FunctionExecutionResult(string name) - { - _name = name; - } - - [JsonPropertyName("function_name")] - public string Name => _name; - - [JsonPropertyName("execution_result")] - public T Result { get; set; } = new T(); -} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index d30fe413..873a5a6a 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Conversations.Models; using System.IO; -using Tensorflow; namespace BotSharp.Core.Conversations.Services; @@ -16,7 +15,7 @@ public class ConversationStorage : IConversationStorage { var conversationFile = GetStorageFile(agentId, conversationId); var sb = new StringBuilder(); - sb.AppendLine($"{dialog.Role}|{dialog.CreatedAt}"); + sb.AppendLine($"{dialog.Role}|{dialog.CreatedAt}|{dialog.FunctionName}"); sb.AppendLine($" - {dialog.Content}"); var conversation = sb.ToString(); File.AppendAllText(conversationFile, conversation); @@ -35,8 +34,10 @@ public class ConversationStorage : IConversationStorage var role = meta.Split('|')[0]; var createdAt = DateTime.Parse(meta.Split('|')[1]); var text = dialog.Substring(4); + var funcName = meta.Split('|')[2]; results.Add(new RoleDialogModel(role, text) { + FunctionName = funcName, CreatedAt = createdAt }); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 3b172314..1738611c 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -8,6 +8,7 @@ using BotSharp.Plugin.AzureOpenAI.Settings; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; @@ -118,9 +119,7 @@ public class ChatCompletionProvider : IChatCompletion await onMessageReceived(funcContextIn); // After function is executed, pass the result to LLM - var fnResult = JsonSerializer.Deserialize>(funcContextIn.ExecutionResult); - var fnJsonResult = JsonSerializer.Serialize(fnResult.Result); - chatCompletionsOptions.Messages.Add(new ChatMessage(ChatRole.Function, fnJsonResult) + chatCompletionsOptions.Messages.Add(new ChatMessage(ChatRole.Function, funcContextIn.ExecutionResult) { Name = funcContextIn.FunctionName }); @@ -216,10 +215,9 @@ public class ChatCompletionProvider : IChatCompletion { if (message.Role == ChatRole.Function) { - var funcContext = JsonSerializer.Deserialize>(message.Content); chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content) { - Name = funcContext.Name + Name = message.FunctionName }); } else @@ -228,6 +226,7 @@ public class ChatCompletionProvider : IChatCompletion } } + _logger.LogInformation(string.Join("\n", chatCompletionsOptions.Messages.Select(x => $"{x.Role}: {x.Content}"))); return chatCompletionsOptions; } }