From fec696493bfab8f6e29992bcbe49a2cce2971f8a Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 13 Feb 2025 13:59:57 -0600 Subject: [PATCH 01/23] enable all ValidateRequest --- .../Controllers/TwilioVoiceController.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 19144673..7b041ca6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -108,7 +108,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { @@ -202,7 +202,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { @@ -367,7 +367,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpPost("twilio/voice/init-call")] public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) { @@ -388,7 +388,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - // [ValidateRequest] + [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { From fab5a17b75f3af7687fa144640b678a3be4d79b4 Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 22 Feb 2025 08:55:47 -0800 Subject: [PATCH 02/23] more projects on ManagePackageVersionsCentrally --- Directory.Packages.props | 14 ++++++++++++++ .../BotSharp.Core.Crontab.csproj | 3 ++- .../BotSharp.Plugin.AzureOpenAI.csproj | 5 +++-- .../BotSharp.Plugin.DeepSeekAI.csproj | 3 ++- .../BotSharp.Plugin.EmailHandler.csproj | 3 ++- .../BotSharp.Plugin.ExcelHandler.csproj | 7 ++++--- .../BotSharp.Plugin.GoogleAI.csproj | 5 +++-- .../BotSharp.Plugin.HuggingFace.csproj | 5 +++-- .../BotSharp.Plugin.JavaScriptInterpreter.csproj | 3 ++- .../BotSharp.Plugin.KnowledgeBase.csproj | 5 +++-- 10 files changed, 38 insertions(+), 15 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a031cec3..55683808 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -33,6 +33,20 @@ + + + + + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj index ae4c9fef..b4bcaf84 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj +++ b/src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj @@ -5,6 +5,7 @@ $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) + true $(SolutionDir)packages enable enable @@ -29,7 +30,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 284e9b3c..61758e2e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj index 56eb14ef..598e1295 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/BotSharp.Plugin.DeepSeekAI.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f1a6c28e..a7f4edcf 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -33,7 +34,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj index 08cfd79a..3b6db5d3 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj @@ -4,6 +4,7 @@ net8.0 enable enable + true @@ -27,9 +28,9 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index b6e0b24e..9e153d76 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj index ae6a05be..2b8854af 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj @@ -8,11 +8,12 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - - + + diff --git a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj index 4596ce1b..6abacd91 100644 --- a/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj +++ b/src/Plugins/BotSharp.Plugin.JavaScriptInterpreter/BotSharp.Plugin.JavaScriptInterpreter.csproj @@ -4,10 +4,11 @@ net8.0 enable enable + true - + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 77a5d076..b7cf3059 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -55,8 +56,8 @@ - - + + From 15c5fc3a5eb5a63ae4acbe31c7e1af794fd972fa Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 22 Feb 2025 09:08:59 -0800 Subject: [PATCH 03/23] more projects on ManagePackageVersionsCentrally --- Directory.Packages.props | 4 ++++ .../BotSharp.Plugin.LLamaSharp.csproj | 3 ++- .../BotSharp.Plugin.VertexAI.csproj | 3 ++- .../BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj | 7 ++++--- .../BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 55683808..1cee14e9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -47,6 +47,10 @@ + + + + diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj index d0bea79a..d5581998 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj index 004d7233..92509518 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj +++ b/src/Plugins/BotSharp.Plugin.LangChain/BotSharp.Plugin.VertexAI.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj index 3d390f56..2c247292 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaAI/BotSharp.Plugin.MetaAI.csproj @@ -8,6 +8,7 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true @@ -15,8 +16,8 @@ - - - + + + diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj index 2e43183c..d1bcb1b4 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/BotSharp.Plugin.MetaGLM.csproj @@ -8,10 +8,11 @@ $(GeneratePackageOnBuild) $(GenerateDocumentationFile) $(SolutionDir)packages + true - + From 3006b468dbb9ef11f4bd95d36c24c69492bfa67c Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Sat, 22 Feb 2025 09:20:51 -0800 Subject: [PATCH 04/23] bump up version of System.IdentityModel.Tokens.Jwt --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 1cee14e9..f6b1545e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,7 +13,7 @@ - + From 8e668b86279f54aef810d3cd56f7affb47e9ef3e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 24 Feb 2025 12:14:25 -0600 Subject: [PATCH 05/23] Add document type of knowledge retrieval --- .../BotSharp.Plugin.KnowledgeBase.csproj | 6 ++---- .../Enum/UtilityName.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Hooks/KnowledgeBaseUtilityHook.cs | 2 +- .../Hooks/KnowledgeHook.cs | 14 ++++++++++++++ ...ieval.json => util-kg-knowledge_retrieval.json} | 2 +- ...iquid => util-kg-knowledge_retrieval.fn.liquid} | 2 +- 7 files changed, 21 insertions(+), 9 deletions(-) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-knowledge-knowledge_retrieval.json => util-kg-knowledge_retrieval.json} (86%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/{util-knowledge-knowledge_retrieval.fn.liquid => util-kg-knowledge_retrieval.fn.liquid} (63%) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 77a5d076..69537fba 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -23,8 +23,6 @@ - - @@ -46,10 +44,10 @@ PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs index de90430d..e4518344 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Enum/UtilityName.cs @@ -2,5 +2,5 @@ namespace BotSharp.Plugin.KnowledgeBase.Enum; public class UtilityName { - public const string KnowledgeRetrieval = "knowledge.knowledge-retrieval"; + public const string KnowledgeRetrieval = "kg.knowledge-base"; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f3f8cbf1..940c3834 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { - public string Name => "util-knowledge-knowledge_retrieval"; + public string Name => "util-kg-knowledge_retrieval"; public string Indication => "searching my brain"; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index 700b8457..84254841 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook { - private static string PREFIX = "util-knowledge-"; + private static string PREFIX = "util-kg-"; private static string KNOWLEDGE_RETRIEVAL_FN = $"{PREFIX}knowledge_retrieval"; public void AddUtilities(List utilities) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs index da4add53..90aa0114 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeHook.cs @@ -40,6 +40,20 @@ public class KnowledgeHook : IKnowledgeHook var result = await _knowledgeService.SearchGraphKnowledge(text, options); results.Add(result.Result); } + else if (knowledgeBase.Type == "document") + { + var options = new VectorSearchOptions + { + Fields = null, + Limit = 5, + Confidence = 0.25f, + WithVector = true + }; + var result = await _knowledgeService.SearchVectorKnowledge(text, knowledgeBase.Name, options); + results.AddRange(result.Where(x => x.Data != null && x.Data.ContainsKey("text")) + .Select(x => x.Data["text"].ToString()) + .Where(x => x != null)!); + } else { var options = new VectorSearchOptions diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json similarity index 86% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json index 088f4ea6..cb79298a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-knowledge-knowledge_retrieval.json +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-kg-knowledge_retrieval.json @@ -1,5 +1,5 @@ { - "name": "util-knowledge-knowledge_retrieval", + "name": "util-kg-knowledge_retrieval", "description": "Retrieve related domain knowledge to handle user request", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid similarity index 63% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid index f0bc60c8..e7d61b7f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-knowledge-knowledge_retrieval.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-kg-knowledge_retrieval.fn.liquid @@ -1,3 +1,3 @@ -Call function util-knowledge-knowledge_retrieval to retrieve related domain knowledge to handle user request. +Call function util-kg-knowledge_retrieval to retrieve related domain knowledge to handle user request. You must retrieve existing KnowledgeBase to get prerequisite informations before you writing SQL query; You must retrieve existing API specification from KnowledgeBase before calling a Web API; From 51d933e3db1616f50b7266f33f5aa4a62ead54b1 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 24 Feb 2025 22:35:22 -0600 Subject: [PATCH 06/23] lazy routing (WIP) --- .../Infrastructures/Enums/StateConst.cs | 3 ++ .../Routing/IRoutingContext.cs | 8 ++-- .../Routing/Models/FallbackArgs.cs | 10 +++++ .../BotSharp.Core/BotSharp.Core.csproj | 14 ++++++- .../ConversationService.SendMessage.cs | 15 +++++++- .../Routing/Functions/FallbackToRouterFn.cs | 29 +++------------ .../Routing/Hooks/RoutingUtilityHook.cs | 20 ++++++++++ .../Routing/Reasoning/NaiveReasoner.cs | 2 +- .../BotSharp.Core/Routing/RoutingContext.cs | 37 ++++++++++++++++--- .../BotSharp.Core/Routing/RoutingPlugin.cs | 2 + .../Routing/RoutingService.InvokeAgent.cs | 3 +- .../functions/route_to_agent.json | 31 ++++++++++++++++ .../instructions/instruction.liquid | 2 + .../util-routing-fallback_to_router.json | 18 +++++++++ .../util-routing-fallback_to_router.fn.liquid | 1 + 15 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..f2e0450a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -8,6 +8,9 @@ public class StateConst public const string NEXT_ACTION_REASON = "next_action_reason"; public const string USER_GOAL_AGENT = "user_goal_agent"; public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; + // lazy or eager + public const string ROUTING_MODE = "routing_mode"; + public const string LAZY_ROUTING_AGENT_ID = "lazy_routing_agent_id"; public const string LANGUAGE = "language"; diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs index da7d5c01..337d63c6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingContext.cs @@ -13,10 +13,10 @@ public interface IRoutingContext bool IsEmpty { get; } string IntentName { get; set; } int AgentCount { get; } - void Push(string agentId, string? reason = null); - void Pop(string? reason = null); - void PopTo(string agentId, string reason); - void Replace(string agentId, string? reason = null); + void Push(string agentId, string? reason = null, bool updateLazyRouting = true); + void Pop(string? reason = null, bool updateLazyRouting = true); + void PopTo(string agentId, string reason, bool updateLazyRouting = true); + void Replace(string agentId, string? reason = null, bool updateLazyRouting = true); void Empty(string? reason = null); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs new file mode 100644 index 00000000..6fc2ce03 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/FallbackArgs.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Routing.Models; + +public class FallbackArgs +{ + [JsonPropertyName("fallback_reason")] + public string Reason { get; set; } = null!; + + [JsonPropertyName("user_question")] + public string Question { get; set; } = null; +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6bc7596c..c8ded3cd 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -66,6 +66,8 @@ + + @@ -82,6 +84,7 @@ + @@ -146,6 +149,15 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index f60d32cf..3967aaaa 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Routing.Settings; @@ -36,7 +37,17 @@ public partial class ConversationService // Enqueue receiving agent first in case it stop completion by OnMessageReceived var routing = _services.GetRequiredService(); routing.Context.SetMessageId(_conversationId, message.MessageId); - routing.Context.Push(agent.Id, reason: "request started"); + + // Check the routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false); + + if (routingMode == "lazy") + { + message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId); + routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false); + } // Save payload in order to assign the payload before hook is invoked if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) @@ -77,7 +88,7 @@ public partial class ConversationService { agent = await agentService.LoadAgent(message.CurrentAgentId); } - + if (agent.Type == AgentType.Routing) { response = await routing.InstructLoop(message, dialogs); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs index 592a2c29..58e43819 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs @@ -5,8 +5,9 @@ namespace BotSharp.Core.Routing.Functions; public class FallbackToRouterFn : IFunctionCallback { - public string Name => "fallback_to_router"; + public string Name => "util-routing-fallback_to_router"; private readonly IServiceProvider _services; + public FallbackToRouterFn(IServiceProvider services) { _services = services; @@ -14,30 +15,10 @@ public class FallbackToRouterFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - var agentService = _services.GetRequiredService(); - var agents = await agentService.GetAgents(new AgentFilter - { - AgentNames = [args.AgentName] - }); - var targetAgent = agents.Items.FirstOrDefault(); - if (targetAgent == null) - { - message.Content = $"Can't find routing agent {args.AgentName}"; - return false; - } - - var conv = _services.GetRequiredService(); - var dialogs = conv.GetDialogHistory(); - + var args = JsonSerializer.Deserialize(message.FunctionArgs); var routing = _services.GetRequiredService(); - routing.Context.Replace(targetAgent.Id); - message.CurrentAgentId = targetAgent.Id; - - var response = await routing.InstructLoop(message, dialogs); - - message.Content = response.Content; - message.StopCompletion = true; + routing.Context.PopTo(routing.Context.EntryAgentId, "pop to entry agent"); + message.Content = args.Question; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs new file mode 100644 index 00000000..e6fedc05 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Core.Routing.Hooks; + +public class RoutingUtilityHook : IAgentUtilityHook +{ + private static string PREFIX = "util-routing-"; + private static string REDIRECT_TO_AGENT = $"{PREFIX}redirect_to_agent"; + private static string FALLBACK_TO_ROUTER = $"{PREFIX}fallback_to_router"; + + public void AddUtilities(List utilities) + { + var utility = new AgentUtility + { + Name = "routing.tools", + Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")], + Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")] + }; + + utilities.Add(utility); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs index 931b53e8..8980246e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/NaiveReasoner.cs @@ -73,7 +73,7 @@ public class NaiveReasoner : IRoutingReasoner }; var response = await completion.GetChatCompletions(router, dialogs); - inst = response.Content.JsonContent(); + inst = (response.FunctionArgs ?? response.Content).JsonContent(); break; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index cf604c55..47e47631 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Routing.Settings; namespace BotSharp.Core.Routing; @@ -79,7 +80,7 @@ public class RoutingContext : IRoutingContext /// /// Id or Name /// - public void Push(string agentId, string? reason = null) + public void Push(string agentId, string? reason = null, bool updateLazyRouting = true) { // Convert id to name if (!Guid.TryParse(agentId, out _)) @@ -99,13 +100,15 @@ public class RoutingContext : IRoutingContext HookEmitter.Emit(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason) ).Wait(); + + UpdateLazyRoutingAgent(updateLazyRouting); } } /// /// Pop current agent /// - public void Pop(string? reason = null) + public void Pop(string? reason = null, bool updateLazyRouting = true) { if (_stack.Count == 0) { @@ -149,15 +152,17 @@ public class RoutingContext : IRoutingContext _stack.Push(agentId); } } + + UpdateLazyRoutingAgent(updateLazyRouting); } - public void PopTo(string agentId, string reason) + public void PopTo(string agentId, string reason, bool updateLazyRouting = true) { var currentAgentId = GetCurrentAgentId(); while (!string.IsNullOrEmpty(currentAgentId) && currentAgentId != agentId) { - Pop(reason); + Pop(reason, updateLazyRouting: updateLazyRouting); currentAgentId = GetCurrentAgentId(); } } @@ -181,7 +186,7 @@ public class RoutingContext : IRoutingContext return _stack.ToArray().Contains(agentId); } - public void Replace(string agentId, string? reason = null) + public void Replace(string agentId, string? reason = null, bool updateLazyRouting = true) { var fromAgent = agentId; var toAgent = agentId; @@ -200,6 +205,8 @@ public class RoutingContext : IRoutingContext await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason) ).Wait(); } + + UpdateLazyRoutingAgent(updateLazyRouting); } public void Empty(string? reason = null) @@ -275,4 +282,24 @@ public class RoutingContext : IRoutingContext { _dialogs = []; } + + private void UpdateLazyRoutingAgent(bool updateLazyRouting) + { + if (!updateLazyRouting) + { + return; + } + + // Set next handling agent for lazy routing mode + var states = _services.GetRequiredService(); + var routingMode = states.GetState(StateConst.ROUTING_MODE, "hard"); + if (routingMode == "lazy") + { + var agentId = GetCurrentAgentId(); + if (agentId != BuiltInAgentId.Fallback) + { + states.SetState(StateConst.LAZY_ROUTING_AGENT_ID, agentId); + } + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs index c5317ba4..2e409fcf 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs @@ -37,5 +37,7 @@ public class RoutingPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index a4d4b37a..273fc996 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -53,7 +53,8 @@ public partial class RoutingService // Handle output routing exception. if (agent.Type == AgentType.Routing) { - response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; + // Forgot about what situation needs to handle in this way + // response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; } message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json new file mode 100644 index 00000000..31c8400f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -0,0 +1,31 @@ +{ + "name": "route_to_agent", + "description": "Route request to appropriate AI agent.", + "visibility_expression": "{% if states.routing_mode == 'lazy' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "next_action_agent": { + "type": "string", + "description": "Agent for next action based on user latest response" + }, + "next_action_reason": { + "type": "string", + "description": "The reason why route to this agent." + }, + "user_goal_agent": { + "type": "string", + "description": "Agent who can acheive user initial task." + }, + "conversation_end": { + "type": "boolean", + "description": "User is ending the conversation." + }, + "args": { + "type": "object", + "description": "Required parameters of next action agent" + } + }, + "required": [ "next_action_agent", "user_goal_agent", "args" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index bc614f81..997012db 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -14,6 +14,7 @@ Follow these steps to handle user request: {%- endfor %} {% endif %} +{% if routing_mode != 'lazy' %} [FUNCTIONS] {% for handler in routing_handlers -%} # {{ handler.description}} @@ -26,6 +27,7 @@ Parameters: {%- endif %} {{ "\r\n" }} {%- endfor %} +{% endif %} [AGENTS] {% for agent in routing_agents -%} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json new file mode 100644 index 00000000..cd9549c3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -0,0 +1,18 @@ +{ + "name": "util-routing-fallback_to_router", + "description": "Get the appropriate agent who can handle the user request.", + "parameters": { + "type": "object", + "properties": { + "fallback_reason": { + "type": "string", + "description": "The reason why you need to reach out to other agent." + }, + "user_question": { + "type": "string", + "description": "User question or statement." + } + }, + "required": [ "user_question" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid new file mode 100644 index 00000000..ab3863f8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid @@ -0,0 +1 @@ +"If you're unsure whether you understand the user's request or if the user brings up an unrelated topic, call the function `util-routing-fallback_to_router` to get the appropriate agent from the router." \ No newline at end of file From 2428349247fff8b80e085abe4d16bf3d90a64f52 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 25 Feb 2025 19:50:11 -0600 Subject: [PATCH 07/23] remove limit --- .../FileRepository/FileRepository.Conversation.cs | 10 ---------- .../Repository/MongoRepository.Conversation.cs | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index ec4848d7..7bd1dd15 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -500,16 +500,6 @@ public partial class FileRepository batchSize = batchLimit; } - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - foreach (var d in Directory.GetDirectories(dir)) { var convFile = Path.Combine(d, CONVERSATION_FILE); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 06512fb7..a78db60c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -466,16 +466,6 @@ public partial class MongoRepository batchSize = batchLimit; } - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - while (true) { var skip = (page - 1) * batchSize; From e68f245f2a1d82fd51ffbc2ec9e41c061e93ddc2 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 11:41:52 -0600 Subject: [PATCH 08/23] Add Twilio HangupPhoneCallFn --- .../Infrastructures/Enums/StateConst.cs | 1 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 3 +- .../Realtime/RealTimeCompletionProvider.cs | 10 +- .../BotSharp.Plugin.Twilio.csproj | 15 ++- .../Enums/UtilityName.cs | 2 +- .../Functions/HandleOutboundPhoneCallFn.cs | 106 --------------- .../Functions/HangupPhoneCallFn.cs | 35 +++++ .../Functions/OutboundPhoneCallFn.cs | 125 ++++++++++++++++++ .../OutboundPhoneCallHandlerUtilityHook.cs | 15 ++- .../LlmContexts/LlmContextIn.cs | 3 + .../util-twilio-hangup_phone_call.json | 15 +++ ...n => util-twilio-outbound_phone_call.json} | 7 +- .../util-twilio-hangup_phone_call.fn.liquid | 1 + .../util-twilio-outbound_phone_call.fn.liquid | 2 + ...wilio-twilio_outbound_phone_call.fn.liquid | 2 - 15 files changed, 218 insertions(+), 124 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json rename src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-twilio-twilio_outbound_phone_call.json => util-twilio-outbound_phone_call.json} (76%) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 466893d3..a21271df 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -12,4 +12,5 @@ public class StateConst public const string LANGUAGE = "language"; public const string SUB_CONVERSATION_ID = "sub_conversation_id"; + public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index c44f74d0..150959bb 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -4,6 +4,7 @@ using System; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Enums; namespace BotSharp.Core.Realtime; @@ -118,7 +119,7 @@ public class RealtimeHub : IRealtimeHub foreach (var message in messages) { // Invoke function - if (message.MessageType == "function_call") + if (message.MessageType == MessageTypeName.FunctionCall) { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 9b981cb1..efa96547 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; @@ -204,12 +205,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { + if (_webSocket.State != WebSocketState.Open) + { + return; + } + if (message is not string data) { data = JsonSerializer.Serialize(message); } var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); } @@ -559,7 +566,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion CurrentAgentId = conn.EntryAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, - ToolCallId = output.CallId + ToolCallId = output.CallId, + MessageType = MessageTypeName.FunctionCall }); } else if (output.Type == "message") diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 91c81e90..ece4e6f6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -9,15 +9,16 @@ - - - - - - + PreserveNewest - + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs index 63252a57..b7244e7d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Enums/UtilityName.cs @@ -2,6 +2,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Enums { public class UtilityName { - public const string OutboundPhoneCall = "twilio.twilio-outbound-phone-call"; + public const string OutboundPhoneCall = "phone.twilio-phone-call"; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs deleted file mode 100644 index 12b827af..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ /dev/null @@ -1,106 +0,0 @@ -using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Abstraction.Options; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.Twilio.Interfaces; -using BotSharp.Plugin.Twilio.Models; -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; -using Twilio.Rest.Api.V2010.Account; -using Twilio.Types; - -namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions -{ - public class HandleOutboundPhoneCallFn : IFunctionCallback - { - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly BotSharpOptions _options; - private readonly TwilioSetting _twilioSetting; - - public string Name => "util-twilio-twilio_outbound_phone_call"; - public string Indication => "Dialing the number"; - - public HandleOutboundPhoneCallFn( - IServiceProvider services, - ILogger logger, - BotSharpOptions options, - TwilioSetting twilioSetting) - { - _services = services; - _logger = logger; - _options = options; - _twilioSetting = twilioSetting; - } - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); - if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) - { - var error = $"Invalid phone number format: {args.PhoneNumber}"; - _logger.LogError(error); - message.Content = error; - return false; - } - - if (string.IsNullOrWhiteSpace(args.InitialMessage)) - { - _logger.LogError("Initial message is empty."); - message.Content = "There is an error when generating phone message."; - return false; - } - - var convService = _services.GetRequiredService(); - var convStorage = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - var fileStorage = _services.GetRequiredService(); - var sessionManager = _services.GetRequiredService(); - var states = _services.GetRequiredService(); - - // Fork conversation - var entryAgentId = routing.EntryAgentId; - var newConv = await convService.NewConversation(new Abstraction.Conversations.Models.Conversation - { - AgentId = entryAgentId, - Channel = ConversationChannel.Phone - }); - var conversationId = newConv.Id; - convStorage.Append(conversationId, new List - { - new RoleDialogModel(AgentRole.User, "Hi") - { - CurrentAgentId = entryAgentId - }, - new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) - { - CurrentAgentId = entryAgentId - } - }); - states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); - - // Generate audio - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); - var fileName = $"intial.mp3"; - fileStorage.SaveSpeechFile(conversationId, fileName, data); - - // Call phone number - /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage - { - Content = args.InitialMessage, - SpeechFileName = fileName - });*/ - - var call = await CallResource.CreateAsync( - // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"), - to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber)); - - message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; - message.StopCompletion = true; - return true; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs new file mode 100644 index 00000000..b4c7bca3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -0,0 +1,35 @@ +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class HangupPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public string Name => "util-twilio-hangup_phone_call"; + public string Indication => "Hangup"; + + public HangupPhoneCallFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + // Have to find the SID by the phone number + var call = CallResource.Update( + status: CallResource.UpdateStatusEnum.Completed, + pathSid: args.CallSid + ); + + message.Content = "The call has ended."; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs new file mode 100644 index 00000000..bc9e6203 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -0,0 +1,125 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; +using Twilio.Types; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class OutboundPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-outbound_phone_call"; + public string Indication => "Dialing the phone number"; + + public OutboundPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + if (args.PhoneNumber.Length != 12 || !args.PhoneNumber.StartsWith("+1", StringComparison.OrdinalIgnoreCase)) + { + var error = $"Invalid phone number format: {args.PhoneNumber}"; + _logger.LogError(error); + message.Content = error; + return false; + } + + if (string.IsNullOrWhiteSpace(args.InitialMessage)) + { + _logger.LogError("Initial message is empty."); + message.Content = "There is an error when generating phone message."; + return false; + } + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + + // Fork conversation + var newConversationId = Guid.NewGuid().ToString(); + states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); + + // Generate initial assistant audio + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + var fileName = $"intial.mp3"; + fileStorage.SaveSpeechFile(newConversationId, fileName, data); + + // Make outbound call + var call = await CallResource.CreateAsync( + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={newConversationId}&init_audio_file={fileName}"), + to: new PhoneNumber(args.PhoneNumber), + from: new PhoneNumber(_twilioSetting.PhoneNumber)); + + var convService = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var originConversationId = convService.ConversationId; + var entryAgentId = routing.EntryAgentId; + + await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); + + message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]"; + message.StopCompletion = true; + return true; + } + + private async Task ForkConversation(LlmContextIn args, + string entryAgentId, + string originConversationId, + string newConversationId, + CallResource resource) + { + // new scope service for isolated conversation + using var scope = _services.CreateScope(); + var services = scope.ServiceProvider; + var convService = services.GetRequiredService(); + var convStorage = services.GetRequiredService(); + + var newConv = await convService.NewConversation(new Conversation + { + Id = newConversationId, + AgentId = entryAgentId, + Channel = ConversationChannel.Phone, + ChannelId = resource.Sid, + Title = args.InitialMessage + }); + + convStorage.Append(newConversationId, new List + { + new RoleDialogModel(AgentRole.User, $"[Calling Phone To: {resource.ToFormatted}, Call SID: {resource.Sid}.]") + { + CurrentAgentId = entryAgentId + }, + new RoleDialogModel(AgentRole.Assistant, args.InitialMessage) + { + CurrentAgentId = entryAgentId + } + }); + + convService.SetConversationId(newConversationId, + [ + new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId), + new MessageState("phone_number", resource.To) + ]); + convService.SaveStates(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index 32639c9a..692610ff 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -6,15 +6,24 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; - private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}twilio_outbound_phone_call"; + private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; public void AddUtilities(List utilities) { var utility = new AgentUtility { Name = UtilityName.OutboundPhoneCall, - Functions = [new($"{OUTBOUND_PHONE_CALL_FN}")], - Templates = [new($"{OUTBOUND_PHONE_CALL_FN}.fn")] + Functions = + [ + new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{HANGUP_PHONE_CALL_FN}") + ], + Templates = + [ + new($"{OUTBOUND_PHONE_CALL_FN}.fn"), + new($"{HANGUP_PHONE_CALL_FN}.fn") + ] }; utilities.Add(utility); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index cde85aa9..b56db84e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -9,4 +9,7 @@ public class LlmContextIn [JsonPropertyName("initial_message")] public string InitialMessage { get; set; } + + [JsonPropertyName("call_sid")] + public string CallSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json new file mode 100644 index 00000000..63dd4327 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -0,0 +1,15 @@ +{ + "name": "util-twilio-hangup_phone_call", + "description": "Call this function if the user wants to end the phone call", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "call_sid": { + "type": "string", + "description": "The unique string SID that we created to identify this Call resource." + } + }, + "required": [ "call_sid" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json similarity index 76% rename from src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json rename to src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json index 768c6fde..a93ec3cf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-twilio_outbound_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-outbound_phone_call.json @@ -1,15 +1,16 @@ { - "name": "util-twilio-twilio_outbound_phone_call", + "name": "util-twilio-outbound_phone_call", "description": "If the user wants to initiate a phone call, you need to capture the phone number and compose the message the users wants to send. Then call this function to make an outbound call via Twilio.", + "visibility_expression": "{% if states.channel != 'phone' %}visible{% endif %}", "parameters": { "type": "object", "properties": { "phone_number": { - "to_read": "string", + "type": "string", "description": "The phone number which will be dialed. It needs to be a valid phone number starting with +1." }, "initial_message": { - "to_read": "string", + "type": "string", "description": "The initial message which will be sent." } }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid new file mode 100644 index 00000000..cc8360be --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid @@ -0,0 +1 @@ +** Please call util-twilio-hangup_phone_call if user wants to end the phone call. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid new file mode 100644 index 00000000..ada72fb9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -0,0 +1,2 @@ +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. +** For outbound calls, phone number format is "XXX-XXX-XXXX". \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid deleted file mode 100644 index 3bfb7dc2..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-twilio_outbound_phone_call.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please take a look at the conversation and decide whether user wants to make an outbound call. -** Please call util-twilio-twilio_outbound_phone_call if user wants to make an outbound call. \ No newline at end of file From 1468c01f4671f48825e2bc8019e7c415899dc087 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 13:16:57 -0600 Subject: [PATCH 09/23] Remove call_sid from function util-twilio-hangup_phone_call --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 2 +- .../Controllers/TwilioStreamController.cs | 4 +++- .../Functions/HangupPhoneCallFn.cs | 15 ++++++++++++--- .../Functions/OutboundPhoneCallFn.cs | 2 +- .../LlmContexts/LlmContextIn.cs | 3 --- .../functions/util-twilio-hangup_phone_call.json | 6 +----- .../util-twilio-outbound_phone_call.fn.liquid | 3 +-- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 150959bb..51c1bd33 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -139,7 +139,7 @@ public class RealtimeHub : IRealtimeHub if (!string.IsNullOrEmpty(message.Content)) { - await hook.OnMessageReceived(message); + await hook.OnResponseGenerated(message); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index e6de9fc5..774d014b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -93,6 +93,7 @@ public class TwilioStreamController : TwilioController Id = request.CallSid, AgentId = _settings.AgentId, Channel = ConversationChannel.Phone, + ChannelId = request.CallSid, Title = $"Phone call from {request.From}", Tags = [], }; @@ -103,7 +104,8 @@ public class TwilioStreamController : TwilioController var states = new List { new("channel", ConversationChannel.Phone), - new("calling_phone", request.From) + new("calling_phone", request.From), + new("twilio_call_sid", request.CallSid), }; convService.SetConversationId(conversation.Id, states); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index b4c7bca3..f54c1b42 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; @@ -21,14 +20,24 @@ public class HangupPhoneCallFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); + var states = _services.GetRequiredService(); + var callSid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(callSid)) + { + message.Content = "The call has not been initiated."; + _logger.LogError(message.Content); + return false; + } + // Have to find the SID by the phone number var call = CallResource.Update( status: CallResource.UpdateStatusEnum.Completed, - pathSid: args.CallSid + pathSid: callSid ); message.Content = "The call has ended."; + message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index bc9e6203..df97cd44 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -105,7 +105,7 @@ public class OutboundPhoneCallFn : IFunctionCallback convStorage.Append(newConversationId, new List { - new RoleDialogModel(AgentRole.User, $"[Calling Phone To: {resource.ToFormatted}, Call SID: {resource.Sid}.]") + new RoleDialogModel(AgentRole.User, "Hi") { CurrentAgentId = entryAgentId }, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs index b56db84e..cde85aa9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LlmContextIn.cs @@ -9,7 +9,4 @@ public class LlmContextIn [JsonPropertyName("initial_message")] public string InitialMessage { get; set; } - - [JsonPropertyName("call_sid")] - public string CallSid { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index 63dd4327..76b0e841 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -5,11 +5,7 @@ "parameters": { "type": "object", "properties": { - "call_sid": { - "type": "string", - "description": "The unique string SID that we created to identify this Call resource." - } }, - "required": [ "call_sid" ] + "required": [] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid index ada72fb9..d8fa9131 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid @@ -1,2 +1 @@ -** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. -** For outbound calls, phone number format is "XXX-XXX-XXXX". \ No newline at end of file +** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. \ No newline at end of file From 9d570311d10538b721cd7eb5b23636488a18b075 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Wed, 26 Feb 2025 22:40:50 -0600 Subject: [PATCH 10/23] realtime route to agent --- .../MLTasks/IRealTimeCompletion.cs | 2 +- .../BotSharp.Core/Realtime/RealtimeHub.cs | 23 ++++++++++++++----- .../Realtime/RealTimeCompletionProvider.cs | 4 ++-- .../Controllers/TwilioStreamController.cs | 18 +++++++-------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 8e1d11d5..bd958805 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -24,7 +24,7 @@ public interface IRealTimeCompletion Task Disconnect(); Task CreateSession(Agent agent, List conversations); - Task UpdateInitialSession(RealtimeHubConnection conn); + Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); Task TriggerModelInference(string? instructions = null); Task> OnResponsedDone(RealtimeHubConnection conn, string response); diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 51c1bd33..c33f28a2 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -1,9 +1,7 @@ using BotSharp.Abstraction.Realtime; using System.Net.WebSockets; -using System; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Enums; namespace BotSharp.Core.Realtime; @@ -12,6 +10,7 @@ public class RealtimeHub : IRealtimeHub { private readonly IServiceProvider _services; private readonly ILogger _logger; + public RealtimeHub(IServiceProvider services, ILogger logger) { _services = services; @@ -82,8 +81,7 @@ public class RealtimeHub : IRealtimeHub onModelReady: async () => { // Control initial session - await completer.UpdateInitialSession(conn); - + await completer.UpdateSession(conn); // Add dialog history foreach (var item in dialogs) @@ -123,8 +121,21 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); + if (message.FunctionName == "route_to_agent") + { + var routedAgentId = routing.Context.GetCurrentAgentId(); + if (conn.EntryAgentId != routedAgentId) + { + conn.EntryAgentId = routedAgentId; + await completer.UpdateSession(conn); + await completer.TriggerModelInference("Reply based on the function's output."); + } + } + else + { + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } } else { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index efa96547..834cdd17 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -254,13 +254,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } - public async Task UpdateInitialSession(RealtimeHubConnection conn) + public async Task UpdateSession(RealtimeHubConnection conn) { var convService = _services.GetRequiredService(); var conv = await convService.GetConversation(conn.ConversationId); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conv.AgentId); + var agent = await agentService.LoadAgent(conn.EntryAgentId); var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 774d014b..cd71ba01 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -1,13 +1,12 @@ using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Twilio.TwiML.Voice; using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; -using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -52,10 +51,6 @@ public class TwilioStreamController : TwilioController { request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; } - else - { - request.ConversationId = request.CallSid; - } await HookEmitter.Emit(_services, async hook => { @@ -65,7 +60,7 @@ public class TwilioStreamController : TwilioController OnlyOnce = true }); - await InitConversation(request); + request.ConversationId = await InitConversation(request); var twilio = _services.GetRequiredService(); @@ -82,7 +77,7 @@ public class TwilioStreamController : TwilioController return TwiML(response); } - private async Task InitConversation(ConversationalVoiceRequest request) + private async Task InitConversation(ConversationalVoiceRequest request) { var convService = _services.GetRequiredService(); var conversation = await convService.GetConversation(request.ConversationId); @@ -90,11 +85,10 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - Id = request.CallSid, AgentId = _settings.AgentId, Channel = ConversationChannel.Phone, ChannelId = request.CallSid, - Title = $"Phone call from {request.From}", + Title = $"Incoming phone call from {request.From}", Tags = [], }; @@ -106,9 +100,13 @@ public class TwilioStreamController : TwilioController new("channel", ConversationChannel.Phone), new("calling_phone", request.From), new("twilio_call_sid", request.CallSid), + // Enable lazy routing mode to optimize realtime experience + new(StateConst.ROUTING_MODE, "lazy"), }; convService.SetConversationId(conversation.Id, states); convService.SaveStates(); + + return conversation.Id; } } From 419b4717d6c333265a6283b8c0c0d09d8b19b027 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 27 Feb 2025 12:52:14 +0800 Subject: [PATCH 11/23] Support AgentId for twilio stream --- .../Controllers/TwilioStreamController.cs | 2 +- .../Models/ConversationalVoiceRequest.cs | 3 +++ .../agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json | 5 +++-- .../agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json | 3 ++- .../agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json | 5 +++-- .../agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json | 3 ++- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index cd71ba01..dcb33959 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -85,7 +85,7 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - AgentId = _settings.AgentId, + AgentId = request.AgentId ?? _settings.AgentId, Channel = ConversationChannel.Phone, ChannelId = request.CallSid, Title = $"Incoming phone call from {request.From}", diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 1fe78116..e2b6ee13 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -4,6 +4,9 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceRequest : VoiceRequest { + [FromQuery(Name = "agent-id")] + public string AgentId { get; set; } + [FromRoute] public string ConversationId { get; set; } diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json index 94746011..3a28b3e0 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/8970b1e5-d260-4e2c-90b1-f1415a257c18/agent.json @@ -1,7 +1,7 @@ { "id": "8970b1e5-d260-4e2c-90b1-f1415a257c18", "name": "Pizza Bot", - "description": "AI assistant that can help customer place pizza order.", + "description": "AI assistant that can help customer place pizza order, make payment or inquiry order status.", "type": "routing", "inheritAgentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", "createdDateTime": "2023-08-18T10:39:32.2349685Z", @@ -10,10 +10,11 @@ "disabled": false, "isPublic": true, "profiles": [ "pizza" ], + "labels": [ "experiment" ], "routingRules": [ { "type": "reasoner", - "field": "NaiveReasoner" + "field": "Naive Reasoner" } ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json index 4f606251..09f7489b 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/agent.json @@ -6,5 +6,6 @@ "id": "b284db86-e9c2-4c25-a59e-4649797dd130", "disabled": false, "isPublic": true, - "profiles": [ "pizza" ] + "profiles": [ "pizza" ], + "labels": [ "experiment" ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json index 62f069bb..40988cb2 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/agent.json @@ -1,10 +1,11 @@ { "name": "Ordering", - "description": "Provide types of pizza available, unit price and total cost. Place the order and returned the order number.", + "description": "Provide types of pizza available, unit price, total cost and place the order.", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd", "disabled": false, "isPublic": true, - "profiles": [ "pizza" ] + "profiles": [ "pizza" ], + "labels": [ "experiment" ] } \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json index 1dab7257..ef47f34d 100644 --- a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json +++ b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/agent.json @@ -1,12 +1,13 @@ { "name": "Payment", - "description": "Make payment when user wants to pay for the order", + "description": "Make payment when user confirmed the price and going to pay for the order", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "fe8c60aa-b114-4ef3-93cb-a8efeac80f75", "disabled": false, "isPublic": true, "profiles": [ "pizza" ], + "labels": [ "experiment" ], "routingRules": [ { "field": "order_number", From 3e0d1a4d2a48130b528c2af6dac5f4d05e69dbaa Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 27 Feb 2025 16:19:43 -0600 Subject: [PATCH 12/23] add try catch --- .../Hooks/ChatHubConversationHook.cs | 101 +++++++++++++----- .../Hooks/ChatHubCrontabHook.cs | 24 ++++- .../Hooks/StreamingLogHook.cs | 68 +++++++++--- .../Hooks/WelcomeHook.cs | 32 ++++-- .../BotSharp.Plugin.ChatHub/SignalRHub.cs | 24 +++-- 5 files changed, 189 insertions(+), 60 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 9f2c1008..99b734e6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -7,6 +7,7 @@ public class ChatHubConversationHook : ConversationHookBase { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly BotSharpOptions _options; private readonly ChatHubSettings _settings; @@ -23,12 +24,14 @@ public class ChatHubConversationHook : ConversationHookBase public ChatHubConversationHook( IServiceProvider services, IHubContext chatHub, + ILogger logger, BotSharpOptions options, ChatHubSettings settings, IUserIdentity user) { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _options = options; _settings = settings; @@ -177,74 +180,122 @@ public class ChatHubConversationHook : ConversationHookBase private async Task InitClientConversation(string conversationId, ConversationViewModel conversation) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(INIT_CLIENT_CONVERSATION, conversation); + _logger.LogWarning($"Failed to init client conversation in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task ReceiveClientMessage(string conversationId, ChatResponseModel model) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_CLIENT_MESSAGE, model); + _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task ReceiveAssistantMessage(string conversationId, string? json) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + _logger.LogWarning($"Failed to receive assistant message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } - + } private async Task GenerateSenderAction(string conversationId, ConversationSenderActionModel action) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_SENDER_ACTION, action); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_SENDER_ACTION, action); + _logger.LogWarning($"Failed to generate sender action in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task DeleteMessage(string conversationId, ChatResponseModel model) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(DELETE_MESSAGE, model); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(DELETE_MESSAGE, model); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(DELETE_MESSAGE, model); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(DELETE_MESSAGE, model); + _logger.LogWarning($"Failed to delete message in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task GenerateNotification(string conversationId, string? json) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_NOTIFICATION, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(GENERATE_NOTIFICATION, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_NOTIFICATION, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(GENERATE_NOTIFICATION, json); + _logger.LogWarning($"Failed to generate notification in {nameof(ChatHubConversationHook)} (conversation id: {conversationId})" + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } #endregion diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs index a00dae90..f4b4266f 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs @@ -8,6 +8,7 @@ public class ChatHubCrontabHook : ICrontabHook { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; @@ -19,6 +20,7 @@ public class ChatHubCrontabHook : ICrontabHook public ChatHubCrontabHook(IServiceProvider services, IHubContext chatHub, + ILogger logger, IUserIdentity user, IConversationStorage storage, BotSharpOptions options, @@ -26,6 +28,7 @@ public class ChatHubCrontabHook : ICrontabHook { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _storage = storage; _options = options; @@ -48,13 +51,26 @@ public class ChatHubCrontabHook : ICrontabHook } }, _options.JsonSerializerOptions); - if (_settings.EventDispatchBy == EventDispatchType.Group) + await SendEvent(item, json); + } + + private async Task SendEvent(CrontabItem item, string json) + { + try { - await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(item.ConversationId).SendAsync(GENERATE_NOTIFICATION, json); + } + else + { + await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(item.UserId).SendAsync(GENERATE_NOTIFICATION, json); + _logger.LogWarning($"Failed to send event in {nameof(ChatHubCrontabHook)} (conversation id: {item.ConversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index a2f527f4..351bb3a3 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -12,6 +12,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR private readonly ChatHubSettings _settings; private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IConversationStateService _state; private readonly IUserIdentity _user; private readonly IAgentService _agentService; @@ -30,6 +31,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR ChatHubSettings settings, IServiceProvider serivces, IHubContext chatHub, + ILogger logger, IConversationStateService state, IUserIdentity user, IAgentService agentService, @@ -40,6 +42,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR _settings = settings; _services = serivces; _chatHub = chatHub; + _logger = logger; _state = state; _user = user; _agentService = agentService; @@ -439,54 +442,85 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR #region Private methods private async Task SendContentLog(string conversationId, ContentLogInputModel input) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(CONTENT_LOG_GENERATED, BuildContentLog(input)); + _logger.LogWarning($"Failed to send content log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendStateLog(string conversationId, string agentId, Dictionary states, RoleDialogModel message) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(STATE_LOG_GENERATED, BuildStateLog(conversationId, agentId, states, message)); + _logger.LogWarning($"Failed to send state log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendAgentQueueLog(string conversationId, string log) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(AGENT_QUEUE_CHANGED, BuildAgentQueueChangedLog(conversationId, log)); + _logger.LogWarning($"Failed to send agent queue log in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } private async Task SendStateChange(string conversationId, StateChangeModel stateChange) { - if (_settings.EventDispatchBy == EventDispatchType.Group) + try { - await _chatHub.Clients.Group(conversationId).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + } } - else + catch (Exception ex) { - await _chatHub.Clients.User(_user.Id).SendAsync(STATE_CHANGED, BuildStateChangeLog(stateChange)); + _logger.LogWarning($"Failed to send state change in {nameof(StreamingLogHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); } } - private string BuildContentLog(ContentLogInputModel input) { var output = new ContentLogOutputModel diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index c7aedd83..70545305 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -6,6 +6,7 @@ public class WelcomeHook : ConversationHookBase { private readonly IServiceProvider _services; private readonly IHubContext _chatHub; + private readonly ILogger _logger; private readonly IUserIdentity _user; private readonly IConversationStorage _storage; private readonly BotSharpOptions _options; @@ -17,6 +18,7 @@ public class WelcomeHook : ConversationHookBase public WelcomeHook(IServiceProvider services, IHubContext chatHub, + ILogger logger, IUserIdentity user, IConversationStorage storage, BotSharpOptions options, @@ -24,6 +26,7 @@ public class WelcomeHook : ConversationHookBase { _services = services; _chatHub = chatHub; + _logger = logger; _user = user; _storage = storage; _options = options; @@ -78,17 +81,30 @@ public class WelcomeHook : ConversationHookBase _storage.Append(conversation.Id, dialog); - if (_settings.EventDispatchBy == EventDispatchType.Group) - { - await _chatHub.Clients.Group(conversation.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); - } - else - { - await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); - } + await SendEvent(conversation.Id, json); } } await base.OnUserAgentConnectedInitially(conversation); } + + private async Task SendEvent(string conversationId, string json) + { + try + { + if (_settings.EventDispatchBy == EventDispatchType.Group) + { + await _chatHub.Clients.Group(conversationId).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + else + { + await _chatHub.Clients.User(_user.Id).SendAsync(RECEIVE_ASSISTANT_MESSAGE, json); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Failed to send event in {nameof(WelcomeHook)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs index 240b5993..63ded25a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs @@ -33,12 +33,7 @@ public class SignalRHub : Hub if (!string.IsNullOrEmpty(conversationId)) { _logger.LogInformation($"Connection {Context.ConnectionId} is with conversation {conversationId}"); - - var settings = _services.GetRequiredService(); - if (settings.EventDispatchBy == EventDispatchType.Group) - { - await Groups.AddToGroupAsync(Context.ConnectionId, conversationId); - } + await AddGroup(conversationId); var conv = await convService.GetConversation(conversationId); if (conv != null) @@ -56,4 +51,21 @@ public class SignalRHub : Hub await base.OnConnectedAsync(); } + + private async Task AddGroup(string conversationId) + { + try + { + var settings = _services.GetRequiredService(); + if (settings.EventDispatchBy == EventDispatchType.Group) + { + await Groups.AddToGroupAsync(Context.ConnectionId, conversationId); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Failed to add chat group in {nameof(SignalRHub)} (conversation id: {conversationId})." + + $"\r\n{ex.Message}\r\n{ex.InnerException}"); + } + } } From a15821061f15e21fb148f2b027f9e13a03147558 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Thu, 27 Feb 2025 21:19:20 -0600 Subject: [PATCH 13/23] Upgrade package --- Directory.Packages.props | 20 +++++++++---------- .../BotSharp.Core/Realtime/RealtimeHub.cs | 4 ++++ .../Realtime/RealTimeCompletionProvider.cs | 7 ++++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index f6b1545e..42521480 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,24 +1,24 @@ 8.0.0 + 2.3.0 - - - - - - - + + + + + + + - - - + + diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index c33f28a2..83a251ae 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -75,6 +75,10 @@ public class RealtimeHub : IRealtimeHub var routing = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); + if (dialogs.Count == 0) + { + dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); + } routing.Context.SetDialogs(dialogs); await completer.Connect(conn, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 834cdd17..d4b74771 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -212,9 +212,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (message is not string data) { - data = JsonSerializer.Serialize(message); + data = JsonSerializer.Serialize(message, options: new JsonSerializerOptions + { + WriteIndented = true + }); } + _logger.LogInformation($"SendEventToModel:\r\n{data}"); + var buffer = Encoding.UTF8.GetBytes(data); await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); From 31fe5757863c39e68207ce5697e76509daa3552b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 28 Feb 2025 11:27:23 +0800 Subject: [PATCH 14/23] Fix SemanticKernel compile issue. --- .../BotSharp.Plugin.SemanticKernel.UnitTests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index c47fcead..ee159e86 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -10,7 +10,6 @@ - From 2ab73ead56f8a0eabb1d29b0a5ce1398f06da4d9 Mon Sep 17 00:00:00 2001 From: Haiping Date: Fri, 28 Feb 2025 09:51:48 -0600 Subject: [PATCH 15/23] Update agent.json --- .../data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json index c54a67e0..3200865e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/agent.json @@ -1,7 +1,7 @@ { "id": "01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d", "name": "Fallback Agent", - "description": "Don't have sufficient confidence to trigger any of existing agent.", + "description": "Handle initiated conversation without specific task given yet or don't have sufficient confidence to handle user task.", "type": "task", "createdDateTime": "2024-05-07T10:00:00Z", "updatedDateTime": "2024-05-07T10:00:00Z", From 152811cc722a1ab5bbbafd51182de48e749cb183 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 10:00:58 -0600 Subject: [PATCH 16/23] add agent id in resource key --- .../BotSharp.Core/Conversations/Services/TokenStatistics.cs | 5 +++-- .../BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 9fa5c718..8387e187 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -60,12 +60,13 @@ public class TokenStatistics : ITokenStatistics stat.SetState("llm_total_cost", total_cost, isNeedVersion: false, source: StateSource.Application); // Save stats + var agentId = message.CurrentAgentId ?? string.Empty; var globalStats = _services.GetRequiredService(); var body = new BotSharpStatsInput { Metric = StatsMetric.AgentLlmCost, Dimension = "agent", - DimRefVal = message.CurrentAgentId, + DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ @@ -75,7 +76,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats("global-llm-cost", body); + globalStats.UpdateStats($"global-llm-cost-{agentId}", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 2cc2c9fb..34fda963 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -25,17 +25,18 @@ public class GlobalStatsConversationHook : IContentGeneratingHook // record agent call var globalStats = _services.GetRequiredService(); + var agentId = message.CurrentAgentId ?? string.Empty; var body = new BotSharpStatsInput { Metric = StatsMetric.AgentCall, Dimension = "agent", - DimRefVal = message.CurrentAgentId ?? string.Empty, + DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats("global-agent-call", body); + globalStats.UpdateStats($"global-agent-call-{agentId}", body); } } From 14b6d41ce521f624fed85df7e7a0aa707ddbef10 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 12:44:59 -0600 Subject: [PATCH 17/23] OnSessionUpdated log --- .../Loggers/IContentGeneratingHook.cs | 11 ++++ .../Options/BotSharpOptions.cs | 2 +- .../Realtime/Models/RealtimeHubConnection.cs | 2 +- .../Conversations/Services/TokenStatistics.cs | 4 ++ .../BotSharp.Core/Realtime/RealtimeHub.cs | 20 +++---- .../Routing/RoutingService.InvokeAgent.cs | 2 +- .../util-routing-fallback_to_router.json | 2 +- .../util-routing-fallback_to_router.fn.liquid | 2 +- .../Hooks/StreamingLogHook.cs | 27 +++++++++ .../BotSharp.Plugin.OpenAI.csproj | 1 + .../Realtime/RealTimeCompletionProvider.cs | 59 ++++++++++++------- .../Providers/Text/TextCompletionProvider.cs | 13 +--- 12 files changed, 100 insertions(+), 45 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs index 9d0662db..c4f63450 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Abstraction.Loggers; /// @@ -37,4 +39,13 @@ public interface IContentGeneratingHook /// /// Task OnRenderingTemplate(Agent agent, string name, string content) => Task.CompletedTask; + + /// + /// Realtime session updated + /// + /// + /// + /// + /// + Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs index 90168872..b8609422 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Options/BotSharpOptions.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Options; public class BotSharpOptions { - private readonly static JsonSerializerOptions defaultJsonOptions = new JsonSerializerOptions() + public readonly static JsonSerializerOptions defaultJsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 60fec1dc..ec521637 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -4,7 +4,7 @@ public class RealtimeHubConnection { public string Event { get; set; } = null!; public string StreamId { get; set; } = null!; - public string EntryAgentId { get; set; } = null!; + public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public string Data { get; set; } = string.Empty; public string Model { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 9fa5c718..2be594f5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -110,6 +110,10 @@ public class TokenStatistics : ITokenStatistics public void StopTimer() { + if (_timer == null) + { + return; + } _timer.Stop(); } } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index 83a251ae..ee59fb62 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -71,9 +71,11 @@ public class RealtimeHub : IRealtimeHub var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conversation.AgentId); - conn.EntryAgentId = agent.Id; + conn.CurrentAgentId = agent.Id; var routing = _services.GetRequiredService(); + routing.Context.Push(agent.Id); + var dialogs = convService.GetDialogHistory(); if (dialogs.Count == 0) { @@ -125,21 +127,19 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - if (message.FunctionName == "route_to_agent") + if (message.FunctionName == "route_to_agent" || + message.FunctionName == "util-routing-fallback_to_router") { var routedAgentId = routing.Context.GetCurrentAgentId(); - if (conn.EntryAgentId != routedAgentId) + if (conn.CurrentAgentId != routedAgentId) { - conn.EntryAgentId = routedAgentId; + conn.CurrentAgentId = routedAgentId; await completer.UpdateSession(conn); - await completer.TriggerModelInference("Reply based on the function's output."); } } - else - { - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); - } + + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); } else { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 273fc996..bec44bc4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -54,7 +54,7 @@ public partial class RoutingService if (agent.Type == AgentType.Routing) { // Forgot about what situation needs to handle in this way - // response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; + response.Content = "Apologies, I'm not quite sure I understand. Could you please provide additional clarification or context?"; } message = RoleDialogModel.From(message, role: AgentRole.Assistant, content: response.Content); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json index cd9549c3..77f0dbfd 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -1,6 +1,6 @@ { "name": "util-routing-fallback_to_router", - "description": "Get the appropriate agent who can handle the user request.", + "description": "Return to the Router to find the appropriate agent who can handle the user's request.", "parameters": { "type": "object", "properties": { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid index ab3863f8..a1bd9ceb 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-routing-fallback_to_router.fn.liquid @@ -1 +1 @@ -"If you're unsure whether you understand the user's request or if the user brings up an unrelated topic, call the function `util-routing-fallback_to_router` to get the appropriate agent from the router." \ No newline at end of file +Carefully consider whether the current user request is related to your responsibilities. Only when it is not relevant should you consider Return to the Router. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 351bb3a3..22ec7d4d 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -85,6 +85,33 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR await SendContentLog(conversationId, input); } + public async Task OnSessionUpdated(Agent agent, string instruction, FunctionDef[] functions) + { + var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + + // Agent queue log + var log = $"{instruction}"; + if (functions.Length > 0) + { + log += $"\r\n\r\n[FUNCTIONS]:\r\n\r\n{string.Join("\r\n\r\n", functions.Select(x => JsonSerializer.Serialize(x, BotSharpOptions.defaultJsonOptions)))}"; + } + _logger.LogInformation(log); + + var message = new RoleDialogModel(AgentRole.Assistant, log) + { + MessageId = _routingCtx.MessageId + }; + var input = new ContentLogInputModel(conversationId, message) + { + Name = agent.Name, + AgentId = agent.Id, + Source = ContentLogSource.Prompt, + Log = log + }; + await SendContentLog(conversationId, input); + } + public async Task OnRenderingTemplate(Agent agent, string name, string content) { if (!_convSettings.ShowVerboseLog) return; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj index 9a9c57fb..0a509b1a 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj @@ -18,6 +18,7 @@ + \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index d4b74771..722972f8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,7 +1,9 @@ using BotSharp.Abstraction.Conversations.Enums; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; using System.Net.WebSockets; @@ -212,14 +214,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (message is not string data) { - data = JsonSerializer.Serialize(message, options: new JsonSerializerOptions - { - WriteIndented = true - }); + data = JsonSerializer.Serialize(message, BotSharpOptions.defaultJsonOptions); } - _logger.LogInformation($"SendEventToModel:\r\n{data}"); - var buffer = Encoding.UTF8.GetBytes(data); await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); @@ -265,13 +262,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var conv = await convService.GetConversation(conn.ConversationId); var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(conn.EntryAgentId); + var agent = await agentService.LoadAgent(conn.CurrentAgentId); var client = ProviderHelper.GetClient(Provider, _model, _services); var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, []); var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + var functions = options.Tools.Select(x => + { + var fn = new FunctionDef + { + Name = x.FunctionName, + Description = x.FunctionDescription + }; + fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); + return fn; + }).ToArray(); var sessionUpdate = new { @@ -287,21 +294,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Voice = "alloy", Instructions = instruction, ToolChoice = "auto", - Tools = options.Tools.Select(x => - { - var fn = new FunctionDef - { - Name = x.FunctionName, - Description = x.FunctionDescription - }; - fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); - return fn; - }).ToArray(), + Tools = functions, Modalities = [ "text", "audio" ], Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) } }; + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionUpdated(agent, instruction, functions); + }); + await SendEventToModel(sessionUpdate); } @@ -568,7 +571,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { outputs.Add(new RoleDialogModel(output.Role, output.Arguments) { - CurrentAgentId = conn.EntryAgentId, + CurrentAgentId = conn.CurrentAgentId, FunctionName = output.Name, FunctionArgs = output.Arguments, ToolCallId = output.CallId, @@ -581,11 +584,27 @@ public class RealTimeCompletionProvider : IRealTimeCompletion outputs.Add(new RoleDialogModel(output.Role, content.Transcript) { - CurrentAgentId = conn.EntryAgentId + CurrentAgentId = conn.CurrentAgentId }); } } + var contentHooks = _services.GetServices().ToList(); + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") + { + CurrentAgentId = conn.CurrentAgentId + }, new TokenStatsModel + { + Provider = Provider, + Model = _model, + CompletionCount = data.Usage.OutputTokens, + PromptCount = data.Usage.InputTokens + }); + } + return outputs; } @@ -594,7 +613,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) { - CurrentAgentId = conn.EntryAgentId + CurrentAgentId = conn.CurrentAgentId }; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index c2180076..097a32b7 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -3,6 +3,7 @@ using System.Net.Http; using System.Net.Mime; using System.Text.Json; using System.Text; +using BotSharp.Abstraction.Options; namespace BotSharp.Plugin.OpenAI.Providers.Text; @@ -13,14 +14,6 @@ public class TextCompletionProvider : ITextCompletion private readonly OpenAiSettings _settings; protected string _model; - protected readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - PropertyNameCaseInsensitive = true, - WriteIndented = true, - AllowTrailingCommas = true, - }; - public virtual string Provider => "openai"; public TextCompletionProvider( @@ -110,7 +103,7 @@ public class TextCompletionProvider : ITextCompletion MaxTokens = maxTokens, Temperature = temperature }; - var data = JsonSerializer.Serialize(request, _jsonOptions); + var data = JsonSerializer.Serialize(request, BotSharpOptions.defaultJsonOptions); var httpRequest = new HttpRequestMessage { Method = HttpMethod.Post, @@ -121,7 +114,7 @@ public class TextCompletionProvider : ITextCompletion var httpResponse = await httpClient.SendAsync(httpRequest); httpResponse.EnsureSuccessStatusCode(); var responseStr = await httpResponse.Content.ReadAsStringAsync(); - var response = JsonSerializer.Deserialize(responseStr, _jsonOptions); + var response = JsonSerializer.Deserialize(responseStr, BotSharpOptions.defaultJsonOptions); return response; } catch (Exception ex) From 606026130a07f7b5e3d326e481abbb357d1f8ca8 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 15:26:45 -0600 Subject: [PATCH 18/23] rollback reset states --- .../Conversations/Services/ConversationStateService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 4d229866..5d19e2d0 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -220,7 +220,6 @@ public class ConversationStateService : IConversationStateService { if (_conversationId == null || _sidecar?.IsEnabled() == true) { - Reset(); return; } @@ -253,7 +252,6 @@ public class ConversationStateService : IConversationStateService } _db.UpdateConversationStates(_conversationId, states); - Reset(); _logger.LogInformation($"Saved states of conversation {_conversationId}"); } From 2851851657df96f3b3d9da04315ca92507fddd75 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 15:34:53 -0600 Subject: [PATCH 19/23] refine lock key --- .../Conversations/Services/TokenStatistics.cs | 8 +++++--- .../BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 8387e187..68f5ecf0 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -60,12 +60,14 @@ public class TokenStatistics : ITokenStatistics stat.SetState("llm_total_cost", total_cost, isNeedVersion: false, source: StateSource.Application); // Save stats + var metric = StatsMetric.AgentLlmCost; + var dim = "agent"; var agentId = message.CurrentAgentId ?? string.Empty; var globalStats = _services.GetRequiredService(); var body = new BotSharpStatsInput { - Metric = StatsMetric.AgentLlmCost, - Dimension = "agent", + Metric = metric, + Dimension = dim, DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, @@ -76,7 +78,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats($"global-llm-cost-{agentId}", body); + globalStats.UpdateStats($"global-llm-cost-{metric}-{dim}-{agentId}", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 34fda963..66b0e8ef 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -25,11 +25,13 @@ public class GlobalStatsConversationHook : IContentGeneratingHook // record agent call var globalStats = _services.GetRequiredService(); + var metric = StatsMetric.AgentCall; + var dim = "agent"; var agentId = message.CurrentAgentId ?? string.Empty; var body = new BotSharpStatsInput { - Metric = StatsMetric.AgentCall, - Dimension = "agent", + Metric = metric, + Dimension = dim, DimRefVal = agentId, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, @@ -37,6 +39,6 @@ public class GlobalStatsConversationHook : IContentGeneratingHook new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats($"global-agent-call-{agentId}", body); + globalStats.UpdateStats($"global-agent-call-{metric}-{dim}-{agentId}", body); } } From 64831f5b3a16cf54338cea425e578b90be7728fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 15:35:46 -0600 Subject: [PATCH 20/23] minor change --- .../BotSharp.Core/Conversations/Services/TokenStatistics.cs | 2 +- .../BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 388c814c..0698c0cb 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -78,7 +78,7 @@ public class TokenStatistics : ITokenStatistics new StatsKeyValuePair("completion_cost_total", deltaCompletionCost) ] }; - globalStats.UpdateStats($"global-llm-cost-{metric}-{dim}-{agentId}", body); + globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 66b0e8ef..8ff9be86 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -39,6 +39,6 @@ public class GlobalStatsConversationHook : IContentGeneratingHook new StatsKeyValuePair("agent_call_count", 1) ] }; - globalStats.UpdateStats($"global-agent-call-{metric}-{dim}-{agentId}", body); + globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body); } } From 235d972ca1f9af8fb2a11fb91f524b79d5482dd8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 28 Feb 2025 16:47:03 -0600 Subject: [PATCH 21/23] remove data --- .../Conversations/Models/Conversation.cs | 7 +------ .../Conversations/Services/ConversationStorage.cs | 15 +++++---------- .../Models/DialogMongoElement.cs | 7 ++----- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index 07258402..ac1efb04 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -58,17 +58,13 @@ public class DialogElement [JsonPropertyName("payload")] public string? Payload { get; set; } - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("data")] - public object? Data { get; set; } - public DialogElement() { } public DialogElement(DialogMetaData meta, string content, string? richContent = null, - string? secondaryContent = null, string? secondaryRichContent = null, string? payload = null, object? data = null) + string? secondaryContent = null, string? secondaryRichContent = null, string? payload = null) { MetaData = meta; Content = content; @@ -76,7 +72,6 @@ public class DialogElement SecondaryContent = secondaryContent; SecondaryRichContent = secondaryRichContent; Payload = payload; - Data = data; } public override string ToString() diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 89e9fcaa..75d1a501 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -52,8 +52,7 @@ public class ConversationStorage : IConversationStorage MetaData = meta, Content = dialog.Content, SecondaryContent = dialog.SecondaryContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } else @@ -84,8 +83,7 @@ public class ConversationStorage : IConversationStorage SecondaryContent = dialog.SecondaryContent, RichContent = richContent, SecondaryRichContent = secondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } @@ -123,8 +121,7 @@ public class ConversationStorage : IConversationStorage MetaData = meta, Content = dialog.Content, SecondaryContent = dialog.SecondaryContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } else @@ -155,8 +152,7 @@ public class ConversationStorage : IConversationStorage SecondaryContent = dialog.SecondaryContent, RichContent = richContent, SecondaryRichContent = secondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }); } } @@ -200,8 +196,7 @@ public class ConversationStorage : IConversationStorage RichContent = richContent, SecondaryContent = secondaryContent, SecondaryRichContent = secondaryRichContent, - Payload = payload, - Data = dialog.Data + Payload = payload }; results.Add(record); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 87022130..030a93ab 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -11,7 +11,6 @@ public class DialogMongoElement public string? RichContent { get; set; } public string? SecondaryRichContent { get; set; } public string? Payload { get; set; } - public object? Data { get; set; } public static DialogMongoElement ToMongoElement(DialogElement dialog) { @@ -22,8 +21,7 @@ public class DialogMongoElement SecondaryContent = dialog.SecondaryContent, RichContent = dialog.RichContent, SecondaryRichContent = dialog.SecondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }; } @@ -36,8 +34,7 @@ public class DialogMongoElement SecondaryContent = dialog.SecondaryContent, RichContent = dialog.RichContent, SecondaryRichContent = dialog.SecondaryRichContent, - Payload = dialog.Payload, - Data = dialog.Data + Payload = dialog.Payload }; } } From e3036ef3ba3630608d32c43955a02040066ff555 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 18:19:44 -0600 Subject: [PATCH 22/23] add welcome for realtime --- .../MLTasks/IRealTimeCompletion.cs | 2 + .../BotSharp.Core/Realtime/RealtimeHub.cs | 40 +++++++++++++------ .../functions/route_to_agent.json | 2 +- .../instructions/instruction.liquid | 2 + .../util-routing-fallback_to_router.json | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 32 ++++++++++++++- 6 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index bd958805..be6f8821 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -26,7 +26,9 @@ public interface IRealTimeCompletion Task CreateSession(Agent agent, List conversations); Task UpdateSession(RealtimeHubConnection conn); Task InsertConversationItem(RoleDialogModel message); + Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); + Task CancelModelResponse(); Task> OnResponsedDone(RealtimeHubConnection conn, string response); Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index ee59fb62..eeac2ab3 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -3,6 +3,7 @@ using System.Net.WebSockets; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Core.Realtime; @@ -97,7 +98,7 @@ public class RealtimeHub : IRealtimeHub if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) { - // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); } else { @@ -127,19 +128,32 @@ public class RealtimeHub : IRealtimeHub { await routing.InvokeFunction(message.FunctionName, message); message.Role = AgentRole.Function; - if (message.FunctionName == "route_to_agent" || - message.FunctionName == "util-routing-fallback_to_router") - { - var routedAgentId = routing.Context.GetCurrentAgentId(); - if (conn.CurrentAgentId != routedAgentId) - { - conn.CurrentAgentId = routedAgentId; - await completer.UpdateSession(conn); - } - } - await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply based on the function's output."); + if (message.FunctionName == "route_to_agent") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + message.Content = $"Connected to agent of {inst.AgentName}"; + conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await completer.UpdateSession(conn); + await completer.InsertConversationItem(message); + await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}."); + } + else if (message.FunctionName == "util-routing-fallback_to_router") + { + var inst = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + message.Content = $"Returned to Router due to {inst.Reason}"; + conn.CurrentAgentId = routing.Context.GetCurrentAgentId(); + + await completer.UpdateSession(conn); + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply user request."); + } + else + { + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } } else { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json index 31c8400f..08f938ec 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json @@ -26,6 +26,6 @@ "description": "Required parameters of next action agent" } }, - "required": [ "next_action_agent", "user_goal_agent", "args" ] + "required": [ "next_action_agent", "user_goal_agent", "next_action_reason", "args" ] } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid index 997012db..7ee8f56e 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid @@ -5,7 +5,9 @@ Follow these steps to handle user request: 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service. 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments. 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared. +{% if routing_mode != 'lazy' %} 5. Response must be in JSON format. +{% endif %} {% if routing_requirements and routing_requirements != empty %} [REQUIREMENTS] diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json index 77f0dbfd..3902d8ae 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json @@ -13,6 +13,6 @@ "description": "User question or statement." } }, - "required": [ "user_question" ] + "required": [ "fallback_reason" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 722972f8..2be07da8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -102,6 +102,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion }); } + public async Task CancelModelResponse() + { + await SendEventToModel(new + { + type = "response.cancel" + }); + } + + public async Task RemoveConversationItem(string itemId) + { + await SendEventToModel(new + { + type = "conversation.item.delete", + item_id = itemId + }); + } + private async Task ReceiveMessage(RealtimeHubConnection conn, Action onModelAudioDeltaReceived, Action onModelAudioResponseDone, @@ -169,7 +186,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "response.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - await Task.Delay(1000); var messages = await OnResponsedDone(conn, receivedText); onModelResponseDone(messages); } @@ -296,7 +312,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) + Temperature = Math.Max(options.Temperature ?? 0f, 0.6f), + MaxResponseOutputTokens = 512, + TurnDetection = new RealtimeSessionTurnDetection + { + Threshold = 0.8f, + SilenceDuration = 800 + } } }; @@ -565,6 +587,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var outputs = new List(); var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + return []; + } + foreach (var output in data.Outputs) { if (output.Type == "function_call") @@ -575,6 +602,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion FunctionName = output.Name, FunctionArgs = output.Arguments, ToolCallId = output.CallId, + MessageId = output.Id, MessageType = MessageTypeName.FunctionCall }); } From c096b6365682386c320fa7993ffc33df6589310a Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Fri, 28 Feb 2025 20:42:03 -0600 Subject: [PATCH 23/23] Output input audio transcript. --- .../BotSharp.Core/Realtime/RealtimeHub.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs index eeac2ab3..7af86a45 100644 --- a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -137,7 +137,7 @@ public class RealtimeHub : IRealtimeHub await completer.UpdateSession(conn); await completer.InsertConversationItem(message); - await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}."); + await completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures."); } else if (message.FunctionName == "util-routing-fallback_to_router") { @@ -147,7 +147,7 @@ public class RealtimeHub : IRealtimeHub await completer.UpdateSession(conn); await completer.InsertConversationItem(message); - await completer.TriggerModelInference("Reply user request."); + await completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}"); } else { @@ -157,7 +157,7 @@ public class RealtimeHub : IRealtimeHub } else { - // append transcript to conversation + // append output audio transcript to conversation storage.Append(conn.ConversationId, message); dialogs.Add(message); @@ -166,10 +166,7 @@ public class RealtimeHub : IRealtimeHub hook.SetAgent(agent) .SetConversation(conversation); - if (!string.IsNullOrEmpty(message.Content)) - { - await hook.OnResponseGenerated(message); - } + await hook.OnResponseGenerated(message); } } } @@ -180,9 +177,17 @@ public class RealtimeHub : IRealtimeHub }, onInputAudioTranscriptionCompleted: async message => { - // append transcript to conversation + // append input audio transcript to conversation storage.Append(conn.ConversationId, message); dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + await hook.OnMessageReceived(message); + } }, onUserInterrupted: async () => {