From 8d5e78a619837903380d324957798e0c20c78e21 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 4 Sep 2024 13:44:05 -0500 Subject: [PATCH 1/2] refine email sender and image generator prompt --- .../Conversations/Models/RoleDialogModel.cs | 2 +- .../templates/handle_email_sender.fn.liquid | 6 ++++-- .../templates/generate_image.fn.liquid | 3 ++- .../BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 8484c5fb..5b8f7017 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -63,7 +63,7 @@ public class RoleDialogModel : ITrackableMessage /// It's ideal to render in rich content in UI. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public object Data { get; set; } + public object? Data { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string ImageUrl { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_sender.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_sender.fn.liquid index 647942c6..eb6f0e6f 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_sender.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_sender.fn.liquid @@ -1,2 +1,4 @@ -Please call handle_email_sender if user wants to send email. -** Please take a look at the conversation and decide whether user wants to send email with files/attachments/images or not. +** Please take a look at the conversation and decide whether user wants to send email. +** If yes, then decide whether user wants to send email with files/attachments/images or not. +** If user does not want to send email explicitly or only upload images or files, please do not respond anything email related message. +** Please call handle_email_sender if user wants to send email. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/generate_image.fn.liquid index 4ab76c60..9ee28f9c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/generate_image.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/generate_image.fn.liquid @@ -1 +1,2 @@ -Please call generate_image if user wants you to provide or generate an image or picture. \ No newline at end of file +** Please call generate_image if user wants you to provide or generate an image or picture. +** If user does not generate image explicitly, please do not call generate_image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index eda46f50..dd55ae7e 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -43,6 +43,7 @@ public class SummaryPlanFn : IFunctionCallback var msgCopy = RoleDialogModel.From(message); await fn.InvokeFunction("get_table_definition", msgCopy); var ddlStatements = msgCopy.Content; + message.Data = null; // Summarize and generate query var summaryPlanPrompt = await GetPlanSummaryPrompt(task, message.Content, ddlStatements); From 348888ae638957edc9fe46fad9a96e0c02bc6a2e Mon Sep 17 00:00:00 2001 From: Joanna Ren <101223@smsassist.com> Date: Wed, 4 Sep 2024 15:58:03 -0500 Subject: [PATCH 2/2] Improve Primary Planning --- .../Functions/PrimaryStagePlanFn.cs | 32 +++++++++++++------ .../Models/PrimaryRequirementRequest.cs | 7 ++-- .../instructions/instruction.liquid | 10 +++++- .../functions/plan_primary_stage.json | 16 ++++++---- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 14846137..d6347641 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -1,4 +1,8 @@ +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Functions; using BotSharp.Abstraction.Knowledges; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Routing; using BotSharp.Plugin.Planner.TwoStaging.Models; namespace BotSharp.Plugin.Planner.Functions; @@ -27,18 +31,26 @@ public class PrimaryStagePlanFn : IFunctionCallback state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - - // Get knowledge from vectordb - var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; ; - var knowledges = await knowledgeService.SearchVectorKnowledge(task.Requirements, collectionName, new VectorSearchOptions + + //get knowledge from vectordb + var knowledges = new List(); + foreach (var question in task.Questions) { - Confidence = 0.1f - }); - message.Content = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); + var retrievalMessage = new RoleDialogModel(AgentRole.User, question) + { + FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge + { + Question = question + }), + Content = "" + }; + await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); + knowledges.Add(retrievalMessage.Content); + } // Send knowledge to AI to refine and summarize the primary planning var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message); + var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, knowledges); var plannerAgent = new Agent { Id = BuiltInAgentId.Planner, @@ -53,7 +65,7 @@ public class PrimaryStagePlanFn : IFunctionCallback return true; } - private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message) + private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest task, List relevantKnowledges) { var agentService = _services.GetRequiredService(); var render = _services.GetRequiredService(); @@ -78,7 +90,7 @@ public class PrimaryStagePlanFn : IFunctionCallback { { "task_description", task.Requirements }, { "global_knowledges", globalKnowledges }, - { "relevant_knowledges", new[]{ message.Content } }, + { "relevant_knowledges", relevantKnowledges }, { "response_format", responseFormat } }); } diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/PrimaryRequirementRequest.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/PrimaryRequirementRequest.cs index a19cc984..87c05f16 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/PrimaryRequirementRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/PrimaryRequirementRequest.cs @@ -5,9 +5,6 @@ public class PrimaryRequirementRequest [JsonPropertyName("requirement_detail")] public string Requirements { get; set; } = null!; - [JsonPropertyName("has_knowledge_reference")] - public bool HasKnowledgeReference { get; set; } - - [JsonPropertyName("question")] - public string Question { get; set; } = null!; + [JsonPropertyName("questions")] + public string[] Questions { get; set; } = []; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 7209f7dd..f6571d18 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,3 +1,11 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, call plan_primary_stage. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. -You must Call plan_summary as the last step to summarize the final planning steps. \ No newline at end of file +You must Call plan_summary as the last step to summarize the final planning steps. + +{% if global_knowledges != empty -%} +===== +Global Knowledge: +{% for k in global_knowledges %} +{{ k }} +{% endfor %} +{%- endif %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/plan_primary_stage.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/plan_primary_stage.json index fcd96595..e1162634 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/plan_primary_stage.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/plan_primary_stage.json @@ -1,6 +1,6 @@ { "name": "plan_primary_stage", - "description": "Extract user's original requirements with detail specifications to make the primary plan, you have to include every important informations.", + "description": "Plan the high level steps to finish the task", "parameters": { "type": "object", "properties": { @@ -8,11 +8,15 @@ "type": "string", "description": "User original requirements in detail, don't miss any information especially for those line items, values and numbers." }, - "question": { - "type": "string", - "description": "Question convert from requirement and reference tables for knowledge search. The question should contain all the detailed information in the requirement" + "questions": { + "type": "array", + "description": "User requirements in detail, don't miss any information especially for those line items, values and numbers.", + "items": { + "type": "string", + "description": "Question converted from requirement in different ways to search in the knowledge base, be short" + } } }, - "required": [ "requirement_detail", "question" ] + "required": [ "requirement_detail", "questions" ] } -} \ No newline at end of file +}