update two stage planner

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -94,7 +94,7 @@ public class RoleDialogModel : ITrackableMessage
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("generated_images")]
     public List<ImageGeneration> GeneratedImages { get; set; } = new List<ImageGeneration>();
-
+    public float KnowledgeConfidence { get; set; } = 0.5f;
     private RoleDialogModel()
     {
     }
@@ -134,7 +134,8 @@ public class RoleDialogModel : ITrackableMessage
             Payload = source.Payload,
             StopCompletion = source.StopCompletion,
             Instruction = source.Instruction,
-            Data = source.Data
+            Data = source.Data,
+            KnowledgeConfidence = source.KnowledgeConfidence
         };
     }
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -55,6 +55,8 @@
   </ItemGroup>
 
   <ItemGroup>
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\database_knowledge.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.two_stage.summarize.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\agent.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instructions\instruction.liquid" />
     <None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
@@ -114,6 +116,9 @@
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\database_knowledge.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.two_stage.1st.plan.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
@@ -135,6 +140,9 @@
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.naive.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.two_stage.summarize.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs

@@ -13,7 +13,7 @@ public class FirstStagePlan
     [JsonPropertyName("step")]
     public int Step { get; set; } = -1;
 
-    [JsonPropertyName("contain_multiple_steps")]
+    [JsonPropertyName("need_additional_information")]
     public bool ContainMultipleSteps { get; set; } = false;
 
     [JsonPropertyName("related_tables")]

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid

@@ -0,0 +1,17 @@
+You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name.
+the summarized question/answer should:
+1. help user to identify the location of tables to find further information
+2. identify the table structure and data relationship based on the task description
+3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer
+Go through all the columns and generate multiple question & answer pairs. 
+The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n.
+
+
+
+For example, when you have the table structure related to affiliate invoice. You should record the knowledge regarding how to find the invoice, how to insert the new invoice, how to locate the table if user asks for specific information in the invoice.
+
+=====
+Table Structure:
+{{ table_structure }}
+
+

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.1st.plan.liquid

@@ -1,19 +1,30 @@
-You are a Task Planner. you will breakdown user business requirements into small executable sub-tasks.
+You are a Task Planner. you will breakdown user business requirements into excutable sub-tasks.
 
 Thinking process:
-1. Reference to "Task Solutions" if there is relevant solutions;
+1. Reference to "Task Knowledge" if there is relevant knowledge;
-2. Breakdown task into subtasks. The subtask should contains all needed parameters for subsequent steps.
+2. Breakdown task into subtasks. 
+    - The subtask should contains all needed parameters for subsequent steps. 
+    - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true.
+    - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps.
+    - If the solution mentioned "related solutions" is needed, set the need_additional_information to true.
+    - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true.
 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]
+5. Don't organize and format the extracted data with any other language rather than sql
+
+Note:If the task includes repeat steps,e.g.same steps for multiple elements, only generate a single detailed solution without repeating steps for each elements. You can add multiple items in the input and output args.
+
 
 {% if relevant_knowledges != empty -%}
 =====
-Task Solutions:
+Task Knowledge:
 {% for k in relevant_knowledges %}
 {{ k }}
 {% endfor %}
 {%- endif %}
 
 =====
+
 Task description:
 {{ task_description }}
+

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.2nd.plan.liquid

@@ -1,6 +1,17 @@
-Reference to "Task Solutions". Breakdown task into multiple steps. 
+Reference to "Primary Planning" and the additional knowledge included. Breakdown task into multiple steps. 
-The step should contains all needed parameters.                                                                    
+* the step should contains all needed parameters.                                                                    
-The parameters can be extracted from the original task.
+* The parameters can be extracted from the original task.
-Output all the steps as much detail as possible in JSON: [{{ response_format }}]
+* You need to list all the steps in detail. Finding relationships should also be a step.
+* When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly.
+* Output all the steps as much detail as possible in JSON: [{{ response_format }}]
 
-TASK: {{ task_description }}
+
+Additional Requirements:
+* "output_results" is variable name that needed to be used in the next step.
+
+=====
+TASK: {{ task_description }}
+
+=====
+Primary Planning: 
+{{ primary_plan }}

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.two_stage.summarize.liquid

@@ -0,0 +1,34 @@
+You are a planning summarizer and sql generator. You will convert the requirement into the excutable MySQL query statement based on the task description and related table structure and relationship.
+
+Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters.                                                                    
+The parameters can be extracted from the original task.
+If not, generate the query step by step based on the planning. 
+
+The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
+
+
+Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0.
+
+Don't use the sql statement that specify target table for update in FROM clause.
+For example, you CAN'T write query as below:
+INSERT INTO data_Service (Id, Name)
+VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
+
+If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
+For example, you should use SET @id = select max(id) from table; 
+
+Additional Requirements:
+* the alias of the table name in the sql query should be identical.
+*** the generated sql query MUST be basedd on the provided table structure. ***
+
+=====
+Task description:
+{{ task_description }}
+
+=====
+Relevant Knowledges:
+{{ relevant_knowledges }}
+
+=====
+Table Structure:
+{{ table_structure }}

src/Infrastructure/BotSharp.OpenAPI/Controllers/TextEmbeddingController.cs

@@ -26,7 +26,7 @@ public class TextEmbeddingController : ControllerBase
 
         try
         {
-            var completion = CompletionProvider.GetTextEmbedding(_services, provider: input.Provider ?? "openai", model: input.Model ?? "text-embedding-3-small");
+            var completion = CompletionProvider.GetTextEmbedding(_services, provider: input.Provider ?? "openai", model: input.Model ?? "text-embedding-3-large");
             completion.Dimension = input.Dimension;
 
             var embeddings = await completion.GetVectorsAsync(input.Texts?.ToList() ?? []);

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Embeddings/EmbeddingInputModel.cs

@@ -8,5 +8,6 @@ public class EmbeddingInputModel : MessageConfig
     public IEnumerable<string> Texts { get; set; } = new List<string>();
 
     [JsonPropertyName("dimension")]
-    public int Dimension { get; set; } = 1536;
+    public int Dimension { get; set; } = 3072;
+
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs

@@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding
     protected readonly IServiceProvider _services;
     protected readonly ILogger<TextEmbeddingProvider> _logger;
 
-    private const int DEFAULT_DIMENSION = 1536;
+    private const int DEFAULT_DIMENSION = 3072;
     protected string _model;
 
     public virtual string Provider => "azure-openai";

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs

@@ -23,11 +23,11 @@ public class KnowledgeRetrievalFn : IFunctionCallback
         var vector = await embedding.GetVectorAsync(args.Question);
         var vectorDb = _services.GetServices<IVectorDb>().FirstOrDefault(x => x.Name == _settings.VectorDb);
         var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
-        var knowledges = await vectorDb.Search(collectionName, vector, new List<string> { KnowledgePayloadName.Answer });
+        var knowledges = await vectorDb.Search(collectionName, vector, new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer });
 
         if (!knowledges.IsNullOrEmpty())
         {
-            var answers = knowledges.Select(x => x.Data[KnowledgePayloadName.Answer]).ToList();
+            var answers = knowledges.Select(x => $"Question: {x.Data[KnowledgePayloadName.Text]}\r\nAnswer: {x.Data[KnowledgePayloadName.Answer]}").ToList();
             message.Content = string.Join("\r\n\r\n=====\r\n", answers);
         }
         else

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs

@@ -8,8 +8,8 @@ public class TextEmbeddingProvider : ITextEmbedding
     protected readonly IServiceProvider _services;
     protected readonly ILogger<TextEmbeddingProvider> _logger;
 
-    private const int DEFAULT_DIMENSION = 1536;
+    private const int DEFAULT_DIMENSION = 3072;
-    protected string _model = "text-embedding-3-small";
+    protected string _model = "text-embedding-3-large";
 
     public virtual string Provider => "openai";
 

src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs

@@ -1,8 +1,14 @@
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Functions;
 using BotSharp.Abstraction.Templating;
-using BotSharp.Plugin.Planner.TwoStaging.Models;
 using System.Threading.Tasks;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Abstraction.MLTasks;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Planner.TwoStaging.Models;
+using BotSharp.Abstraction.Knowledges;
+using Microsoft.Extensions.Logging;
+using BotSharp.Abstraction.Knowledges.Models;
 
 namespace BotSharp.Plugin.Planner.Functions;
 
@@ -10,32 +16,149 @@ public class PrimaryStagePlanFn : IFunctionCallback
 {
     public string Name => "plan_primary_stage";
     private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    private object aiAssistant;
 
-    public PrimaryStagePlanFn(IServiceProvider services)
+    public PrimaryStagePlanFn(IServiceProvider services, ILogger<PrimaryStagePlanFn> logger)
     {
         _services = services;
+        _logger = logger;
     }
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
+        //debug
+        var state = _services.GetRequiredService<IConversationStateService>();
+        state.SetState("max_tokens", "4096");
         var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
+        
+        //get knowledge from vectordb
+        var fn = _services.GetRequiredService<IRoutingService>();
 
-        if (!task.HasKnowledgeReference)
+        var msg = new ExtractedKnowledge
         {
-            message.Content = "Search knowledge base for the solution instructions";
+            Question = task.Question,
-            return false;
+        };
-        }
+        var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements)
+        {
+            FunctionArgs = JsonSerializer.Serialize(msg),
+            KnowledgeConfidence = 0.1f,
+            Content = ""
+        };
+        await fn.InvokeFunction("knowledge_retrieval", retrievalMessage);
+        message.Content = retrievalMessage.Content;
 
         var agentService = _services.GetRequiredService<IAgentService>();
-        var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
+        var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
-        var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.1st.plan").Content;
+
-        var render = _services.GetRequiredService<ITemplateRender>();
+        //send knowledge to AI to refine and summarize the primary planning
-        render.Render(template, new Dictionary<string, object>
+        var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message);
+        var plannerAgent = new Agent
         {
-            { "relevant_knowledges", message.Content }
+            Id = "",
-        });
+            Name = "planning_1st",
-        //message.Content = task.Requirements;
+            Instruction = firstPlanningPrompt,
-        //message.Content += "\r\n\r\n=====\r\nGet the first primary step, plan the secondary steps.";
+            TemplateDict = new Dictionary<string, object>(),
+            LlmConfig = currentAgent.LlmConfig
+        };
+        var response = await GetAIResponse(plannerAgent);
+        message.Content = response.Content; 
+
+        await fn.InvokeFunction("plan_secondary_stage", message);
+        var items = message.Content.JsonArrayContent<SecondStagePlan>();
+
+        //get all the related tables
+        List<string> allTables = new List<string>();
+        foreach (var item in items)
+        {
+            allTables.AddRange(item.Tables);
+        }
+        message.Data = allTables.Distinct().ToList();
+
+        //get table DDL and stores in content
+        var msg2 = RoleDialogModel.From(message);
+        await fn.InvokeFunction("get_table_definition", msg2);
+        message.SecondaryContent = msg2.Content;
+
+        //summarize and generate query
+        var summaryPlanningPrompt = await GetPlanSummaryPrompt(task, message);
+        _logger.LogInformation(summaryPlanningPrompt);
+        plannerAgent = new Agent
+        {
+            Id = "",
+            Name = "planner_summary",
+            Instruction = summaryPlanningPrompt,
+            TemplateDict = new Dictionary<string, object>(),
+            LlmConfig = currentAgent.LlmConfig
+        };
+        var response_summary = await GetAIResponse(plannerAgent);
+        _logger.LogInformation(response_summary.Content);
+
+        message.Content = response_summary.Content;
+        message.StopCompletion = true;
         return true;
     }
+    private async Task<string> GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.1st.plan").Content;
+        var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
+        {
+            Parameters = new JsonDocument[] { JsonDocument.Parse("{}") },
+            Results = new string[] { "" }
+        });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "task_description", task.Requirements },
+            { "relevant_knowledges", new[]{ message.Content } },
+            { "response_format", responseFormat }
+        });
+    }
+    private async Task<string> GetPlanSummaryPrompt(PrimaryRequirementRequest task, RoleDialogModel message)
+    {
+        // save to knowledge base
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.summarize").Content;
+        var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
+        {
+            Parameters = new JsonDocument[] { JsonDocument.Parse("{}") },
+            Results = new string[] { "" }
+        });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "table_structure", message.SecondaryContent }, ////check
+            { "task_description", task.Requirements},
+            { "relevant_knowledges", message.Content },
+            { "response_format", responseFormat }
+        });
+    }
+    private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var wholeDialogs = conv.GetDialogHistory();
+        //add "test" to wholeDialogs' last element
+        if(plannerAgent.Name == "planner_summary")
+        {
+            //add "test" to wholeDialogs' last element in a new paragraph
+            wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;";
+            wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs";
+        }
+        if (plannerAgent.Name == "planning_1st")
+        {
+            //add "test" to wholeDialogs' last element in a new paragraph
+            wholeDialogs.Last().Content += "\n\nYou must analyze the table description to infer the table relations.";
+        }
+
+        var completion = CompletionProvider.GetChatCompletion(_services, 
+            provider: plannerAgent.LlmConfig.Provider, 
+            model: plannerAgent.LlmConfig.Model);
+
+        return await completion.GetChatCompletions(plannerAgent, wholeDialogs);
+    }
 }

src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs

@@ -1,19 +1,104 @@
+using Azure;
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Functions;
+using BotSharp.Abstraction.Knowledges.Models;
+using BotSharp.Abstraction.MLTasks;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Abstraction.Templating;
+using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Planner.TwoStaging.Models;
+using NetTopologySuite.Index.HPRtree;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
 
 namespace BotSharp.Plugin.Planner.Functions;
 
 public class SecondaryStagePlanFn : IFunctionCallback
 {
     public string Name => "plan_secondary_stage";
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
 
+    public SecondaryStagePlanFn(IServiceProvider services, ILogger<SecondaryStagePlanFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
     public async Task<bool> Execute(RoleDialogModel message)
     {
-        var task = JsonSerializer.Deserialize<SecondaryBreakdownTask>(message.FunctionArgs);
+        var fn = _services.GetRequiredService<IRoutingService>();
-        message.Content = task.SolutionQuestion;
+        var msg_secondary = RoleDialogModel.From(message);
-        message.Content += $"\r\n\r\n=====\r\nUse tool of `knowledge_retrieval` for expert instructions.";
+        var task_primary = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
+        msg_secondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask
+        {
+            TaskDescription = task_primary.Requirements
+        });
+        var task_secondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msg_secondary.FunctionArgs);
+        var items = msg_secondary.Content.JsonArrayContent<FirstStagePlan>();
+
+        msg_secondary.KnowledgeConfidence = 0.5f; 
+        foreach (var item in items)
+        {
+            if (item.NeedAdditionalInformation)
+            {
+                msg_secondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge
+                {
+                    Question = item.Task
+                });
+                await fn.InvokeFunction("knowledge_retrieval", msg_secondary);
+                message.Content += msg_secondary.Content;
+            }
+        }
+        // load agent
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
+
+        var secondPlanningPrompt = await GetSecondStagePlanPrompt(task_secondary, message);
+        _logger.LogInformation(secondPlanningPrompt);
+
+        var plannerAgent = new Agent
+        {
+            Id = "",
+            Name = "test",
+            Instruction = secondPlanningPrompt,
+            TemplateDict = new Dictionary<string, object>(),
+            LlmConfig = currentAgent.LlmConfig
+        };
+
+        var response = await GetAIResponse(plannerAgent);
+        message.Content = response.Content;
+        _logger.LogInformation(response.Content);
         return true;
     }
+    private async Task<string> GetSecondStagePlanPrompt(SecondaryBreakdownTask task, RoleDialogModel message)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.2nd.plan").Content;
+        var responseFormat = JsonSerializer.Serialize(new SecondStagePlan
+        {
+            Tool = "tool name if task solution provided", 
+            Parameters = new JsonDocument[] { JsonDocument.Parse("{}") },
+            Results = new string[] { "" }
+        });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "task_description", task.TaskDescription },
+            { "primary_plan", new[]{ message.Content } },
+            { "response_format",  responseFormat }
+        });
+    }
+    private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var wholeDialogs = conv.GetDialogHistory();
+
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: plannerAgent.LlmConfig.Provider,
+            model: plannerAgent.LlmConfig.Model);
+
+        return await completion.GetChatCompletions(plannerAgent, wholeDialogs);
+    }
 }

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs

@@ -11,8 +11,8 @@ public class FirstStagePlan
     [JsonPropertyName("step")]
     public int Step { get; set; } = -1;
 
-    [JsonPropertyName("contain_multiple_steps")]
+    [JsonPropertyName("need_additional_information")]
-    public bool ContainMultipleSteps { get; set; } = false;
+    public bool NeedAdditionalInformation { get; set; } = false;
 
     [JsonPropertyName("related_tables")]
     public string[] Tables { get; set; } = [];

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/PrimaryRequirementRequest.cs

@@ -7,4 +7,7 @@ public class PrimaryRequirementRequest
 
     [JsonPropertyName("has_knowledge_reference")]
     public bool HasKnowledgeReference { get; set; }
+
+    [JsonPropertyName("question")]
+    public string Question { get; set; } = null!;
 }

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.Planner.TwoStaging.Models;
+
+public class SecondStagePlan
+{
+    [JsonPropertyName("related_tables")]
+    public string[] Tables { get; set; } = new string[0];
+
+    [JsonPropertyName("description")]
+    public string Description { get; set; } = "";
+
+    [JsonPropertyName("tool_name")]
+    public string Tool { get; set; } = "";
+
+    [JsonPropertyName("input_args")]
+    public JsonDocument[] Parameters { get; set; } = new JsonDocument[0];
+
+    [JsonPropertyName("output_results")]
+    public string[] Results { get; set; } = new string[0];
+}

src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/plan_primary_stage.json

@@ -11,8 +11,12 @@
       "has_knowledge_reference": {
         "type": "boolean",
         "description": "If there is knowledge retrieved from memory"
+      },
+      "question": {
+        "type": "string",
+        "description": "Question convert from requirement and reference tables for knowledge search. The question should contain all the detailed information in the requirement"
       }
     },
-    "required": [ "requirement_detail", "has_knowledge_reference" ]
+    "required": [ "requirement_detail", "has_knowledge_reference", "question" ]
   }
 }

src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/plan_secondary_stage.json

@@ -1,18 +1,18 @@
-{
+//{
-  "name": "plan_secondary_stage",
+//  "name": "plan_secondary_stage",
-  "description": "Based on the main tasks of the first phase, plan the implementation steps of the second phase.",
+//  "description": "Based on the main tasks of the first phase, plan the implementation steps of the second phase.",
-  "parameters": {
+//  "parameters": {
-    "type": "object",
+//    "type": "object",
-    "properties": {
+//    "properties": {
-      "task_description": {
+//      "task_description": {
-        "type": "string",
+//        "type": "string",
-        "description": "task description from primary steps"
+//        "description": "task description from primary steps"
-      },
+//      },
-      "solution_search_question": {
+//      "solution_search_question": {
-        "type": "string",
+//        "type": "string",
-        "description": "Provide solution query text"
+//        "description": "Provide solution query text"
-      }
+//      }
-    },
+//    },
-    "required": [ "task_description", "solution_search_question" ]
+//    "required": [ "task_description", "solution_search_question" ]
-  }
+//  }
-}
+//}

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs

@@ -0,0 +1,116 @@
+using BotSharp.Abstraction.Routing;
+using BotSharp.Core.Infrastructures;
+using MySqlConnector;
+using static Dapper.SqlMapper;
+using BotSharp.Abstraction.Agents.Enums;
+
+
+namespace BotSharp.Plugin.Planner.Functions;
+public class AddDatabaseKnowledgeFn : IFunctionCallback
+{
+    public string Name => "add_database_knowledge";
+    private readonly IServiceProvider _services;
+    private object aiAssistant;
+
+    public AddDatabaseKnowledgeFn(IServiceProvider services)
+    {
+        _services = services;
+    }
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var sqlDriver = _services.GetRequiredService<SqlDriverService>();
+        var fn = _services.GetRequiredService<IRoutingService>();
+        var settings = _services.GetRequiredService<SqlDriverSetting>();
+        using var connection = new MySqlConnection(settings.MySqlConnectionString);
+        var dictionary = new Dictionary<string, object>();
+
+        List<string> allTables = new List<string>();
+        //var sql = $"select table_name from information_schema.tables where table_schema='GSMPMetaData' and (table_name like 'affiliate_%' or table_name like 'client_%' or table_name like 'data_%' or table_name like 'sms_%') limit 10";
+        var sql = $"select * from GSMPMetaData.joanna_knowledgebase where id>1057;";
+        //var sql = $"select table_name from information_schema.tables where table_schema='GSMPMetaData' and table_name like 'data_%'";
+        //var sql = $"select table_name from information_schema.tables where table_schema='GSMPMetaData' and (table_name like 'client_%Reactive%' or table_name like 'data_%' or table_name like 'sms_%');";
+        var result = connection.Query(sql: sql,dictionary);
+        //var result2 = new string[] { "client_ServiceCategory", "client_ServiceCode", "client_ServiceCodeNTE", "client_ServiceType" };
+        foreach (var item in result)
+        {
+            allTables.Add(item.TABLE_NAME);
+        }
+        message.Data = allTables.Distinct().ToList();
+
+        var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
+        var note = "";
+        foreach (var item in allTables)
+        {
+            message.Data = new List<string> { item };
+
+            await fn.InvokeFunction("get_table_definition", message);
+            var PlanningPrompt = await GetPrompt(message);
+            var plannerAgent = new Agent
+            {
+                Id = "",
+                Name = "database_knowledge",
+                Instruction = PlanningPrompt,
+                TemplateDict = new Dictionary<string, object>(),
+                LlmConfig = currentAgent.LlmConfig
+            };
+            var response = await GetAIResponse(plannerAgent);
+            try
+            {
+                var knowledge = response.Content.JsonArrayContent<ExtractedKnowledge>();
+                foreach (var k in knowledge)
+                {
+                    try
+                    {
+                        message.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge
+                        {
+                            Question = k.Question,
+                            Answer = k.Answer
+                        });
+                        await fn.InvokeFunction("memorize_knowledge", message);
+                        message.SecondaryContent += $"Table: {item}, Question:{k.Question}, {message.Content} \r\n";
+                    }
+                    catch (Exception e)
+                    {
+                        note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}";
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}";
+            }
+        }
+        //message.Data = allTables.Distinct().ToList();
+        return true;
+    }
+    private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var wholeDialogs = conv.GetDialogHistory();
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: plannerAgent.LlmConfig.Provider,
+            model: plannerAgent.LlmConfig.Model);
+        //wholeDialogs.Last().Content += "\n\n=========\n\nthe summarized question/answer should:\n1. help user to identify the location of tables to find further information\n2. identify the table structure and data relationship based on the task description\n3. summarize all the table to table relationship  information based on the FOREIGN KEY, and include both table in the answer";
+
+        return await completion.GetChatCompletions(plannerAgent, wholeDialogs);
+    }
+    private async Task<string> GetPrompt(RoleDialogModel message)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var template = aiAssistant.Templates.First(x => x.Name == "database_knowledge").Content;
+        //var responseFormat = JsonSerializer.Serialize(new JsonDocument[] { JsonDocument.Parse("{}") });
+        var responseFormat = JsonSerializer.Serialize(new ExtractedKnowledge
+        {
+            Question = "question",
+            Answer = "answer"
+        });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "table_structure", message.Content }
+        });
+    }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs

@@ -0,0 +1,53 @@
+using BotSharp.Plugin.SqlDriver.Models;
+using Microsoft.EntityFrameworkCore.Metadata.Internal;
+using MySqlConnector;
+using static Dapper.SqlMapper;
+
+namespace BotSharp.Plugin.SqlDriver.Functions;
+
+public class GetTableDefinitionFn : IFunctionCallback
+{
+    public string Name => "get_table_definition";
+    private readonly IServiceProvider _services;
+
+    public GetTableDefinitionFn(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        // get agent service
+        var agentService = _services.GetRequiredService<IAgentService>();
+
+        // var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
+        var sqlDriver = _services.GetRequiredService<SqlDriverService>();
+
+        //get table DDL from database
+        var settings = _services.GetRequiredService<SqlDriverSetting>();
+        using var connection = new MySqlConnection(settings.MySqlConnectionString);
+        var dictionary = new Dictionary<string, object>();
+
+        var table_ddl = "";
+        foreach (var p in (List<string>)message.Data)
+        {
+            dictionary["@" + "table_name"] = p;
+            var escapedTableName = MySqlHelper.EscapeString(p);
+            dictionary["table_name"] = escapedTableName;
+            // can you replace this with a parameterized query?
+            var sql = $"select * from information_schema.tables where table_name ='{dictionary["table_name"]}'";
+
+            var result = connection.QueryFirstOrDefault(sql: sql, dictionary);
+            if (result != null)
+            {
+                sql = $"SHOW CREATE TABLE `{dictionary["table_name"]}`";
+                result = connection.QueryFirstOrDefault(sql: sql, dictionary);
+                table_ddl += "\r\n" + result;
+            }
+            
+        }
+        message.Content = table_ddl;
+
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/get_table_definition.json

@@ -0,0 +1,24 @@
+{
+  "name": "get_table_definition",
+  "description": "Get the DDL, including data structure, data field and relationship for table",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "table": {
+        "type": "string",
+        "description": "table need to check"
+      },
+      "return_field": {
+        "type": "object",
+        "description": "the name and alias for the return field",
+        "properties": {
+          "ddl": {
+            "type": "string",
+            "description": "DDL for the table"
+          }
+        }
+      }
+    },
+    "required": [ "table" ]
+  }
+}