refine planner

This commit is contained in:
Jicheng Lu 2024-08-30 16:12:20 -05:00
parent 6aece1d73c
commit 4102f6fa1f
11 changed files with 123 additions and 103 deletions

View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Routing.Enums;
using BotSharp.Abstraction.Routing.Planning;
using BotSharp.Core.Routing.Planning;

View file

@ -1,8 +1,10 @@
You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name.
the summarized question/answer should:
You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname, data type and comments, generate the related knowledge for DBA and BA. When users ask the question, they don't know the table name.
The summarized question/answer should:
1. help user to identify the location of tables to find further information
2. identify the table structure and data relationship based on the task description
3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer
3. summarize all the tables to table relationship information based on the FOREIGN KEY, and include both tables in the answer
Go through all the columns and generate multiple question & answer pairs.
The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n.

View file

@ -25,6 +25,6 @@ public class LlmProviderController : ControllerBase
public IEnumerable<LlmModelSetting> GetLlmProviderModels([FromRoute] string provider)
{
var list = _llmProvider.GetProviderModels(provider);
return list.Where(x => !x.ImageGeneration);
return list.Where(x => x.Type == LlmModelType.Chat);
}
}

View file

@ -9,7 +9,7 @@ public class TextEmbeddingProvider : ITextEmbedding
protected readonly ILogger<TextEmbeddingProvider> _logger;
private const int DEFAULT_DIMENSION = 1536;
protected string _model = "text-embedding-3-large";
protected string _model = "text-embedding-3-small";
protected int _dimension = DEFAULT_DIMENSION;
public virtual string Provider => "openai";

View file

@ -13,9 +13,9 @@ namespace BotSharp.Plugin.Planner.Functions;
public class PrimaryStagePlanFn : IFunctionCallback
{
public string Name => "plan_primary_stage";
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private object aiAssistant;
private readonly ILogger<PrimaryStagePlanFn> _logger;
public PrimaryStagePlanFn(IServiceProvider services, ILogger<PrimaryStagePlanFn> logger)
{
@ -25,31 +25,33 @@ public class PrimaryStagePlanFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
//debug
// Debug
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("max_tokens", "4096");
var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
//get knowledge from vectordb
// Get knowledge from vectordb
var fn = _services.GetRequiredService<IRoutingService>();
var msg = new ExtractedKnowledge
{
Question = task.Question,
};
var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements)
{
FunctionArgs = JsonSerializer.Serialize(msg),
KnowledgeConfidence = 0.1f,
Content = ""
Content = string.Empty
};
await fn.InvokeFunction("knowledge_retrieval", retrievalMessage);
message.Content = retrievalMessage.Content;
var agentService = _services.GetRequiredService<IAgentService>();
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
//send knowledge to AI to refine and summarize the primary planning
// Send knowledge to AI to refine and summarize the primary planning
var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message);
var plannerAgent = new Agent
{
@ -59,7 +61,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
TemplateDict = new Dictionary<string, object>(),
LlmConfig = currentAgent.LlmConfig
};
var response = await GetAIResponse(plannerAgent);
var response = await GetAiResponse(plannerAgent);
message.Content = response.Content;
/*await fn.InvokeFunction("plan_secondary_stage", message);
@ -96,12 +98,14 @@ public class PrimaryStagePlanFn : IFunctionCallback
message.StopCompletion = true;*/
return true;
}
private async Task<string> GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner);
var render = _services.GetRequiredService<ITemplateRender>();
var template = aiAssistant.Templates.First(x => x.Name == "two_stage.1st.plan").Content;
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner);
var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.1st.plan")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
{
Parameters = [JsonDocument.Parse("{}")],
@ -115,13 +119,15 @@ public class PrimaryStagePlanFn : IFunctionCallback
{ "response_format", responseFormat }
});
}
private async Task<string> GetPlanSummaryPrompt(PrimaryRequirementRequest task, RoleDialogModel message)
{
// save to knowledge base
var agentService = _services.GetRequiredService<IAgentService>();
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
var render = _services.GetRequiredService<ITemplateRender>();
var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.summarize").Content;
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "planner_prompt.two_stage.summarize")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
{
Parameters = [JsonDocument.Parse("{}")],
@ -136,10 +142,12 @@ public class PrimaryStagePlanFn : IFunctionCallback
{ "response_format", responseFormat }
});
}
private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
{
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
//add "test" to wholeDialogs' last element
if(plannerAgent.Name == "planner_summary")
{
@ -147,6 +155,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;";
wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs";
}
if (plannerAgent.Name == "planning_1st")
{
//add "test" to wholeDialogs' last element in a new paragraph

View file

@ -1,13 +1,10 @@
using Azure;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Templating;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Planner.TwoStaging.Models;
using NetTopologySuite.Index.HPRtree;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@ -16,8 +13,9 @@ namespace BotSharp.Plugin.Planner.Functions;
public class SecondaryStagePlanFn : IFunctionCallback
{
public string Name => "plan_secondary_stage";
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private readonly ILogger<SecondaryStagePlanFn> _logger;
public SecondaryStagePlanFn(IServiceProvider services, ILogger<SecondaryStagePlanFn> logger)
{
@ -28,45 +26,49 @@ public class SecondaryStagePlanFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var fn = _services.GetRequiredService<IRoutingService>();
var msg_secondary = RoleDialogModel.From(message);
var task_primary = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
msg_secondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask
{
TaskDescription = task_primary.Requirements
});
var task_secondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msg_secondary.FunctionArgs);
var items = msg_secondary.Content.JsonArrayContent<FirstStagePlan>();
msg_secondary.KnowledgeConfidence = 0.5f;
var msgSecondary = RoleDialogModel.From(message);
var taskPrimary = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask
{
TaskDescription = taskPrimary.Requirements
});
var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
var items = msgSecondary.Content.JsonArrayContent<FirstStagePlan>();
msgSecondary.KnowledgeConfidence = 0.5f;
foreach (var item in items)
{
if (item.NeedAdditionalInformation)
{
msg_secondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge
msgSecondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge
{
Question = item.Task
});
await fn.InvokeFunction("knowledge_retrieval", msg_secondary);
message.Content += msg_secondary.Content;
await fn.InvokeFunction("knowledge_retrieval", msgSecondary);
message.Content += msgSecondary.Content;
}
}
// load agent
var agentService = _services.GetRequiredService<IAgentService>();
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
var secondPlanningPrompt = await GetSecondStagePlanPrompt(task_secondary, message);
var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary, message);
_logger.LogInformation(secondPlanningPrompt);
var plannerAgent = new Agent
{
Id = "",
Id = string.Empty,
Name = "test",
Instruction = secondPlanningPrompt,
TemplateDict = new Dictionary<string, object>(),
LlmConfig = currentAgent.LlmConfig
};
var response = await GetAIResponse(plannerAgent);
var response = await GetAiResponse(plannerAgent);
message.Content = response.Content;
_logger.LogInformation(response.Content);
return true;
@ -74,9 +76,10 @@ public class SecondaryStagePlanFn : IFunctionCallback
private async Task<string> GetSecondStagePlanPrompt(SecondaryBreakdownTask task, RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var planner = await agentService.GetAgent(message.CurrentAgentId);
var render = _services.GetRequiredService<ITemplateRender>();
var template = planner.Templates.First(x => x.Name == "two_stage.2nd.plan").Content;
var planner = await agentService.GetAgent(message.CurrentAgentId);
var template = planner.Templates.FirstOrDefault(x => x.Name == "two_stage.2nd.plan")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new SecondStagePlan
{
Tool = "tool name if task solution provided",
@ -91,7 +94,7 @@ public class SecondaryStagePlanFn : IFunctionCallback
{ "response_format", responseFormat }
});
}
private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
{
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();

View file

@ -32,6 +32,7 @@ public class SummaryPlanFn : IFunctionCallback
// summarize and generate query
var summaryPlanningPrompt = await GetPlanSummaryPrompt(task, message);
_logger.LogInformation(summaryPlanningPrompt);
var plannerAgent = new Agent
{
Id = BuiltInAgentId.Planner,
@ -39,7 +40,7 @@ public class SummaryPlanFn : IFunctionCallback
Instruction = summaryPlanningPrompt,
TemplateDict = new Dictionary<string, object>()
};
var response_summary = await GetAIResponse(plannerAgent);
var response_summary = await GetAiResponse(plannerAgent);
message.Content = response_summary.Content;
message.StopCompletion = true;
@ -51,9 +52,10 @@ public class SummaryPlanFn : IFunctionCallback
{
// save to knowledge base
var agentService = _services.GetRequiredService<IAgentService>();
var aiAssistant = await agentService.GetAgent(message.CurrentAgentId);
var render = _services.GetRequiredService<ITemplateRender>();
var template = aiAssistant.Templates.First(x => x.Name == "two_stage.summarize").Content;
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner);
var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
{
Parameters = [JsonDocument.Parse("{}")],
@ -68,17 +70,19 @@ public class SummaryPlanFn : IFunctionCallback
{ "response_format", responseFormat }
});
}
private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
{
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
//add "test" to wholeDialogs' last element
if(plannerAgent.Name == "planner_summary")
if (plannerAgent.Name == "planner_summary")
{
//add "test" to wholeDialogs' last element in a new paragraph
wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;";
wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs";
}
if (plannerAgent.Name == "planning_1st")
{
//add "test" to wholeDialogs' last element in a new paragraph

View file

@ -3,7 +3,7 @@ You are a Task Planner. you will breakdown user business requirements into excut
Thinking process:
1. Reference to "Task Knowledge" if there is relevant knowledge;
2. Breakdown task into subtasks.
- The subtask should contains all needed parameters for subsequent steps.
- The subtask should contain all needed parameters for subsequent steps.
- If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true.
- If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps.
- If the solution mentioned "related solutions" is needed, set the need_additional_information to true.
@ -22,9 +22,7 @@ Task Knowledge:
{{ k }}
{% endfor %}
{%- endif %}
=====
Task description:
{{ task_description }}
{{ task_description }}

View file

@ -1,5 +1,5 @@
Reference to "Primary Planning" and the additional knowledge included. Breakdown task into multiple steps.
* the step should contains all needed parameters.
* The step should contains all needed parameters.
* The parameters can be extracted from the original task.
* You need to list all the steps in detail. Finding relationships should also be a step.
* When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly.

View file

@ -3,58 +3,69 @@ using BotSharp.Core.Infrastructures;
using MySqlConnector;
using static Dapper.SqlMapper;
using BotSharp.Abstraction.Agents.Enums;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.Planner.Functions;
public class AddDatabaseKnowledgeFn : IFunctionCallback
{
public string Name => "add_database_knowledge";
private readonly IServiceProvider _services;
private object aiAssistant;
public AddDatabaseKnowledgeFn(IServiceProvider services)
private readonly IServiceProvider _services;
private readonly ILogger<AddDatabaseKnowledgeFn> _logger;
public AddDatabaseKnowledgeFn(
IServiceProvider services,
ILogger<AddDatabaseKnowledgeFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var sqlDriver = _services.GetRequiredService<SqlDriverService>();
var fn = _services.GetRequiredService<IRoutingService>();
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlConnectionString);
var dictionary = new Dictionary<string, object>();
List<string> allTables = new List<string>();
var allTables = new HashSet<string>();
using var connection = new MySqlConnection(settings.MySqlConnectionString);
var sql = $"select table_name from information_schema.tables;";
var result = connection.Query(sql: sql,dictionary);
foreach (var item in result)
var results = connection.Query(sql, new Dictionary<string, object>());
foreach (var item in results)
{
if (item == null) continue;
allTables.Add(item.TABLE_NAME);
}
message.Data = allTables.Distinct().ToList();
message.Data = allTables.ToList();
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
var note = "";
foreach (var item in allTables)
var errorNote = string.Empty;
foreach (var table in allTables)
{
message.Data = new List<string> { item };
message.Data = new List<string> { table };
await fn.InvokeFunction("get_table_definition", message);
var PlanningPrompt = await GetPrompt(message);
var planningPrompt = await GetPrompt(message);
var plannerAgent = new Agent
{
Id = "",
Name = "database_knowledge",
Instruction = PlanningPrompt,
TemplateDict = new Dictionary<string, object>(),
Id = string.Empty,
Name = "Database Knowledge",
Instruction = planningPrompt,
LlmConfig = currentAgent.LlmConfig
};
var response = await GetAIResponse(plannerAgent);
try
{
var knowledge = response.Content.JsonArrayContent<ExtractedKnowledge>();
foreach (var k in knowledge)
var response = await GetAiResponse(plannerAgent);
var knowledges = response.Content.JsonArrayContent<ExtractedKnowledge>();
foreach (var k in knowledges)
{
try
{
@ -64,42 +75,44 @@ public class AddDatabaseKnowledgeFn : IFunctionCallback
Answer = k.Answer
});
await fn.InvokeFunction("memorize_knowledge", message);
message.SecondaryContent += $"Table: {item}, Question:{k.Question}, {message.Content} \r\n";
message.SecondaryContent += $"Table: {table}, Question: {k.Question}, {message.Content}\r\n";
}
catch (Exception e)
{
note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}";
var note = $"Error processing table {table}: {e.Message}\r\n{e.InnerException}";
errorNote += note;
_logger.LogWarning(note);
}
}
}
catch (Exception e)
{
note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}";
errorNote += $"Error processing table {table}: {e.Message}\r\n{e.InnerException}\r\n";
_logger.LogWarning(errorNote);
}
}
return true;
}
private async Task<RoleDialogModel> GetAIResponse(Agent plannerAgent)
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
{
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
var completion = CompletionProvider.GetChatCompletion(_services,
var completion = CompletionProvider.GetChatCompletion(_services,
provider: plannerAgent.LlmConfig.Provider,
model: plannerAgent.LlmConfig.Model);
return await completion.GetChatCompletions(plannerAgent, wholeDialogs);
}
private async Task<string> GetPrompt(RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
var render = _services.GetRequiredService<ITemplateRender>();
var template = aiAssistant.Templates.First(x => x.Name == "database_knowledge").Content;
var responseFormat = JsonSerializer.Serialize(new ExtractedKnowledge
{
Question = "question",
Answer = "answer"
});
var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty;
return render.Render(template, new Dictionary<string, object>
{

View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.SqlDriver.Models;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using MySqlConnector;
using static Dapper.SqlMapper;
@ -17,37 +15,31 @@ public class GetTableDefinitionFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
// get agent service
var agentService = _services.GetRequiredService<IAgentService>();
// var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
var sqlDriver = _services.GetRequiredService<SqlDriverService>();
//get table DDL from database
var settings = _services.GetRequiredService<SqlDriverSetting>();
// Get table DDL from database
using var connection = new MySqlConnection(settings.MySqlConnectionString);
var dictionary = new Dictionary<string, object>();
var tableDdls = new List<string>();
var table_ddl = "";
foreach (var p in (List<string>)message.Data)
{
dictionary["@" + "table_name"] = p;
var escapedTableName = MySqlHelper.EscapeString(p);
dictionary["@" + "table_name"] = p;
dictionary["table_name"] = escapedTableName;
// can you replace this with a parameterized query?
var sql = $"select * from information_schema.tables where table_name ='{dictionary["table_name"]}'";
var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'";
var result = connection.QueryFirstOrDefault(sql: sql, dictionary);
if (result != null)
{
sql = $"SHOW CREATE TABLE `{dictionary["table_name"]}`";
result = connection.QueryFirstOrDefault(sql: sql, dictionary);
table_ddl += "\r\n" + result;
}
}
message.Content = table_ddl;
if (result == null) continue;
sql = $"SHOW CREATE TABLE `{escapedTableName}`";
result = connection.QueryFirstOrDefault(sql: sql, dictionary);
tableDdls.Add(result);
}
message.Content = string.Join("\r\n", tableDdls);
return true;
}
}