SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #836 from Qtoss-AI/master

Browse source 
clean code of sql planner
This commit is contained in:
Haiping 2025-01-17 22:22:46 -06:00 committed by GitHub
parent 5f0f4f7276 6f4448d66b
commit e30519c8dd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 382 additions and 223 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/HttpRequestParams.cs
View file

@ -1,7 +1,9 @@
using System.Diagnostics;
using System.Net.Http;
namespace BotSharp.Abstraction.Browsing.Models;
[DebuggerStepThrough]
public class HttpRequestParams
{
[JsonPropertyName("url")]

2
src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs
View file

@ -49,7 +49,7 @@ public interface IAuthenticationHook
/// </summary>
/// <param name="user"></param>
/// <returns></returns>
Task VerificationCodeResetPassword(User user);
Task SendVerificationCode(User user);
/// <summary>
/// Delete users

7
src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
View file

@ -18,12 +18,15 @@ public interface IUserService
Task<Token?> GetAdminToken(string authorization);
Task<Token?> GetToken(string authorization);
Task<Token> CreateTokenByUser(User user);
Task<Token> RenewToken();
Task<User> GetMyProfile();
Task<bool> VerifyUserNameExisting(string userName);
Task<bool> VerifyEmailExisting(string email);
Task<bool> VerifyPhoneExisting(string phone, string regionCode);
Task<bool> SendVerificationCodeResetPasswordNoLogin(User user);
Task<bool> SendVerificationCodeResetPasswordLogin();
Task<User> ResetVerificationCode(User user);
Task<bool> SendVerificationCodeNoLogin(User user);
Task<bool> SendVerificationCodeLogin();
Task<bool> SetUserPassword(User user);
Task<bool> ResetUserPassword(User user);
Task<bool> ModifyUserEmail(string email);
Task<bool> ModifyUserPhone(string phone, string regionCode);

1
src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs
View file

@ -20,6 +20,7 @@ public class User
public string Type { get; set; } = UserType.Client;
public string Role { get; set; } = UserRole.User;
public string? VerificationCode { get; set; }
public DateTime? VerificationCodeExpireAt { get; set; }
public bool Verified { get; set; }
public string RegionCode { get; set; } = "CN";
public string? AffiliateId { get; set; }

112
src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
View file

@ -53,14 +53,7 @@ public class UserService : IUserService
record = db.GetUserByUserName(user.UserName);
}
if (record != null && record.Verified)
{
// account is already activated
_logger.LogWarning($"User account already exists: {record.Id} {record.UserName}");
return record;
}
if (!string.IsNullOrWhiteSpace(user.Phone))
if (record == null && !string.IsNullOrWhiteSpace(user.Phone))
{
record = db.GetUserByPhone(user.Phone, regionCode: (string.IsNullOrWhiteSpace(user.RegionCode) ? "CN" : user.RegionCode));
}
@ -70,6 +63,13 @@ public class UserService : IUserService
record = db.GetUserByEmail(user.Email);
}
if (record != null && record.Verified)
{
// account is already activated
_logger.LogWarning($"User account already exists: {record.Id} {record.UserName}");
return record;
}
if (record != null)
{
hasRegisterId = record.Id;
@ -94,8 +94,13 @@ public class UserService : IUserService
//record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value;
record.Phone = Regex.Match(user.Phone, @"\d+").Value;
}
record.Salt = Guid.NewGuid().ToString("N");
record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}");
if (!string.IsNullOrWhiteSpace(user.Password))
{
record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}");
}
if (_setting.NewUserVerification)
{
@ -482,7 +487,7 @@ public class UserService : IUserService
return default;
}
if (record.VerificationCode != model.VerificationCode)
if (record.VerificationCode != model.VerificationCode || (record.VerificationCodeExpireAt != null && DateTime.UtcNow > record.VerificationCodeExpireAt))
{
return default;
}
@ -520,6 +525,16 @@ public class UserService : IUserService
return token;
}
public async Task<Token> RenewToken()
{
var newToken = GenerateJwtToken(await GetMyProfile());
var newJwt = new JwtSecurityTokenHandler().ReadJwtToken(newToken);
Token token = new Token();
token.AccessToken = newToken;
token.ExpireTime = newJwt.Payload.Exp.Value;
return token;
}
public async Task<bool> VerifyUserNameExisting(string userName)
{
if (string.IsNullOrEmpty(userName))
@ -572,17 +587,34 @@ public class UserService : IUserService
return false;
}
public async Task<bool> SendVerificationCodeResetPasswordNoLogin(User user)
public async Task<bool> SendVerificationCodeNoLogin(User user)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
User? record = await ResetVerificationCode(user);
User? record = null;
if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
if (record == null)
{
return false;
}
//send code to user Email.
var hooks = _services.GetServices<IAuthenticationHook>();
foreach (var hook in hooks)
{
await hook.SendVerificationCode(record);
}
return true;
}
public async Task<User> ResetVerificationCode(User user)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
User record = null;
if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
{
return null;
}
if (!string.IsNullOrEmpty(user.Phone))
{
record = db.GetUserByPhone(user.Phone, regionCode: user.RegionCode);
@ -595,7 +627,7 @@ public class UserService : IUserService
if (record == null)
{
return false;
return null;
}
record.VerificationCode = Nanoid.Generate(alphabet: "0123456789", size: 6);
@ -603,17 +635,10 @@ public class UserService : IUserService
//update current verification code.
db.UpdateUserVerificationCode(record.Id, record.VerificationCode);
//send code to user Email.
var hooks = _services.GetServices<IAuthenticationHook>();
foreach (var hook in hooks)
{
await hook.VerificationCodeResetPassword(record);
}
return true;
return record;
}
public async Task<bool> SendVerificationCodeResetPasswordLogin()
public async Task<bool> SendVerificationCodeLogin()
{
var db = _services.GetRequiredService<IBotSharpRepository>();
@ -638,7 +663,7 @@ public class UserService : IUserService
var hooks = _services.GetServices<IAuthenticationHook>();
foreach (var hook in hooks)
{
await hook.VerificationCodeResetPassword(record);
await hook.SendVerificationCode(record);
}
return true;
@ -669,7 +694,40 @@ public class UserService : IUserService
return false;
}
if (user.VerificationCode != record.VerificationCode)
if (user.VerificationCode != record.VerificationCode || (record.VerificationCodeExpireAt != null && DateTime.UtcNow > record.VerificationCodeExpireAt))
{
return false;
}
var newPassword = Utilities.HashTextMd5($"{user.Password}{record.Salt}");
db.UpdateUserPassword(record.Id, newPassword);
return true;
}
public async Task<bool> SetUserPassword(User user)
{
if (!string.IsNullOrEmpty(user.Id) && !string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
{
return false;
}
var db = _services.GetRequiredService<IBotSharpRepository>();
User? record = null;
if (!string.IsNullOrEmpty(user.Id))
{
record = db.GetUserById(user.Id);
}
else if (!string.IsNullOrEmpty(user.Phone))
{
record = db.GetUserByPhone(user.Phone, regionCode: (string.IsNullOrWhiteSpace(user.RegionCode) ? "CN" : user.RegionCode));
}
else if (!string.IsNullOrEmpty(user.Email))
{
record = db.GetUserByEmail(user.Email);
}
if (record == null)
{
return false;
}

10
src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
View file

@ -11,7 +11,6 @@ using Microsoft.Net.Http.Headers;
using Microsoft.OpenApi.Models;
using Microsoft.IdentityModel.JsonWebTokens;
using BotSharp.OpenAPI.BackgroundServices;
using BotSharp.OpenAPI.Filters;
namespace BotSharp.OpenAPI;
@ -33,15 +32,6 @@ public static class BotSharpOpenApiExtensions
services.AddScoped<IUserIdentity, UserIdentity>();
services.AddHostedService<ConversationTimeoutService>();
var enableSingleLogin = bool.Parse(config["Jwt:EnableSingleLogin"] ?? "false");
if (enableSingleLogin)
{
services.AddMvc(options =>
{
options.Filters.Add<UserSingleLoginFilter>();
});
}
// Add bearer authentication
var schema = "MIXED_SCHEME";
var builder = services.AddAuthentication(options =>

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -132,13 +132,13 @@ public class UserController : ControllerBase
[HttpPost("/user/verifycode-out")]
public async Task<bool> SendVerificationCodeResetPassword([FromBody] UserCreationModel user)
{
return await _userService.SendVerificationCodeResetPasswordNoLogin(user.ToUser());
return await _userService.SendVerificationCodeNoLogin(user.ToUser());
}
[HttpPost("/user/verifycode-in")]
public async Task<bool> SendVerificationCodeResetPasswordLogined()
{
return await _userService.SendVerificationCodeResetPasswordLogin();
return await _userService.SendVerificationCodeLogin();
}
[AllowAnonymous]

78
src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs
View file

@ -1,78 +0,0 @@
using BotSharp.Abstraction.Users.Settings;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.AspNetCore.Mvc.Filters;
using Microsoft.Net.Http.Headers;
using System.IdentityModel.Tokens.Jwt;
namespace BotSharp.OpenAPI.Filters;
public class UserSingleLoginFilter : IAuthorizationFilter
{
private readonly IUserService _userService;
private readonly IServiceProvider _services;
public UserSingleLoginFilter(IUserService userService, IServiceProvider services)
{
_userService = userService;
_services = services;
}
public void OnAuthorization(AuthorizationFilterContext context)
{
var isAllowAnonymous = context.ActionDescriptor.EndpointMetadata
.Any(em => em.GetType() == typeof(AllowAnonymousAttribute));
if (isAllowAnonymous)
{
return;
}
var bearerToken = GetBearerToken(context);
if (!string.IsNullOrWhiteSpace(bearerToken))
{
var config = _services.GetRequiredService<AccountSetting>();
var token = GetJwtToken(bearerToken);
if (config.AllowMultipleDeviceLoginUserIds.Contains(token.Claims.First(x => x.Type == "nameid").Value))
{
return;
}
var validTo = token.ValidTo.ToLongTimeString();
var currentExpires = GetUserExpires().ToLongTimeString();
if (validTo != currentExpires)
{
Serilog.Log.Warning($"Token expired. Token expires at {validTo}, current expires at {currentExpires}");
// login confict
context.Result = new ConflictResult();
}
}
}
private string GetBearerToken(AuthorizationFilterContext context)
{
if (context.HttpContext.Request.Headers.TryGetValue(HeaderNames.Authorization, out var bearerToken)
&& !string.IsNullOrWhiteSpace(bearerToken.ToString()))
{
var tokenType = bearerToken.ToString().Split(" ").First();
if (tokenType == JwtBearerDefaults.AuthenticationScheme)
{
return bearerToken.ToString().Split(" ").Last();
}
}
return null;
}
private JwtSecurityToken GetJwtToken(string jwtToken)
{
var handler = new JwtSecurityTokenHandler();
var token = handler.ReadJwtToken(jwtToken);
return token;
}
private DateTime GetUserExpires()
{
return _userService.GetUserTokenExpires().GetAwaiter().GetResult();
}
}

2
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs
View file

@ -17,6 +17,7 @@ public class UserDocument : MongoBase
public string Type { get; set; } = UserType.Client;
public string Role { get; set; } = null!;
public string? VerificationCode { get; set; }
public DateTime? VerificationCodeExpireAt { get; set; }
public bool Verified { get; set; }
public string? RegionCode { get; set; }
public string? AffiliateId { get; set; }
@ -48,6 +49,7 @@ public class UserDocument : MongoBase
EmployeeId = EmployeeId,
IsDisabled = IsDisabled,
VerificationCode = VerificationCode,
VerificationCodeExpireAt = VerificationCodeExpireAt,
Verified = Verified,
RegionCode = RegionCode,
Permissions = Permissions,

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
View file

@ -132,6 +132,7 @@ public partial class MongoRepository
{
var filter = Builders<UserDocument>.Filter.Eq(x => x.Id, userId);
var update = Builders<UserDocument>.Update.Set(x => x.VerificationCode, verficationCode)
.Set(x => x.VerificationCodeExpireAt, DateTime.UtcNow.AddMinutes(5))
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.Users.UpdateOne(filter, update);
}

12
src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj
View file

@ -72,10 +72,10 @@
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\functions\plan_primary_stage.json">
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\functions\sql_primary_stage.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\functions\plan_secondary_stage.json">
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\functions\sql_secondary_stage.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\functions\sql_review.json">
@ -87,16 +87,16 @@
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\instructions\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\two_stage.1st.plan.liquid">
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\sql.primary.plan.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\two_stage.2nd.plan.liquid">
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\sql.secondary.plan.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\two_stage.next.liquid">
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\sql.next.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\two_stage.summarize.liquid">
<Content Include="data\agents\da7aad2c-8112-48a2-ab7b-1f87da524741\templates\sql.generation.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

6
src/Plugins/BotSharp.Plugin.Planner/PlannerPlugin.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Plugin.Planner.Sequential;
using BotSharp.Plugin.Planner.SqlGeneration;
using BotSharp.Plugin.Planner.SqlGeneration.Hooks;
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Hooks;
namespace BotSharp.Plugin.Planner;
@ -26,7 +28,7 @@ public class PlannerPlugin : IBotSharpPlugin
services.AddScoped<ITaskPlanner, SequentialPlanner>();
services.AddScoped<ITaskPlanner, TwoStageTaskPlanner>();
services.AddScoped<ITaskPlanner, SqlGenerationPlanner>();
services.AddScoped<IAgentHook, PlannerAgentHook>();
services.AddScoped<IAgentUtilityHook, PlannerUtilityHook>();
services.AddScoped<IAgentHook, SqlPlannerAgentHook>();
services.AddScoped<IAgentUtilityHook, TwoStagingPlannerUtilityHook>();
}
}

15
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Plugin.Planner.SqlGeneration;
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Models;
@ -54,12 +55,12 @@ public class SqlGenerationFn : IFunctionCallback
states.SetState("table_ddls", ddlStatements);
// Summarize and generate query
var prompt = await GetSummaryPlanPrompt(msgCopy, taskRequirement, domainKnowledge, dictionaryItems, ddlStatements, excelImportResult);
_logger.LogInformation($"Summary plan prompt:\r\n{prompt}");
var prompt = await GetSqlGenerationPrompt(msgCopy, taskRequirement, domainKnowledge, dictionaryItems, ddlStatements, excelImportResult);
_logger.LogInformation($"SQL Generation plan prompt:\r\n{prompt}");
var plannerAgent = new Agent
{
Id = PlannerAgentId.TwoStagePlanner,
Id = PlannerAgentId.SqlPlanner,
Name = Name,
Instruction = prompt,
LlmConfig = currentAgent.LlmConfig
@ -75,19 +76,19 @@ public class SqlGenerationFn : IFunctionCallback
return true;
}
private async Task<string> GetSummaryPlanPrompt(RoleDialogModel message, string taskDescription, string domainKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult)
private async Task<string> GetSqlGenerationPrompt(RoleDialogModel message, string taskDescription, string domainKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult)
{
var agentService = _services.GetRequiredService<IAgentService>();
var render = _services.GetRequiredService<ITemplateRender>();
var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
var agent = await agentService.GetAgent(PlannerAgentId.TwoStagePlanner);
var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty;
var agent = await agentService.GetAgent(PlannerAgentId.SqlPlanner);
var template = agent.Templates.FirstOrDefault(x => x.Name == "sql.generation")?.Content ?? string.Empty;
var additionalRequirements = new List<string>();
await HookEmitter.Emit<IPlanningHook>(_services, async x =>
{
var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner), message);
var requirement = await x.GetSummaryAdditionalRequirements(nameof(SqlGenerationPlanner), message);
additionalRequirements.Add(requirement);
});

105
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlPrimaryStageFn.cs Normal file
View file

@ -0,0 +1,105 @@
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.SqlGeneration.Functions;
public class SqlPrimaryStageFn : IFunctionCallback
{
public string Name => "sql_primary_stage";
public string Indication => "Currently analyzing and breaking down user requirements.";
private readonly IServiceProvider _services;
private readonly ILogger<SqlPrimaryStageFn> _logger;
public SqlPrimaryStageFn(
IServiceProvider services,
ILogger<SqlPrimaryStageFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
state.SetState("max_tokens", "4096");
var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
var searchQuestions = new List<string>(task.Questions);
searchQuestions.AddRange(task.NormQuestions);
searchQuestions = searchQuestions.Distinct().ToList();
// Get knowledge from vectordb
var hooks = _services.GetServices<IKnowledgeHook>();
var knowledges = new List<string>();
foreach (var question in searchQuestions)
{
foreach (var hook in hooks)
{
var k = await hook.GetDomainKnowledges(message, question);
knowledges.AddRange(k);
}
}
knowledges = knowledges.Distinct().ToList();
var knowledgeState = string.Join("\r\n", knowledges);
state.SetState("domain_knowledges", knowledgeState);
// Get first stage planning prompt
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
var prompt = await GetPrimaryStagePlanPrompt(message, task.Requirements, knowledges);
var plannerAgent = new Agent
{
Id = message.CurrentAgentId,
Name = Name,
Instruction = prompt,
TemplateDict = new Dictionary<string, object>(),
LlmConfig = currentAgent.LlmConfig
};
var response = await GetAiResponse(plannerAgent);
message.Content = response.Content;
var states = _services.GetRequiredService<IConversationStateService>();
states.SetState("planning_result", response.Content);
return true;
}
private async Task<string> GetPrimaryStagePlanPrompt(RoleDialogModel message, string taskDescription, List<string> domainKnowledges)
{
var agentService = _services.GetRequiredService<IAgentService>();
var render = _services.GetRequiredService<ITemplateRender>();
var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
var agent = await agentService.GetAgent(PlannerAgentId.SqlPlanner);
var template = agent.Templates.FirstOrDefault(x => x.Name == "sql.primary.plan")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { });
// Get global knowledges
var globalKnowledges = new List<string>();
foreach (var hook in knowledgeHooks)
{
var k = await hook.GetGlobalKnowledges(message);
globalKnowledges.AddRange(k);
}
return render.Render(template, new Dictionary<string, object>
{
{ "task_description", taskDescription },
{ "global_knowledges", globalKnowledges },
{ "domain_knowledges", domainKnowledges },
{ "response_format", responseFormat }
});
}
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
{
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
var completion = CompletionProvider.GetChatCompletion(_services,
provider: plannerAgent.LlmConfig.Provider,
model: plannerAgent.LlmConfig.Model);
return await completion.GetChatCompletions(plannerAgent, wholeDialogs);
}
}

6
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs
View file

@ -1,6 +1,4 @@
using BotSharp.Plugin.Planner.SqlGeneration.Models;
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.SqlGeneration.Functions;
@ -23,14 +21,16 @@ public class SqlReviewFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<SqlReviewArgs>(message.FunctionArgs);
if (!message.Content.StartsWith("```sql"))
{
message.Content = $"```sql\r\n{args.SqlStatement}\r\n```";
}
if (args != null && !args.IsSqlTemplate && args.ContainsSqlStatements)
{
await HookEmitter.Emit<IPlanningHook>(_services, async hook =>
await hook.OnSourceCodeGenerated(nameof(TwoStageTaskPlanner), message, "sql")
await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql")
);
}
return true;

106
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlSecondaryStageFn.cs Normal file
View file

@ -0,0 +1,106 @@
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.SqlGeneration.Functions;
public class SqlSecondaryStageFn : IFunctionCallback
{
public string Name => "sql_secondary_stage";
public string Indication => "Further analyzing and breaking down user sub-needs.";
private readonly IServiceProvider _services;
private readonly ILogger<SqlSecondaryStageFn> _logger;
public SqlSecondaryStageFn(
IServiceProvider services,
ILogger<SqlSecondaryStageFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
var states = _services.GetRequiredService<IConversationStateService>();
var msgSecondary = RoleDialogModel.From(message);
var collectionName = knowledgeSettings.Default.CollectionName;
var planResult = states.GetState("planning_result");
var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
// Search knowledgebase
var hooks = _services.GetServices<IKnowledgeHook>();
var knowledges = new List<string>();
foreach (var hook in hooks)
{
var k = await hook.GetDomainKnowledges(message, taskSecondary.SolutionQuestion);
knowledges.AddRange(k);
}
knowledges = knowledges.Distinct().ToList();
var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges);
var knowledgeState = states.GetState("domain_knowledges");
knowledgeState += string.Join("\r\n", knowledges);
states.SetState("domain_knowledges", knowledgeState);
// Get second stage planning prompt
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
var prompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, planResult, knowledgeResults, message);
_logger.LogInformation(prompt);
var plannerAgent = new Agent
{
Id = PlannerAgentId.SqlPlanner,
Name = Name,
Instruction = prompt,
TemplateDict = new Dictionary<string, object>(),
LlmConfig = currentAgent.LlmConfig
};
var response = await GetAiResponse(plannerAgent);
message.Content = response.Content;
_logger.LogInformation(response.Content);
states.SetState("planning_result", response.Content);
return true;
}
private async Task<string> GetSecondStagePlanPrompt(string taskDescription, string planResult, string knowledgeResults, RoleDialogModel message)
{
var agentService = _services.GetRequiredService<IAgentService>();
var render = _services.GetRequiredService<ITemplateRender>();
var agent = await agentService.GetAgent(message.CurrentAgentId);
var template = agent.Templates.FirstOrDefault(x => x.Name == "sql.secondary.plan")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new SecondStagePlan
{
Parameters = [JsonDocument.Parse("{}")],
Results = [string.Empty]
});
return render.Render(template, new Dictionary<string, object>
{
{ "task_description", taskDescription },
{ "primary_plan", planResult },
{ "additional_knowledge", knowledgeResults },
{ "response_format", responseFormat }
});
}
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
{
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
wholeDialogs.Last().Content += "\r\nOutput in JSON format.";
var completion = CompletionProvider.GetChatCompletion(_services,
provider: plannerAgent.LlmConfig.Provider,
model: plannerAgent.LlmConfig.Model);
return await completion.GetChatCompletions(plannerAgent, wholeDialogs);
}
}

10
src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerAgentHook.cs → src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Hooks/SqlPlannerAgentHook.cs
View file

@ -1,10 +1,10 @@
namespace BotSharp.Plugin.Planner.Hooks;
namespace BotSharp.Plugin.Planner.SqlGeneration.Hooks;
public class PlannerAgentHook : AgentHookBase
public class SqlPlannerAgentHook : AgentHookBase
{
public override string SelfId => PlannerAgentId.TwoStagePlanner;
public override string SelfId => PlannerAgentId.SqlPlanner;
public PlannerAgentHook(IServiceProvider services, AgentSettings settings)
public SqlPlannerAgentHook(IServiceProvider services, AgentSettings settings)
: base(services, settings)
{
}
@ -19,7 +19,7 @@ public class PlannerAgentHook : AgentHookBase
{
var k = hook.GetGlobalKnowledges(new RoleDialogModel(AgentRole.User, template)
{
CurrentAgentId = PlannerAgentId.TwoStagePlanner
CurrentAgentId = PlannerAgentId.SqlPlanner
}).Result;
Knowledges.AddRange(k);
}

13
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Models/PrimaryRequirementRequest.cs
View file

@ -1,13 +0,0 @@
namespace BotSharp.Plugin.Planner.SqlGeneration.Models;
public class PrimaryRequirementRequest
{
[JsonPropertyName("requirement_detail")]
public string Requirements { get; set; } = null!;
[JsonPropertyName("questions")]
public string[] Questions { get; set; } = [];
[JsonPropertyName("norm_questions")]
public string[] NormQuestions { get; set; } = [];
}

13
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Models/SecondaryBreakdownTask.cs
View file

@ -1,13 +0,0 @@
namespace BotSharp.Plugin.Planner.SqlGeneration.Models;
public class SecondaryBreakdownTask
{
[JsonPropertyName("task_description")]
public string TaskDescription { get; set; } = null!;
[JsonPropertyName("solution_search_question")]
public string SolutionQuestion { get; set; } = null!;
[JsonPropertyName("need_lookup_dictionary")]
public bool NeedLookupDictionary { get; set; }
}

14
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Models/FirstStagePlan.cs → src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Models/SqlPrimaryStagePlan.cs
View file

@ -1,13 +1,10 @@
namespace BotSharp.Plugin.Planner.SqlGeneration.Models;
public class FirstStagePlan
public class SqlPrimaryStagePlan
{
[JsonPropertyName("task_detail")]
public string Task { get; set; } = "";
//[JsonPropertyName("reason")]
//public string Reason { get; set; } = "";
[JsonPropertyName("step")]
public int Step { get; set; } = -1;
@ -23,15 +20,6 @@ public class FirstStagePlan
[JsonPropertyName("has_found_relevant_knowledge")]
public bool HasFoundRelevantKnowledge { get; set; } = false;
//[JsonPropertyName("related_urls")]
//public string[] Urls { get; set; } = [];
//[JsonPropertyName("input_args")]
//public JsonDocument[] Parameters { get; set; } = [];
//[JsonPropertyName("output_results")]
//public string[] Results { get; set; } = [];
public override string ToString()
{
return $"STEP {Step}: {Task}";

8
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Models/SecondStagePlan.cs → src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Models/SqlSecondStagePlan.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Plugin.Planner.SqlGeneration.Models;
public class SecondStagePlan
public class SqlSecondStagePlan
{
[JsonPropertyName("related_tables")]
public string[] Tables { get; set; } = [];
@ -10,10 +10,4 @@ public class SecondStagePlan
[JsonPropertyName("description")]
public string Description { get; set; } = "";
[JsonPropertyName("input_args")]
public JsonDocument[] Parameters { get; set; } = [];
[JsonPropertyName("output_results")]
public string[] Results { get; set; } = [];
}

4
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/SqlGenerationPlanner.cs
View file

@ -94,8 +94,8 @@ public class SqlGenerationPlanner : ITaskPlanner
private async Task<string> GetNextStepPrompt(Agent router)
{
var agentService = _services.GetRequiredService<IAgentService>();
var planner = await agentService.LoadAgent(PlannerAgentId.TwoStagePlanner);
var template = planner.Templates.First(x => x.Name == "two_stage.next").Content;
var planner = await agentService.LoadAgent(PlannerAgentId.SqlPlanner);
var template = planner.Templates.First(x => x.Name == "sql.next").Content;
var states = _services.GetRequiredService<IConversationStateService>();
var render = _services.GetRequiredService<ITemplateRender>();
return render.Render(template, new Dictionary<string, object>

2
src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs → src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/PrimaryStagePlanFn.cs
View file

@ -1,6 +1,6 @@
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.Functions;
namespace BotSharp.Plugin.Planner.TwoStaging.Functions;
public class PrimaryStagePlanFn : IFunctionCallback
{

2
src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs → src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SecondaryStagePlanFn.cs
View file

@ -1,6 +1,6 @@
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.Functions;
namespace BotSharp.Plugin.Planner.TwoStaging.Functions;
public class SecondaryStagePlanFn : IFunctionCallback
{

3
src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs → src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.Functions;
namespace BotSharp.Plugin.Planner.TwoStaging.Functions;
public class SummaryPlanFn : IFunctionCallback
{

4
src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerUtilityHook.cs → src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Hooks/TwoStagingPlannerUtilityHook.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Plugin.Planner.Hooks;
namespace BotSharp.Plugin.Planner.TwoStaging.Hooks;
public class PlannerUtilityHook : IAgentUtilityHook
public class TwoStagingPlannerUtilityHook : IAgentUtilityHook
{
private const string PRIMARY_STAGE_FN = "plan_primary_stage";
private const string SECONDARY_STAGE_FN = "plan_secondary_stage";

1
src/Plugins/BotSharp.Plugin.Planner/Using.cs
View file

@ -32,5 +32,4 @@ global using BotSharp.Abstraction.Routing.Models;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Core.Routing.Reasoning;
global using BotSharp.Plugin.Planner.Hooks;
global using BotSharp.Plugin.Planner.Enums;

2
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/functions/plan_primary_stage.json → src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/functions/sql_primary_stage.json
View file

@ -1,5 +1,5 @@
{
"name": "plan_primary_stage",
"name": "sql_primary_stage",
"description": "Plan the high level steps to finish the task",
"parameters": {
"type": "object",

2
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/functions/plan_secondary_stage.json → src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/functions/sql_secondary_stage.json
View file

@ -1,5 +1,5 @@
{
"name": "plan_secondary_stage",
"name": "sql_secondary_stage",
"description": "Based on the primary stage planning, make more detail steps of the second stage if the primary stage needs more information.",
"parameters": {
"type": "object",

6
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/instructions/instruction.liquid
View file

@ -3,12 +3,12 @@ You are going convert the user requirement into sql statements.
The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs.
Follow these steps strictly and in order.
1. If user raised a new task, call plan_primary_stage to generate the primary plan.
1. If user raised a new task, call sql_primary_stage to generate the primary plan.
If the sql response can be generate directly based on the context, directly go to step 6 to call function sql_review.
2. If need_lookup_dictionary is True, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name.
* If you no items retured, you can pull 100 records from the table and look for the match.
* If there is no any item found, you can pull up to 100 records from the table and look for the match.
* If need_lookup_dictionary is False, skip calling verify_dictionary_term.
3. If need_breakdown_task is true, call plan_secondary_stage for the specific primary stage.
3. If need_breakdown_task is true, call sql_secondary_stage for the specific primary stage.
4. Repeat step 3 until you processed all the primary steps.
5. Call sql_generation function to generate SQL statements.
6. Call sql_review function to review SQL statements. This is the step you must go through before reply to the user.

0
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/two_stage.summarize.liquid → src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/sql.generation.liquid
View file

0
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/two_stage.next.liquid → src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/sql.next.liquid
View file

4
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/two_stage.1st.plan.liquid → src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/sql.primary.plan.liquid
View file

@ -36,4 +36,6 @@ Domain Knowledge:
=====
Task description:
{{ task_description }}
{{ task_description }}
Output the sub-tasks in JSON format:

0
src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/two_stage.2nd.plan.liquid → src/Plugins/BotSharp.Plugin.Planner/data/agents/da7aad2c-8112-48a2-ab7b-1f87da524741/templates/sql.secondary.plan.liquid
View file

1
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlValidateFn.cs
View file

@ -27,6 +27,7 @@ public class SqlValidateFn : IFunctionCallback
}
sql = Regex.Match(sql, pattern).Groups[1].Value;
message.Content = sql;
var dbHook = _services.GetRequiredService<ISqlDriverHook>();
var dbType = dbHook.GetDatabaseType(message);

29
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
View file

@ -43,26 +43,17 @@ public class SqlDriverPlanningHook : IPlanningHook
return;
}
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
wholeDialogs.Add(RoleDialogModel.From(msg));
wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, $"call execute_sql to run query, set formatting_result as {settings.FormattingResult}"));
var agent = await _services.GetRequiredService<IAgentService>().LoadAgent(BuiltInAgentId.SqlDriver);
var completion = CompletionProvider.GetChatCompletion(_services,
provider: agent.LlmConfig.Provider,
model: agent.LlmConfig.Model);
var response = await completion.GetChatCompletions(agent, wholeDialogs);
// Invoke "execute_sql"
await routing.InvokeFunction(response.FunctionName, response);
msg.CurrentAgentId = agent.Id;
msg.FunctionName = response.FunctionName;
msg.FunctionArgs = response.FunctionArgs;
msg.Content = response.Content;
msg.StopCompletion = response.StopCompletion;
var executionMsg = new RoleDialogModel(AgentRole.Function, "execute sql and format the result")
{
FunctionArgs = JsonSerializer.Serialize(new ExecuteQueryArgs
{
SqlStatements = [msg.Content],
FormattingResult = settings.FormattingResult
})
};
await routing.InvokeFunction("execute_sql", executionMsg);
msg.Content = $"The SQL query has been reviewed and executed, the formatted result is: \r\n{executionMsg.Content}";
}
public async Task OnPlanningCompleted(string planner, RoleDialogModel msg)

13
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -1,3 +1,4 @@
using Azure;
using System.IO;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
@ -123,6 +124,11 @@ public class PlaywrightInstance : IDisposable
return page;
}
page.Request += async (sender, e) =>
{
await HandleFetchRequest(e, message, args);
};
page.Response += async (sender, e) =>
{
await HandleFetchResponse(e, message, args);
@ -131,6 +137,13 @@ public class PlaywrightInstance : IDisposable
return page;
}
public async Task HandleFetchRequest(IRequest request, MessageInfo message, PageActionArgs args)
{
if (request.ResourceType == "fetch" || request.ResourceType == "xhr")
{
}
}
public async Task HandleFetchResponse(IResponse response, MessageInfo message, PageActionArgs args)
{
if (response.Status != 204 &&

5
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs
View file

@ -21,6 +21,11 @@ public partial class PlaywrightWebDriver
if (args.EnableResponseCallback)
{
page.Request += async (sender, e) =>
{
await _instance.HandleFetchRequest(e, message, args);
};
page.Response += async (sender, e) =>
{
await _instance.HandleFetchResponse(e, message, args);