Merge pull request #682 from Qtoss-AI/master

optimize sql driver

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs

@@ -43,4 +43,6 @@ public class PageActionArgs
     /// Wait time in seconds after page is opened
     /// </summary>
     public int WaitTime { get; set; }
+
+    public bool ReadInnerHTMLAsBody { get; set; } = false;
 }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -27,6 +27,7 @@ public interface IBotSharpRepository
     User? GetUserByAffiliateId(string affiliateId) => throw new NotImplementedException();
     User? GetUserByUserName(string userName) => throw new NotImplementedException();
     void CreateUser(User user) => throw new NotImplementedException();
+    void UpdateExistUser(string userId, User user) => throw new NotImplementedException();
     void UpdateUserVerified(string userId) => throw new NotImplementedException();
     void UpdateUserVerificationCode(string userId, string verficationCode) => throw new NotImplementedException();
     void UpdateUserPassword(string userId, string password) => throw new NotImplementedException();

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationRequestModel.cs

@@ -1,5 +1,3 @@
-using BotSharp.Abstraction.Infrastructures.Enums;
-
 namespace BotSharp.OpenAPI.ViewModels.Translations;
 
 public class TranslationRequestModel
@@ -7,3 +5,15 @@ public class TranslationRequestModel
     public string Text { get; set; } = null!;
     public string ToLang { get; set; } = LanguageType.CHINESE;
 }
+
+public class TranslationScriptTimestamp
+{
+    public string Text {  set; get; } = null!;
+    public string Timestamp { get; set; } = null!;
+}
+
+public class TranslationLongTextRequestModel
+{
+    public TranslationScriptTimestamp[] Texts { get; set; } = null!;
+    public string ToLang { get; set; } = LanguageType.CHINESE;
+}

src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs

@@ -33,4 +33,6 @@ public class UserRole
     /// AI Assistant
     /// </summary>
     public const string Assistant = "assistant";
+
+    public const string Root = "root";
 }

src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs

@@ -13,7 +13,8 @@ public interface IUserService
     Task<User> GetMyProfile();
     Task<bool> VerifyUserNameExisting(string userName);
     Task<bool> VerifyEmailExisting(string email);
-    Task<bool> SendVerificationCodeResetPassword(User user);
+    Task<bool> SendVerificationCodeResetPasswordNoLogin(User user);
+    Task<bool> SendVerificationCodeResetPasswordLogin();
     Task<bool> ResetUserPassword(User user);
     Task<bool> ModifyUserEmail(string email);
     Task<bool> ModifyUserPhone(string phone);

src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs

@@ -101,12 +101,12 @@ public class TranslationService : ITranslationService
             {
                 var translatedStringList = await InnerTranslate(texts, language, template);
 
-                int retry = 0;
+                /*int retry = 0;
                 while (translatedStringList.Texts.Length != texts.Count && retry < 3)
                 {
                     translatedStringList = await InnerTranslate(texts, language, template);
                     retry++;
-                }
+                }*/
 
                 // Override language if it's Unknown, it's used to output the corresponding language.
                 var states = _services.GetRequiredService<IConversationStateService>();
@@ -119,7 +119,7 @@ public class TranslationService : ITranslationService
                 var translatedTexts = translatedStringList.Texts;
                 var memoryInputs = new List<TranslationMemoryInput>();
 
-                for (var i = 0; i < texts.Count; i++)
+                for (var i = 0; i < Math.Min(texts.Count, translatedTexts.Length); i++)
                 {
                     map[outOfMemoryList[i].OriginalText] = translatedTexts[i].Text;
                     memoryInputs.Add(new TranslationMemoryInput
@@ -375,6 +375,8 @@ public class TranslationService : ITranslationService
         var render = _services.GetRequiredService<ITemplateRender>();
         var prompt = render.Render(template, translator.TemplateDict);
 
+        _logger.LogInformation($"Translation prompt: {prompt}");
+
         var translationDialogs = new List<RoleDialogModel>
         {
             new RoleDialogModel(AgentRole.User, prompt)
@@ -384,6 +386,8 @@ public class TranslationService : ITranslationService
             }
         };
         var response = await _completion.GetChatCompletions(translator, translationDialogs);
+
+        _logger.LogInformation(response.Content);
         return response.Content.JsonContent<TranslationOutput>();
     }
 

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -32,6 +32,7 @@ public class UserService : IUserService
 
     public async Task<User> CreateUser(User user)
     {
+        string hasRegisterId = null;
         if (string.IsNullOrEmpty(user.UserName))
         {
             // generate unique name
@@ -48,7 +49,7 @@ public class UserService : IUserService
 
         if (record != null)
         {
-            return record;
+            hasRegisterId = record.Id;
         }
 
         if (string.IsNullOrEmpty(user.Id))
@@ -71,7 +72,14 @@ public class UserService : IUserService
             record.Verified = false;
         }
 
-        db.CreateUser(record);
+        if (hasRegisterId == null)
+        {
+            db.CreateUser(record);
+        }
+        else
+        {
+            db.UpdateExistUser(hasRegisterId, record);
+        }
 
         _logger.LogWarning($"Created new user account: {record.Id} {record.UserName}");
         Utilities.ClearCache();
@@ -386,8 +394,9 @@ public class UserService : IUserService
         }
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
+
         var user = db.GetUserByUserName(userName);
-        if (user != null)
+        if (user != null && user.Verified)
         {
             return true;
         }
@@ -404,7 +413,7 @@ public class UserService : IUserService
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var emailName = db.GetUserByEmail(email);
-        if (emailName != null)
+        if (emailName != null && emailName.Verified)
         {
             return true;
         }
@@ -412,7 +421,48 @@ public class UserService : IUserService
         return false;
     }
 
-    public async Task<bool> SendVerificationCodeResetPassword(User user)
+    public async Task<bool> SendVerificationCodeResetPasswordNoLogin(User user)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+
+        User? record = null;
+
+        if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
+        {
+            return false;
+        }
+
+        if (!string.IsNullOrEmpty(user.Phone))
+        {
+            record = db.GetUserByPhone(user.Phone);
+        }
+
+        if (!string.IsNullOrEmpty(user.Email))
+        {
+            record = db.GetUserByEmail(user.Email);
+        }
+
+        if (record == null)
+        {
+            return false;
+        }
+
+        record.VerificationCode = Nanoid.Generate(alphabet: "0123456789", size: 6);
+
+        //update current verification code.
+        db.UpdateUserVerificationCode(record.Id, record.VerificationCode);
+
+        //send code to user Email.
+        var hooks = _services.GetServices<IAuthenticationHook>();
+        foreach (var hook in hooks)
+        {
+            hook.VerificationCodeResetPassword(record);
+        }
+
+        return true;
+    }
+
+    public async Task<bool> SendVerificationCodeResetPasswordLogin()
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
 
@@ -422,23 +472,6 @@ public class UserService : IUserService
         {
             record = db.GetUserById(_user.Id);
         }
-        else
-        {
-            if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
-            {
-                return false;
-            }
-
-            if (!string.IsNullOrEmpty(user.Email))
-            {
-                record = db.GetUserByEmail(user.Email);
-            }
-
-            if (!string.IsNullOrEmpty(user.Phone))
-            {
-                record = db.GetUserByPhone(user.Phone);
-            }
-        }
 
         if (record == null)
         {
@@ -520,6 +553,11 @@ public class UserService : IUserService
             return false;
         }
 
+        if ((record.UserName.Substring(0, 3) == "+86" || record.FirstName.Substring(0, 3) == "+86") && phone.Substring(0, 3) != "+86")
+        {
+            phone = $"+86{phone}";
+        }
+
         db.UpdateUserPhone(record.Id, phone);
         return true;
     }

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid

@@ -1,6 +1,19 @@
 {{ text_list }}
 
 =====
+{% if language == "Chinese" %}
+将以上所有句子翻译成中文。
+
+要求：
+* 以 JSON 格式输出翻译后的文本 {"input_lang":"原始文本语言", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""},{"id": 2, "text":""}]}。
+* output_count 必须等于输出中texts数组的长度。
+{% else %}
 Translate all the above sentences into {{ language }}.
-Output the translated text in JSON {"input_lang":"original text language", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""},{"id": 2, "text":""}]}.
-The "output_count" must equal the length of the "texts" array in the output.
+
+Requirements:
+* Output the translated text in JSON {"input_lang":"original text language", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""},{"id": 2, "text":""}]}.
+* The "output_count" must equal the length of the "texts" array in the output.
+{% endif %}
+
+
+

src/Infrastructure/BotSharp.OpenAPI/Controllers/TranslationController.cs

@@ -1,4 +1,4 @@
-using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.Translation;
 using BotSharp.OpenAPI.ViewModels.Translations;
 
@@ -9,10 +9,13 @@ namespace BotSharp.OpenAPI.Controllers;
 public class TranslationController : ControllerBase
 {
     private readonly IServiceProvider _services;
+    private readonly JsonSerializerOptions _jsonOptions;
 
-    public TranslationController(IServiceProvider services)
+    public TranslationController(IServiceProvider services,
+        BotSharpOptions options)
     {
         _services = services;
+        _jsonOptions = InitJsonOptions(options);
     }
 
     [HttpPost("/translate")]
@@ -21,10 +24,79 @@ public class TranslationController : ControllerBase
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(BuiltInAgentId.AIAssistant);
         var translator = _services.GetRequiredService<ITranslationService>();
-        var text = await translator.Translate(agent, Guid.NewGuid().ToString(), model.Text, language: model.ToLang);
+        var states = _services.GetRequiredService<IConversationStateService>();
+        states.SetState("max_tokens", "8192");
+        var text = await translator.Translate(agent, Guid.NewGuid().ToString(), model.Text.Split("\r\n"), language: model.ToLang);
         return new TranslationResponseModel
         {
-            Text = text
+            Text = string.Join("\r\n", text)
         };
     }
+
+    [HttpPost("/translate/long-text")]
+    public async Task SendMessageSse([FromBody] TranslationLongTextRequestModel model)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(BuiltInAgentId.AIAssistant);
+        var translator = _services.GetRequiredService<ITranslationService>();
+
+        Response.StatusCode = 200;
+        Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.ContentType, "text/event-stream");
+        Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.CacheControl, "no-cache");
+        Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.Connection, "keep-alive");
+
+        foreach (var script in model.Texts)
+        {
+            var translatedText = await translator.Translate(agent, Guid.NewGuid().ToString(), script.Text, language: model.ToLang);
+
+            var json = JsonSerializer.Serialize(new TranslationScriptTimestamp
+            {
+                Text = translatedText,
+                Timestamp = script.Timestamp
+            }, _jsonOptions);
+
+            await OnChunkReceived(Response, json);
+        }
+
+        await OnEventCompleted(Response);
+    }
+
+    private async Task OnChunkReceived(HttpResponse response, string text)
+    {
+        var buffer = Encoding.UTF8.GetBytes($"data:{text}\n");
+        await response.Body.WriteAsync(buffer, 0, buffer.Length);
+        await Task.Delay(10);
+
+        buffer = Encoding.UTF8.GetBytes("\n");
+        await response.Body.WriteAsync(buffer, 0, buffer.Length);
+    }
+
+    private async Task OnEventCompleted(HttpResponse response)
+    {
+        var buffer = Encoding.UTF8.GetBytes("data:[DONE]\n");
+        await response.Body.WriteAsync(buffer, 0, buffer.Length);
+
+        buffer = Encoding.UTF8.GetBytes("\n");
+        await response.Body.WriteAsync(buffer, 0, buffer.Length);
+    }
+
+    private JsonSerializerOptions InitJsonOptions(BotSharpOptions options)
+    {
+        var jsonOption = new JsonSerializerOptions
+        {
+            PropertyNameCaseInsensitive = true,
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+            AllowTrailingCommas = true
+        };
+
+        if (options?.JsonSerializerOptions != null)
+        {
+            foreach (var option in options.JsonSerializerOptions.Converters)
+            {
+                jsonOption.Converters.Add(option);
+            }
+        }
+
+        return jsonOption;
+    }
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs

@@ -108,12 +108,20 @@ public class UserController : ControllerBase
     {
         return await _userService.VerifyEmailExisting(email);
     }
+
     [AllowAnonymous]
-    [HttpPost("/user/verifycode")]
+    [HttpPost("/user/verifycode-out")]
     public async Task<bool> SendVerificationCodeResetPassword([FromBody] UserCreationModel user)
     {
-        return await _userService.SendVerificationCodeResetPassword(user.ToUser());
+        return await _userService.SendVerificationCodeResetPasswordNoLogin(user.ToUser());
     }
+
+    [HttpPost("/user/verifycode-in")]
+    public async Task<bool> SendVerificationCodeResetPasswordLogined()
+    {
+        return await _userService.SendVerificationCodeResetPasswordLogin();
+    }
+
     [AllowAnonymous]
     [HttpPost("/user/resetpassword")]
     public async Task<bool> ResetUserPassword([FromBody] UserResetPasswordModel user)

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -79,6 +79,18 @@ public partial class MongoRepository
         _dc.Users.InsertOne(userCollection);
     }
 
+    public void UpdateExistUser(string userId, User user)
+    {
+        var filter = Builders<UserDocument>.Filter.Eq(x => x.Id, userId);
+        var update = Builders<UserDocument>.Update
+            .Set(x => x.Email, user.Email)
+            .Set(x => x.Phone, user.Phone)
+            .Set(x => x.Salt, user.Salt)
+            .Set(x => x.Password, user.Password)
+            .Set(x => x.VerificationCode, user.VerificationCode);
+        _dc.Users.UpdateOne(filter, update);
+    }
+
     public void UpdateUserVerified(string userId)
     {
         var filter = Builders<UserDocument>.Filter.Eq(x => x.Id, userId);

src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs

@@ -1,4 +1,5 @@
 using BotSharp.Plugin.Planner.TwoStaging.Models;
+using System.Threading.Tasks;
 
 namespace BotSharp.Plugin.Planner.Functions;
 
@@ -27,14 +28,18 @@ public class SecondaryStagePlanFn : IFunctionCallback
         var planPrimary = states.GetState("planning_result");
 
         var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
- 
-        // Search knowledgebase
-        var knowledges = await knowledgeService.SearchVectorKnowledge(taskSecondary.SolutionQuestion, collectionName, new VectorSearchOptions
-        {
-            Confidence = 0.7f
-        });
 
-        var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
+        // Search knowledgebase
+        var hooks = _services.GetServices<IKnowledgeHook>();
+        var knowledges = new List<string>();
+        foreach (var hook in hooks)
+        {
+            var k = await hook.GetRelevantKnowledges(message, taskSecondary.SolutionQuestion);
+            knowledges.AddRange(k);
+        }
+        knowledges = knowledges.Distinct().ToList();
+
+        var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges);
 
         // Get second stage planning prompt
         var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);

src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs

@@ -1,8 +1,6 @@
-using BotSharp.Abstraction.Knowledges;
 using BotSharp.Abstraction.Planning;
 using BotSharp.Plugin.Planner.TwoStaging;
 using BotSharp.Plugin.Planner.TwoStaging.Models;
-using Microsoft.EntityFrameworkCore.Metadata.Internal;
 
 namespace BotSharp.Plugin.Planner.Functions;
 

src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerAgentHook.cs

@@ -2,13 +2,32 @@ namespace BotSharp.Plugin.Planner.Hooks;
 
 public class PlannerAgentHook : AgentHookBase
 {
-    public override string SelfId => string.Empty;
+    public override string SelfId => BuiltInAgentId.Planner;
 
     public PlannerAgentHook(IServiceProvider services, AgentSettings settings)
         : base(services, settings)
     {
     }
 
+    public override bool OnInstructionLoaded(string template, Dictionary<string, object> dict)
+    {
+        var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
+
+        // Get global knowledges
+        var Knowledges = new List<string>();
+        foreach (var hook in knowledgeHooks)
+        {
+            var k = hook.GetGlobalKnowledges(new RoleDialogModel(AgentRole.User, template)
+            {
+                CurrentAgentId = BuiltInAgentId.Planner
+            }).Result;
+            Knowledges.AddRange(k);
+        }
+        dict["global_knowledges"] = Knowledges;
+
+        return true;
+    }
+
     public override void OnAgentLoaded(Agent agent)
     {
         var conv = _services.GetRequiredService<IConversationService>();

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid

@@ -1,4 +1,6 @@
+The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs.
 Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 
+
 1. Call plan_primary_stage to generate the primary plan.
    If you've already got the plan to meet the user goal, directly go to step 5.
 2. If need_lookup_dictionary is True, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name.
@@ -15,6 +17,7 @@ Don't run the planning process repeatedly if you have already got the result of
 {% if global_knowledges != empty -%}
 =====
 Global Knowledge:
+Current date time is: {{ "now" | date: "%Y-%m-%d %H:%M" }}
 {% for k in global_knowledges %}
 {{ k }}
 {% endfor %}

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid

@@ -6,7 +6,6 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown
 * If need_lookup_dictionary is true, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name/code.
 * Output all the steps as much detail as possible in JSON: [{{ response_format }}]
 
-
 Additional Requirements:
 * "output_results" is variable name that needed to be used in the next step.
 

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -30,6 +30,7 @@
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_select.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\query_result_formatting.liquid" />
+    <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_statement_correctness.liquid" />
   </ItemGroup>
 
   <ItemGroup>
@@ -69,6 +70,9 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_executor.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_statement_correctness.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\query_result_formatting.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs

@@ -1,8 +1,10 @@
 using BotSharp.Abstraction.Agents.Enums;
+using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.SqlDriver.Models;
 using Dapper;
 using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Logging;
 using MySqlConnector;
 
 namespace BotSharp.Plugin.SqlDriver.Functions;
@@ -13,31 +15,47 @@ public class ExecuteQueryFn : IFunctionCallback
     public string Indication => "Performing data retrieval operation.";
     private readonly SqlDriverSetting _setting;
     private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
 
-    public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting)
+    public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting, ILogger<ExecuteQueryFn> logger)
     {
         _services = services;
         _setting = setting;
+        _logger = logger;
     }
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<ExecuteQueryArgs>(message.FunctionArgs);
-        var settings = _services.GetRequiredService<SqlDriverSetting>();
-        var results = settings.DatabaseType switch
-        {
-            "MySql" => RunQueryInMySql(args.SqlStatements),
-            "SqlServer" => RunQueryInSqlServer(args.SqlStatements),
-            _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
-        };
-        
-        if (results.Count() == 0)
-        {
-            message.Content = "No record found";
-            return true;
-        }
 
-        message.Content = JsonSerializer.Serialize(results);
+        var refinedArgs = await RefineSqlStatement(message, args);
+
+        var settings = _services.GetRequiredService<SqlDriverSetting>();
+
+        try
+        {
+            var results = settings.DatabaseType switch
+            {
+                "MySql" => RunQueryInMySql(refinedArgs.SqlStatements),
+                "SqlServer" => RunQueryInSqlServer(refinedArgs.SqlStatements),
+                _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
+            };
+
+            if (results.Count() == 0)
+            {
+                message.Content = "No record found";
+                return true;
+            }
+
+            message.Content = JsonSerializer.Serialize(results);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error occurred while executing SQL query.");
+            message.Content = "Error occurred while retrieving information.";
+            message.StopCompletion = true;
+            return false;
+        }
 
         if (args.FormattingResult)
         {
@@ -59,6 +77,7 @@ public class ExecuteQueryFn : IFunctionCallback
             });
 
             message.Content = result.Content;
+            message.StopCompletion = true;
         }
 
         return true;
@@ -77,4 +96,54 @@ public class ExecuteQueryFn : IFunctionCallback
         using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
         return connection.Query(string.Join("\r\n", sqlTexts));
     }
+
+    private async Task<ExecuteQueryArgs> RefineSqlStatement(RoleDialogModel message, ExecuteQueryArgs args)
+    {
+        // get table DDL
+        var fn = _services.GetRequiredService<IRoutingService>();
+        var msg = RoleDialogModel.From(message);
+        await fn.InvokeFunction("sql_table_definition", msg);
+
+        // refine SQL
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
+        var dictionarySqlPrompt = await GetDictionarySQLPrompt(string.Join("\r\n\r\n", args.SqlStatements), msg.Content);
+        var agent = new Agent
+        {
+            Id = message.CurrentAgentId ?? string.Empty,
+            Name = "sqlDriver_ExecuteQuery",
+            Instruction = dictionarySqlPrompt,
+            TemplateDict = new Dictionary<string, object>(),
+            LlmConfig = currentAgent.LlmConfig
+        };
+
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: agent.LlmConfig.Provider,
+            model: agent.LlmConfig.Model);
+
+        var refinedMessage = await completion.GetChatCompletions(agent, new List<RoleDialogModel> 
+        { 
+            new RoleDialogModel(AgentRole.User, "Check and output the correct SQL statements") 
+        });
+        
+        return refinedMessage.Content.JsonContent<ExecuteQueryArgs>();
+    }
+
+    private async Task<string> GetDictionarySQLPrompt(string originalSql, string tableStructure)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
+
+        var agent = await agentService.GetAgent(BuiltInAgentId.SqlDriver);
+        var template = agent.Templates.FirstOrDefault(x => x.Name == "sql_statement_correctness")?.Content ?? string.Empty;
+        var responseFormat = JsonSerializer.Serialize(new ExecuteQueryArgs { });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "original_sql", originalSql },
+            { "table_structure", tableStructure },
+            { "response_format", responseFormat }
+        });
+    }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs

@@ -33,20 +33,25 @@ public class SqlDriverPlanningHook : IPlanningHook
         var conv = _services.GetRequiredService<IConversationService>();
         var wholeDialogs = conv.GetDialogHistory();
         wholeDialogs.Add(RoleDialogModel.From(msg));
-        wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query"));
-
-        var agent = await _services.GetRequiredService<IAgentService>().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f");
+        wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, $"call execute_sql to run query, set formatting_result as {settings.FormattingResult}"));
 
+        var agent = await _services.GetRequiredService<IAgentService>().LoadAgent(BuiltInAgentId.SqlDriver);
         var completion = CompletionProvider.GetChatCompletion(_services,
             provider: agent.LlmConfig.Provider,
             model: agent.LlmConfig.Model);
 
         var response = await completion.GetChatCompletions(agent, wholeDialogs);
+
+        // Invoke "execute_sql"
         var routing = _services.GetRequiredService<IRoutingService>();
         await routing.InvokeFunction(response.FunctionName, response);
         msg.CurrentAgentId = agent.Id;
         msg.FunctionName = response.FunctionName;
         msg.FunctionArgs = response.FunctionArgs;
         msg.Content = response.Content;
+        msg.StopCompletion = response.StopCompletion;
+
+        /*var routing = _services.GetRequiredService<IRoutingService>();
+        await routing.InvokeAgent(BuiltInAgentId.SqlDriver, wholeDialogs);*/
     }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs

@@ -7,8 +7,12 @@ public class ExecuteQueryArgs
     [JsonPropertyName("sql_statements")]
     public string[] SqlStatements { get; set; } = [];
 
+    [JsonPropertyName("tables")]
+    public string[] Tables { get; set; } = [];
+
     /// <summary>
     /// Beautifying query result
     /// </summary>
+    [JsonPropertyName("formatting_result")]
     public bool FormattingResult { get; set; }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs

@@ -10,4 +10,5 @@ public class SqlDriverSetting
     public string SqlServerExecutionConnectionString { get; set; } = null!;
     public string SqlLiteConnectionString { get; set; } = null!;
     public bool ExecuteSqlSelectAutonomous { get; set; } = false;
+    public bool FormattingResult { get; set; } = true;
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json

@@ -10,7 +10,7 @@
   "profiles": [ "database" ],
   "llmConfig": {
     "provider": "openai",
-    "model": "gpt-4o-mini"
+    "model": "gpt-4o"
   },
   "routingRules": [
     {

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json

@@ -11,8 +11,22 @@
           "type": "string",
           "description": "sql statement"
         }
+      },
+
+      "formatting_result": {
+        "type": "boolean",
+        "description": "formatting the results"
+      },
+
+      "tables": {
+        "type": "array",
+        "description": "all related tables",
+        "items": {
+          "type": "string",
+          "description": "table name"
+        }
       }
     },
-    "required": [ "sql_statement" ]
+    "required": [ "sql_statement", "tables", "formatting_result" ]
   }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid

@@ -1 +1,5 @@
-Output in human readable format. If there is large amount of information, shape it in tabular.
+Output in human readable format. If there is large amount of rows, shape it in tabular, otherwise, output in plain text.
+Put user task description in the first line in the same language, for example, user is using Chinese, you have to output the result in Chinese.
+
+User Task Description:
+{{ requirement_detail }}

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid

@@ -0,0 +1,11 @@
+You are a sql statement corrector. You will need to refer to the table structure and rewrite the original sql statement so it's using the correct information, e.g. column name.
+Output the sql statement only without comments, in JSON format: {{ response_format }}
+Make sure all the column names are defined in the Table Structure.
+
+=====
+Original SQL statements:
+{{ original_sql }}
+
+=====
+Table Structure:
+{{ table_structure }}

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs

@@ -67,9 +67,13 @@ public partial class PlaywrightWebDriver
             result.ResponseStatusCode = response.Status;
             if (response.Status == 200)
             {
-                // Disable this due to performance issue, some page is too large
-                // result.Body = await page.InnerHTMLAsync("body");
                 result.IsSuccess = true;
+
+                // Be careful if page is too large, it will cause performance issue
+                if (args.ReadInnerHTMLAsBody)
+                {
+                    result.Body = await page.InnerHTMLAsync("body");
+                }
             }
             else
             {