Merge pull request #1074 from Joannall/master

Refactor SQL Driver Plugin and Update Database Handling

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -108,4 +108,8 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Helpers\" />
+  </ItemGroup>
+
 </Project>

src/Plugins/BotSharp.Plugin.SqlDriver/Helpers/SqlDriverHelper.cs

@@ -1,20 +0,0 @@
-namespace BotSharp.Plugin.SqlDriver.Helpers;
-
-internal static class SqlDriverHelper
-{
-    internal static string GetDatabaseType(IServiceProvider services)
-    {
-        var settings = services.GetRequiredService<SqlDriverSetting>();
-        var dbType = "mysql";
-
-        if (!string.IsNullOrWhiteSpace(settings?.SqlServerConnectionString))
-        {
-            dbType = "sqlserver";
-        }
-        else if (!string.IsNullOrWhiteSpace(settings?.SqlLiteConnectionString))
-        {
-            dbType = "sqllite";
-        }
-        return dbType;
-    }
-}

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverAgentHook.cs

@@ -1,19 +0,0 @@
-using BotSharp.Abstraction.Agents.Settings;
-
-namespace BotSharp.Plugin.SqlDriver.Hooks;
-
-public class SqlDriverAgentHook : AgentHookBase, IAgentHook
-{
-    public override string SelfId => string.Empty;
-
-    public SqlDriverAgentHook(IServiceProvider services, AgentSettings settings)
-        : base(services, settings)
-    {
-    }
-
-    public override void OnAgentLoaded(Agent agent)
-    {
-        var dbType = SqlDriverHelper.GetDatabaseType(_services);
-        agent.TemplateDict["db_type"] = dbType;
-    }
-}

src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs

@@ -4,12 +4,18 @@ namespace BotSharp.Plugin.SqlDriver.Models;
 
 public class SqlStatement
 {
+    [JsonPropertyName("db_provider")]
+    public string DBProvider { get; set; } = null!;
+
     [JsonPropertyName("sql_statement")]
     public string Statement { get; set; } = null!;
 
     [JsonPropertyName("reason")]
     public string Reason { get; set; } = null!;
 
+    [JsonPropertyName("schema")]
+    public string Schema { get; set; } = null!;
+
     [JsonPropertyName("tables")]
     public string[] Tables { get; set; } = null!;
 

src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs

@@ -27,7 +27,6 @@ public class SqlDriverPlugin : IBotSharpPlugin
         services.AddScoped<DbKnowledgeService>();
         services.AddScoped<IPlanningHook, SqlDriverPlanningHook>();
         services.AddScoped<IKnowledgeHook, SqlDriverKnowledgeHook>();
-        services.AddScoped<IAgentHook, SqlDriverAgentHook>();
         services.AddScoped<IConversationHook, SqlDriverConversationHook>();
         services.AddScoped<IAgentUtilityHook, SqlUtilityHook>();
         services.AddScoped<ICrontabHook, SqlDriverCrontabHook>();

src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs

@@ -30,5 +30,4 @@ global using BotSharp.Plugin.SqlDriver.Models;
 global using BotSharp.Plugin.SqlDriver.Hooks;
 global using BotSharp.Plugin.SqlDriver.Services;
 global using BotSharp.Plugin.SqlDriver.Interfaces;
-global using BotSharp.Plugin.SqlDriver.Helpers;
 global using BotSharp.Plugin.SqlDriver.Settings;

src/Plugins/BotSharp.Plugin.SqlDriver/UtilFunctions/GetTableDefinitionFn.cs

@@ -23,16 +23,16 @@ public class GetTableDefinitionFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
         var tables = args.Tables;
+        var dbType = args.DBProvider;
+        var schema = args.Schema;
         var agentService = _services.GetRequiredService<IAgentService>();
-        var dbHook = _services.GetRequiredService<ISqlDriverHook>();
-        var dbType = dbHook.GetDatabaseType(message);
 
         // Get table DDL from database
         var tableDdls = dbType switch
         {
             "mysql" => GetDdlFromMySql(tables),
             "sqlserver" => GetDdlFromSqlServer(tables),
-            "redshift" => GetDdlFromRedshift(tables),
+            "redshift" => GetDdlFromRedshift(tables,schema),
             _ => throw new NotImplementedException($"Database type {dbType} is not supported.")
         };
     
@@ -127,7 +127,7 @@ public class GetTableDefinitionFn : IFunctionCallback
         return tableDdls;
     }
 
-    private List<string> GetDdlFromRedshift(string[] tables)
+    private List<string> GetDdlFromRedshift(string[] tables, string schema)
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         var tableDdls = new List<string>();

src/Plugins/BotSharp.Plugin.SqlDriver/UtilFunctions/SqlSelect.cs

@@ -25,9 +25,7 @@ public class SqlSelect : IFunctionCallback
             return false;
         }
 
-        // check if need to instantely
+        var dbType = args.DBProvider.ToLowerInvariant();
-        var dbHook = _services.GetRequiredService<ISqlDriverHook>();
-        var dbType = dbHook.GetDatabaseType(message);
 
         var result = dbType switch
         {

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-db-sql_select.json

@@ -4,6 +4,16 @@
   "parameters": {
     "type": "object",
     "properties": {
+      "db_provider": {
+        "type": "string",
+        "enum": [
+          "mysql",
+          "postgresql",
+          "mssql",
+          "redshift"
+        ],
+        "description": "The database engine."
+      },
       "sql_statement": {
         "type": "string",
         "description": "SQL statement with SELECT"
@@ -55,6 +65,6 @@
         "required": [ "name", "value" ]
       }
     },
-    "required": [ "sql_statement", "reason", "table", "parameters", "return_field" ]
+    "required": [ "db_provider", "sql_statement", "reason", "table", "parameters", "return_field" ]
   }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-db-sql_table_definition.json

@@ -4,6 +4,24 @@
   "parameters": {
     "type": "object",
     "properties": {
+      "db_provider": {
+        "type": "string",
+        "enum": [
+          "mysql",
+          "postgresql",
+          "mssql",
+          "redshift"
+        ],
+        "description": "The database engine."
+      },
+      "schema": {
+        "type": "string",
+        "description": "schema name for tables. Typically, the part before the dot is the schema name, e.g.smsonebi.affiliate_profile, schema name is smsonebi.",
+        "items": {
+          "type": "string",
+          "description": "schema name"
+        }
+      },
       "tables": {
         "type": "array",
         "description": "table name in planning steps",
@@ -17,6 +35,6 @@
         "description": "the reason why you need to call sql_table_definition"
       }
     },
-    "required": [ "tables", "reason" ]
+    "required": [ "db_provider", "schema", "tables", "reason" ]
   }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-db-sql_executor.fn.liquid

@@ -1,6 +1,9 @@
-You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
+Please call function util-db-sql_select if user wants to retrieve data from database tables.
-
+- Ensure you obtain the exact provider, schema, table and column names.
-Please call function util-db-sql_select if user wants to get or retrieve data from data tables.
+    If user provide them, directly call util-db-sql_select to execute the query.
-If there are any parameters, please add them in the WHERE clause, each of which starts with "@".
+    If user didn't provide the provider(e.g. mysql, redshift), schema or table name, ask user to provide.
-Avoid returning the entire record and only return the fields you need.
+    If the user doesn't provide exact column name, try using other available functions to infer the information.
-For example, SELECT Id FROM table WHERE Id=@Id AND Name=@Name
+    If that fails or no function can use, prompt the user to provide the missing information.
+- If there are any parameters, please add them in the WHERE clause, each of which starts with "@".
+- Avoid returning the entire record and only return the fields you need.
+    For example, SELECT Id FROM table WHERE Id=@Id AND Name=@Name

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-db-verify_dictionary_term.fn.liquid

@@ -4,8 +4,6 @@ Rules to call util-db-verify_dictionary_term:
 2. You must return the id and name/code for existing dictionary. 
 3. Call the function only if need_lookup_dictionary is true.
 
-You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
-
 The dictionary table is identified by a name that begins with "data_".
 You are only allowed to query the dictionary table without joining it with other non-dictionary tables.