diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 02d4f3a5..9895aedf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -24,7 +24,7 @@ public class PrimaryStagePlanFn : IFunctionCallback state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + var collectionName = knowledgeSettings.Default.CollectionName; // Get knowledge from vectordb var hooks = _services.GetServices(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index b818c458..ba7c7047 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -17,16 +17,14 @@ public class SecondaryStagePlanFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var fn = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var knowledgeService = _services.GetRequiredService(); var knowledgeSettings = _services.GetRequiredService(); var states = _services.GetRequiredService(); var msgSecondary = RoleDialogModel.From(message); - var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + var collectionName = knowledgeSettings.Default.CollectionName; var planPrimary = states.GetState("planning_result"); - var taskPrimary = states.GetState("requirement_detail"); var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); @@ -35,8 +33,8 @@ public class SecondaryStagePlanFn : IFunctionCallback { Confidence = 0.6f }); - var knowledgeResults = ""; - knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); + + var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); // Get second stage planning prompt var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); @@ -45,7 +43,7 @@ public class SecondaryStagePlanFn : IFunctionCallback var plannerAgent = new Agent { - Id = string.Empty, + Id = BuiltInAgentId.Planner, Name = "planning_2nd", Instruction = secondPlanningPrompt, TemplateDict = new Dictionary(), diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 7c930c39..12724c24 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -33,7 +33,7 @@ public class SummaryPlanFn : IFunctionCallback var states = _services.GetRequiredService(); var steps = states.GetState("planning_result").JsonArrayContent(); var allTables = new List(); - var ddlStatements = ""; + var ddlStatements = string.Empty; var relevantKnowledge = states.GetState("planning_result"); var dictionaryItems = states.GetState("dictionary_items"); @@ -42,6 +42,7 @@ public class SummaryPlanFn : IFunctionCallback allTables.AddRange(step.Tables); } var distinctTables = allTables.Distinct().ToList(); + foreach (var table in distinctTables) { var msgCopy = RoleDialogModel.From(message); @@ -92,7 +93,7 @@ public class SummaryPlanFn : IFunctionCallback return render.Render(template, new Dictionary { { "task_description", taskDescription }, - { "summary_requirements", string.Join("\r\n",additionalRequirements) }, + { "summary_requirements", string.Join("\r\n", additionalRequirements) }, { "relevant_knowledges", relevantKnowledge }, { "dictionary_items", dictionaryItems }, { "table_structure", ddlStatement }, diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json index d4e8bb77..07534191 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json @@ -11,7 +11,7 @@ "profiles": [ "planning" ], "utilities": [ "two-stage-planner" ], "llmConfig": { - "provider": "azure-openai", + "provider": "openai", "model": "gpt-4o", "max_recursion_depth": 10 } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 35f16707..02bca766 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -11,8 +11,11 @@ + + + @@ -80,8 +83,4 @@ - - - - diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs index f863adb7..6f16aafa 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs @@ -19,6 +19,6 @@ public class SqlDriverController : ControllerBase public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) { var dbKnowledge = _services.GetRequiredService(); - return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema); + return await dbKnowledge.Import(request); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 7833bfb4..d8ae5097 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -44,7 +44,6 @@ public class ExecuteQueryFn : IFunctionCallback { var settings = _services.GetRequiredService(); using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); - var dictionary = new Dictionary(); return connection.Query(string.Join("\r\n", sqlTexts)); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index b3a3c134..b5c9283e 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,5 +1,4 @@ using BotSharp.Plugin.SqlDriver.Models; -using Fluid.Ast.BinaryExpressions; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using MySqlConnector; @@ -37,10 +36,6 @@ public class GetTableDefinitionFn : IFunctionCallback }; message.Content = string.Join("\r\n\r\n", tableDdls); - - //var states = _services.GetRequiredService(); - //states.SetState($"table_definition_{args.Table}", message.Content); - return true; } @@ -48,7 +43,7 @@ public class GetTableDefinitionFn : IFunctionCallback { var settings = _services.GetRequiredService(); var tableDdls = new List(); - using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString); connection.Open(); foreach (var table in tables) @@ -76,7 +71,6 @@ public class GetTableDefinitionFn : IFunctionCallback } connection.Close(); - return tableDdls; } @@ -92,27 +86,27 @@ public class GetTableDefinitionFn : IFunctionCallback try { var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}'; -DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' ('; + DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' ('; -SELECT @SQL = @SQL + ' - ' + COLUMN_NAME + ' ' + - DATA_TYPE + - CASE - WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' - THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')' - WHEN DATA_TYPE IN ('decimal', 'numeric') - THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')' - ELSE '' - END + ' ' + - CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' -FROM INFORMATION_SCHEMA.COLUMNS -WHERE TABLE_NAME = @TableName -ORDER BY ORDINAL_POSITION; + SELECT @SQL = @SQL + ' + ' + COLUMN_NAME + ' ' + + DATA_TYPE + + CASE + WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%' + THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')' + WHEN DATA_TYPE IN ('decimal', 'numeric') + THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')' + ELSE '' + END + ' ' + + CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ',' + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = @TableName + ORDER BY ORDINAL_POSITION; --- Remove the last comma and add closing parenthesis -SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');'; + -- Remove the last comma and add closing parenthesis + SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');'; -SELECT @SQL;"; + SELECT @SQL;"; using var command = new SqlCommand(sql, connection); using var reader = command.ExecuteReader(); @@ -129,7 +123,6 @@ SELECT @SQL;"; } connection.Close(); - return tableDdls; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs index e0f9d529..2c985c9e 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs @@ -1,14 +1,9 @@ -using Azure; using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Routing; -using BotSharp.Core.Agents.Services; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.SqlDriver.Models; using MySqlConnector; -using System.Text.RegularExpressions; using static Dapper.SqlMapper; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Plugin.SqlDriver.Functions; @@ -35,15 +30,16 @@ public class LookupDictionaryFn : IFunctionCallback var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); var dictionarySqlPrompt = await GetDictionarySQLPrompt(args.SqlStatement, msgCopy.Content); - var plannerAgent = new Agent + var agent = new Agent { - Id = string.Empty, + Id = message.CurrentAgentId ?? string.Empty, Name = "sqlDriver_DictionarySearch", Instruction = dictionarySqlPrompt, TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAiResponse(plannerAgent); + + var response = await GetAiResponse(agent); args = JsonSerializer.Deserialize(response.Content); // check if need to instantely @@ -59,13 +55,16 @@ public class LookupDictionaryFn : IFunctionCallback { message.Content = JsonSerializer.Serialize(result); } + var states = _services.GetRequiredService(); var dictionaryItems = states.GetState("dictionary_items", ""); - dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n"; + var newItem = BuildDictionaryItem(args.Table, args.Reason, message.Content); + dictionaryItems += !string.IsNullOrWhiteSpace(newItem) ? $"\r\n{newItem}\r\n" : string.Empty; states.SetState("dictionary_items", dictionaryItems); return true; } + private async Task GetDictionarySQLPrompt(string originalSql, string tableStructure) { var agentService = _services.GetRequiredService(); @@ -83,15 +82,45 @@ public class LookupDictionaryFn : IFunctionCallback { "response_format", responseFormat } }); } - private async Task GetAiResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent agent) { var text = "Check and correct the SQL statement."; var message = new RoleDialogModel(AgentRole.User, text); var completion = CompletionProvider.GetChatCompletion(_services, - provider: plannerAgent.LlmConfig.Provider, - model: plannerAgent.LlmConfig.Model); + provider: agent.LlmConfig.Provider, + model: agent.LlmConfig.Model); - return await completion.GetChatCompletions(plannerAgent, new List { message }); + return await completion.GetChatCompletions(agent, new List { message }); + } + + private string BuildDictionaryItem(string? table, string? reason, string? result) + { + var res = string.Empty; + if (!string.IsNullOrWhiteSpace(table)) + { + res += $"Table: {table}"; + } + + if (!string.IsNullOrWhiteSpace(reason)) + { + if (!string.IsNullOrWhiteSpace(res)) + { + res += "\r\n"; + } + res += $"Reason: {reason}"; + } + + if (!string.IsNullOrWhiteSpace(result)) + { + if (!string.IsNullOrWhiteSpace(res)) + { + res += "\r\n"; + } + res += $"Result: {result}"; + } + + return res; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs index 0c0c9f24..edaa0cf3 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs @@ -2,7 +2,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; -using System.Collections.Generic; namespace BotSharp.Plugin.SqlDriver.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index c867bbe7..ac99c699 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Planning; using BotSharp.Abstraction.Routing; -using BotSharp.Core.Agents.Services; using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.SqlDriver.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs index 504ba9b8..810aa44f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs @@ -5,11 +5,11 @@ namespace BotSharp.Plugin.SqlDriver.Models; public class LookupDictionary { [JsonPropertyName("sql_statement")] - public string SqlStatement { get; set; } + public string? SqlStatement { get; set; } [JsonPropertyName("reason")] - public string Reason { get; set; } + public string? Reason { get; set; } [JsonPropertyName("table")] - public string Table { get; set; } + public string? Table { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs index 48ba820c..fa2b70e9 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs @@ -16,4 +16,7 @@ public class ImportDbKnowledgeRequest : RequestBase { [JsonPropertyName("schema")] public string Schema { get; set; } + + [JsonPropertyName("knowledgebase_collection")] + public string KnowledgebaseCollection { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs index 129da04c..2412d3ce 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -3,9 +3,9 @@ using Microsoft.Extensions.Logging; using BotSharp.Core.Infrastructures; using MySqlConnector; using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Plugin.SqlDriver.Models; namespace BotSharp.Plugin.SqlDriver.Services; @@ -22,12 +22,14 @@ public class DbKnowledgeService _logger = logger; } - public async Task Import(string provider, string model, string schema) + public async Task Import(ImportDbKnowledgeRequest request) { var sqlDriverSettings = _services.GetRequiredService(); - var knowledgeSettings = _services.GetRequiredService(); var knowledgeService = _services.GetRequiredService(); - var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + var provider = request.Provider ?? "openai"; + var model = request.Model ?? "gpt-4o"; + var schema = request.Schema; + var collectionName = request.KnowledgebaseCollection; var tables = new HashSet(); using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString); diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 2df8a124..60309dff 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -9,7 +9,7 @@ "isPublic": true, "profiles": [ "database" ], "llmConfig": { - "provider": "azure-openai", + "provider": "openai", "model": "gpt-4o-mini" }, "routingRules": [