diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs index b98f62e4..66fdac4e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -16,7 +16,7 @@ public class FunctionCallFromLlm : RoutingArgs public bool ExecutingDirectly { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.Always)] - public bool HideDialogContext { get; set; } + public bool HandleDialogsByPlanner { get; set; } /// /// Router routed to a wrong agent. diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs new file mode 100644 index 00000000..4d94a52a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Knowledges.Models; + +namespace BotSharp.Abstraction.Knowledges; + +public interface IKnowledgeHook +{ + Task> CollectChunkedKnowledge(); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 4f960fee..5b2b795b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -4,6 +4,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + Task> CollectChunkedKnowledge(); + Task EmbedKnowledge(List chunks); + Task Feed(KnowledgeFeedModel knowledge); Task EmbedKnowledge(KnowledgeCreationModel knowledge); Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeChunk.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeChunk.cs new file mode 100644 index 00000000..6a049d77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeChunk.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeChunk +{ + public string Id { get; set; } + public string Name { get; set; } + public string Content { get; set; } + public string SourceAgentId { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs index 0e8bf803..dc5b2ba2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Abstraction.Routing.Planning; @@ -10,7 +9,11 @@ namespace BotSharp.Abstraction.Routing.Planning; public interface IPlaner { Task GetNextInstruction(Agent router, string messageId, List dialogs); - Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); - Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); + Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs); + Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs); + List BeforeHandleContext(FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) + => dialogs; + bool AfterHandleContext(List dialogs, List taskAgentDialogs) + => true; int MaxLoopCount => 5; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index 2a1b5e9b..ccb4f274 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -67,7 +67,7 @@ public class HFPlanner : IPlaner return inst; } - public async Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message) + public async Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) { if (!string.IsNullOrEmpty(inst.AgentName)) { @@ -82,7 +82,7 @@ public class HFPlanner : IPlaner return true; } - public async Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message) + public async Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) { var context = _services.GetRequiredService(); context.Empty(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs index fc445a06..91f35801 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs @@ -76,7 +76,7 @@ public class NaivePlanner : IPlaner return inst; } - public async Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message) + public async Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) { // Set user content as Planner's question message.FunctionName = inst.Function; @@ -85,7 +85,7 @@ public class NaivePlanner : IPlaner return true; } - public async Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message) + public async Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) { var context = _services.GetRequiredService(); if (inst.UnmatchedAgent) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs index 7ecd27b6..84246d40 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs @@ -1,3 +1,4 @@ +using Amazon.SecurityToken.Model.Internal.MarshallTransformations; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.MLTasks; @@ -102,14 +103,33 @@ public class SequentialPlanner : IPlaner { inst.Response = decomposation.Description; inst.Reason = $"{decomposation.TotalRemainingSteps} steps left."; - inst.HideDialogContext = true; + inst.HandleDialogsByPlanner = true; } _lastInst = inst; return inst; } - public async Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message) + public List BeforeHandleContext(FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) + { + var taskAgentDialogs = new List + { + new RoleDialogModel(AgentRole.User, inst.Response) + { + MessageId = message.MessageId, + } + }; + + return taskAgentDialogs; + } + + public bool AfterHandleContext(List dialogs, List taskAgentDialogs) + { + dialogs.AddRange(taskAgentDialogs.Skip(1)); + return true; + } + + public async Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) { // Set user content as Planner's question message.FunctionName = inst.Function; @@ -118,7 +138,7 @@ public class SequentialPlanner : IPlaner return true; } - public async Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message) + public async Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message, List dialogs) { var context = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs index 677157d6..586a9943 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; +using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Routing.Settings; using BotSharp.Abstraction.Settings; using BotSharp.Core.Routing.Hooks; @@ -34,8 +35,8 @@ public class RoutingPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs index 2d47f1ab..9522d733 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs @@ -9,13 +9,17 @@ public partial class RoutingService { public IPlaner GetPlanner(Agent router) { - var planner = router.RoutingRules.FirstOrDefault(x => x.Type == RuleType.Planner); + var rule = router.RoutingRules.FirstOrDefault(x => x.Type == RuleType.Planner); - if (planner?.Field == nameof(HFPlanner)) - return _services.GetRequiredService(); - else if (planner?.Field == nameof(SequentialPlanner)) - return _services.GetRequiredService(); - else + var planner = _services.GetServices(). + FirstOrDefault(x => x.GetType().Name.EndsWith(rule.Field)); + + if (planner == null) + { + _logger.LogError($"Can't find specific planner named {rule.Field}"); return _services.GetRequiredService(); + } + + return planner; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index b71a0a29..793793b4 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -106,24 +106,21 @@ public partial class RoutingService : IRoutingService #else _logger.LogInformation($"*** Next Instruction *** {inst}"); #endif - await planner.AgentExecuting(_router, inst, message); + await planner.AgentExecuting(_router, inst, message, dialogs); // Handover to Task Agent - if (inst.HideDialogContext) + if (inst.HandleDialogsByPlanner) { - var dialogWithoutContext = new List - { - new RoleDialogModel(AgentRole.User, inst.Response) - }; + var dialogWithoutContext = planner.BeforeHandleContext(inst, message, dialogs); response = await executor.Execute(this, inst, message, dialogWithoutContext); - dialogs.AddRange(dialogWithoutContext.Skip(1)); + planner.AfterHandleContext(dialogs, dialogWithoutContext); } else { response = await executor.Execute(this, inst, message, dialogs); } - await planner.AgentExecuted(_router, inst, response); + await planner.AgentExecuted(_router, inst, response, dialogs); } return response; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 7e346b75..89f26f45 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; -using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -17,6 +16,13 @@ public class KnowledgeBaseController : ControllerBase _services = services; } + [HttpPost("/knowledge-base/embed")] + public async Task EmbedKnowledge() + { + var chunks = await _knowledgeService.CollectChunkedKnowledge(); + await _knowledgeService.EmbedKnowledge(chunks); + } + [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index bb24ad19..dc1700e2 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -17,25 +17,7 @@ - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - + diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs deleted file mode 100644 index 68d5bd25..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/SearchKnowledgesFn.cs +++ /dev/null @@ -1,34 +0,0 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Plugin.KnowledgeBase.LlmContexts; - -namespace BotSharp.Plugin.KnowledgeBase.Functions; - -public class SearchKnowledgesFn : IFunctionCallback -{ - public string Name => "search_knowledges"; - private readonly IServiceProvider _services; - - public SearchKnowledgesFn(IServiceProvider services) - { - _services = services; - } - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - - var knowledgeService = _services.GetRequiredService(); - var knowledge = await knowledgeService.GetKnowledges(new KnowledgeRetrievalModel - { - AgentId = message.CurrentAgentId, - Question = args.Question - }); - - if (string.IsNullOrEmpty(knowledge)) - { - message.Content = "Can't find any relevant data in local knowledge base."; - } - - return true; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 70cddede..60fb0e8a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -10,7 +10,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin public string Name => "Knowledge Base"; public string Description => "Embedding private data and feed them into LLM in the conversation."; public string IconUrl => "https://cdn-icons-png.flaticon.com/512/9592/9592995.png"; - public string[] AgentIds => new[] { "f5679799-ba89-4fef-936a-bcc311e5f14d" }; public void RegisterDI(IServiceCollection services, IConfiguration config) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs deleted file mode 100644 index a5e3f84b..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/LlmContexts/KnowledgeContextIn.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Plugin.KnowledgeBase.LlmContexts; - -public class KnowledgeContextIn -{ - [JsonPropertyName("question")] - public string Question { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 4c40ea0c..5f545078 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; -public class KnowledgeService : IKnowledgeService +public partial class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs new file mode 100644 index 00000000..eccfed78 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> CollectChunkedKnowledge() + { + throw new NotImplementedException(); + } + + public async Task EmbedKnowledge(List chunks) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/agent.json b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/agent.json deleted file mode 100644 index 9eb33118..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/agent.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "Knowledge Base", - "description": "Local knowledge base, providing relevant answers or background knowledge to help handle user questions.", - "createdDateTime": "2024-01-02T00:00:00Z", - "updatedDateTime": "2024-01-02T00:00:00Z", - "id": "f5679799-ba89-4fef-936a-bcc311e5f14d", - "allowRouting": true, - "isPublic": true - } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/functions.json b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/functions.json deleted file mode 100644 index 5adc8d82..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/functions.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "name": "search_knowledges", - "description": "Retrieve relevant knowledges", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "User question" - } - }, - "required": ["question"] - } - } -] \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/instruction.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/instruction.liquid deleted file mode 100644 index 2bdfa8be..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/f5679799-ba89-4fef-936a-bcc311e5f14d/instruction.liquid +++ /dev/null @@ -1 +0,0 @@ -You are a local knowledge base that retrieves the most relevant answers to certain questions. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index b7b43e17..fcc88874 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -35,6 +35,11 @@ + + + + + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Actions/ExecuteQueryAction.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs similarity index 73% rename from src/Plugins/BotSharp.Plugin.SqlDriver/Actions/ExecuteQueryAction.cs rename to src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 90807e79..1d2226db 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Actions/ExecuteQueryAction.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -7,23 +7,22 @@ using MySqlConnector; using System.Text.Json; using System.Threading.Tasks; -namespace BotSharp.Plugin.SqlDriver.Actions; +namespace BotSharp.Plugin.SqlDriver.Functions; -public class ExecuteQueryAction : IFunctionCallback +public class ExecuteQueryFn : IFunctionCallback { public string Name => "execute_sql"; private readonly SqlDriverSetting _setting; - public ExecuteQueryAction(SqlDriverSetting setting) + public ExecuteQueryFn(SqlDriverSetting setting) { _setting = setting; } public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs); - message.Content = "executed successully"; + message.Content = "Executed"; /*using var connection = new MySqlConnection(_setting.MySqlConnectionString); message.Content = JsonSerializer.Serialize(connection.Query(args.SqlStatement), new JsonSerializerOptions { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableColumnsFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableColumnsFn.cs new file mode 100644 index 00000000..07a7a765 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableColumnsFn.cs @@ -0,0 +1,38 @@ +using BotSharp.Abstraction.Repositories; +using BotSharp.Plugin.SqlDriver.Models; +using System.IO; + +namespace BotSharp.Plugin.SqlDriver.Functions; + +public class GetTableColumnsFn : IFunctionCallback +{ + public string Name => "get_table_columns"; + private readonly IServiceProvider _services; + + public GetTableColumnsFn(IServiceProvider services) + { + _services = services; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + message.Content = $"Success. Columns of table '{args.Table}':\r\n\r\n"; + + var dbSettings = _services.GetRequiredService(); + var dir = Path.Combine(dbSettings.FileRepository, "agents", "ec46f15b-8790-400f-a37f-1e7995b7d6e2", "schemas"); + + // Search related document by message.Content + args.Description + var files = Directory.GetFiles(dir); + foreach (var file in files) + { + var fileName = file.Split(Path.DirectorySeparatorChar).Last(); + if (fileName.Split('.').First() == args.Table) + { + message.Content += File.ReadAllText(file); + break; + } + } + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlInsertFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlInsertFn.cs new file mode 100644 index 00000000..00d3eada --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlInsertFn.cs @@ -0,0 +1,38 @@ +using BotSharp.Plugin.SqlDriver.Models; + +namespace BotSharp.Plugin.SqlDriver.Functions; + +public class SqlInsertFn : IFunctionCallback +{ + public string Name => "sql_insert"; + private readonly IServiceProvider _services; + + public SqlInsertFn(IServiceProvider services) + { + _services = services; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + var sqlDriver = _services.GetRequiredService(); + if (sqlDriver.Statements.Exists(x => x.Statement == args.Statement)) + { + message.Content = "Skipped duplicated statement."; + return false; + } + sqlDriver.Enqueue(args); + message.Content = $"Inserted new record {JsonSerializer.Serialize(args.Parameters)} successfully"; + if (args.Return != null) + { + /*sqlDriver.Enqueue(new SqlStatement + { + Statement = $"SELECT LAST_INSERT_ID() INTO @{args.Return.Alias};", + Reason = $"select auto-incremented id into '{args.Return.Alias}'" + });*/ + message.Content += $" The {args.Return.Name} is saved to @{args.Return.Alias}."; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs new file mode 100644 index 00000000..c845a8d1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs @@ -0,0 +1,53 @@ +using BotSharp.Plugin.SqlDriver.Models; +using MySqlConnector; +using static Dapper.SqlMapper; + +namespace BotSharp.Plugin.SqlDriver.Functions; + +public class SqlSelect : IFunctionCallback +{ + public string Name => "sql_select"; + private readonly IServiceProvider _services; + + public SqlSelect(IServiceProvider services) + { + _services = services; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + // check if need to instantely + var execNow = !args.Parameters.Any(x => x.Value.StartsWith("@")); + if (execNow) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + var dictionary = new Dictionary(); + foreach(var p in args.Parameters) + { + dictionary["@" + p.Name] = p.Value; + } + var result = connection.QueryFirst(args.Statement, dictionary); + + if (args.IsCheckExistence) + { + message.Content = result == null ? + $"The record does not exist" : + $"The record already exists"; + } + else + { + message.Content = $"Retrieved result is {result} ({args.Reason})"; + } + } + else + { + var sqlDriver = _services.GetRequiredService(); + sqlDriver.Enqueue(args); + message.Content = $"Success."; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverContentGeneratingHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverContentGeneratingHook.cs new file mode 100644 index 00000000..82ac6717 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverContentGeneratingHook.cs @@ -0,0 +1,32 @@ +using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.Repositories; +using System.IO; + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverContentGeneratingHook : IContentGeneratingHook +{ + private readonly IServiceProvider _services; + public SqlDriverContentGeneratingHook(IServiceProvider services) + { + _services = services; + } + + /// + /// Inject useful variables generated by previous SQL query. + /// + /// + /// + /// + public async Task BeforeGenerating(Agent agent, List conversations) + { + if (agent.Id != "beda4c12-e1ec-4b4b-b328-3df4a6687c4f") + { + return; + } + + var sqlDriver = _services.GetRequiredService(); + agent.TemplateDict["return_variables"] = sqlDriver.Statements.Select(x => x.Return.Alias).ToArray(); + await Task.CompletedTask; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverKnowledgeHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverKnowledgeHook.cs new file mode 100644 index 00000000..a6ab9d04 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverKnowledgeHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDriverKnowledgeHook : IKnowledgeHook +{ + public async Task> CollectChunkedKnowledge() + { + return new List(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/GetTableColumnsArgs.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/GetTableColumnsArgs.cs new file mode 100644 index 00000000..f0d42fab --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/GetTableColumnsArgs.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class GetTableColumnsArgs +{ + [JsonPropertyName("table")] + public string Table { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LlmInputArgs.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LlmInputArgs.cs deleted file mode 100644 index e17d57de..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LlmInputArgs.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Plugin.SqlDriver.Models; - -public class LlmInputArgs -{ - [JsonPropertyName("sql_statement")] - public string SqlStatement { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlParamater.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlParamater.cs new file mode 100644 index 00000000..2b1661eb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlParamater.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class SqlParamater +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } + + public override string ToString() + { + return $"{Name}: {Value}"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlReturn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlReturn.cs new file mode 100644 index 00000000..248c40cc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlReturn.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class SqlReturn +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("alias")] + public string Alias { get; set; } + + public override string ToString() + { + return $"{Alias} - {Name}"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs new file mode 100644 index 00000000..739faf23 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs @@ -0,0 +1,29 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class SqlStatement +{ + [JsonPropertyName("sql_statement")] + public string Statement { get; set; } + + [JsonPropertyName("reason")] + public string Reason { get; set; } + + [JsonPropertyName("table")] + public string Table { get; set; } + + [JsonPropertyName("is_check_existence")] + public bool IsCheckExistence { get; set; } + + [JsonPropertyName("parameters")] + public SqlParamater[] Parameters { get; set; } = new SqlParamater[0]; + + [JsonPropertyName("return_field")] + public SqlReturn Return { get; set; } + + public override string ToString() + { + return $"{Statement}\t {string.Join(", ", Parameters.Select(x => x.Name + ": " + x.Value))}"; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/SqlDriverService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/SqlDriverService.cs new file mode 100644 index 00000000..9fdca5b0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/SqlDriverService.cs @@ -0,0 +1,43 @@ +using BotSharp.Plugin.SqlDriver.Models; + +namespace BotSharp.Plugin.SqlDriver.Services; + +public class SqlDriverService +{ + private readonly IServiceProvider _services; + public List Statements => _statements; + private static List _statements = new List(); + + public SqlDriverService(IServiceProvider services) + { + _services = services; + } + + public void Enqueue(SqlStatement statement) + { + var state = _services.GetRequiredService(); + + _statements.Add(statement); + + foreach (var sql in _statements) + { + Console.WriteLine(); + + Console.Write($"Reason: "); + Console.WriteLine($"{sql.Reason}", Color.Green); + + Console.Write($"Statement: "); + Console.WriteLine(sql.Statement, Color.Green); + foreach (var p in sql.Parameters) + { + Console.Write($"@{p.Name}: "); + Console.WriteLine($"{p.Value}", Color.Green); + } + if (sql.Return != null) + { + Console.Write($"Return: "); + Console.WriteLine($"{sql.Return.Name} as @{sql.Return.Alias}", Color.Green); + } + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index 963304a3..f056bb3d 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Plugins; -using BotSharp.Plugin.SqlHero.Settings; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using System; -using System.Drawing; -using System.Text.RegularExpressions; +using BotSharp.Abstraction.Loggers; namespace BotSharp.Plugin.SqlDriver; @@ -12,16 +6,19 @@ public class SqlDriverPlugin : IBotSharpPlugin { public string Id => "da7b6f7a-b1f0-455a-9939-ad2d493e929e"; public string Name => "SQL Driver"; - public string Description => "Convert the requirements into corresponding SQL statements and execute if needed"; + public string Description => "Convert the user requirements into corresponding SQL statements"; + public string IconUrl => "https://uxwing.com/wp-content/themes/uxwing/download/file-and-folder-type/sql-icon.png"; public void RegisterDI(IServiceCollection services, IConfiguration config) { - var settings = new SqlDriverSetting(); - config.Bind("SqlDriver", settings); - services.AddSingleton(x => + services.AddScoped(provider => { - Console.WriteLine($"Loaded SqlHero settings:: {Regex.Replace(settings.MySqlConnectionString, "password=.*?;", "password=******;")}", Color.Yellow); - return settings; + var settingService = provider.GetRequiredService(); + return settingService.Bind("SqlDriver"); }); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs new file mode 100644 index 00000000..31780fb4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs @@ -0,0 +1,24 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using System.Text.Json; +global using BotSharp.Abstraction.Conversations.Models; +global using Microsoft.Extensions.Configuration; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Templating; +global using Microsoft.Extensions.DependencyInjection; +global using System.Linq; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Knowledges; +global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Settings; +global using BotSharp.Plugin.SqlDriver.Hooks; +global using BotSharp.Plugin.SqlDriver.Services; +global using BotSharp.Plugin.SqlHero.Settings; +global using System.Drawing; +global using Console = Colorful.Console; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 73b5151f..80f66ba9 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -1,14 +1,14 @@ { "id": "beda4c12-e1ec-4b4b-b328-3df4a6687c4f", "name": "SQL Driver", - "description": "Convert the requirements into corresponding SQL statements according to the table structure and execute them if needed.", + "description": "Convert the requirements into corresponding SQL statements according to the table structure.", "type": "task", "createdDateTime": "2023-11-15T13:49:00Z", "updatedDateTime": "2023-11-15T13:49:00Z", "disabled": false, - "isPublic": false, + "isPublic": true, "profiles": [ "tool", "sql" ], "llmConfig": { - "max_recursion_depth": 1 + "max_recursion_depth": 10 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions.json index e09e23a1..0ab52f8d 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions.json @@ -1,14 +1,106 @@ -[{ - "name": "execute_sql", - "description": "generate sql statement and execute the query.", +[ + { + "name": "sql_insert", + "description": "Insert query is generated if the record doesn't exist.", "parameters": { "type": "object", "properties": { "sql_statement": { - "type": "string", - "description": "SQL statement" + "type": "string", + "description": "INSERT SQL statement. The value should use the parameter name like @field." + }, + "reason": { + "type": "string", + "description": "reason" + }, + "parameters": { + "type": "array", + "description": "parameters for the sql", + "items": { + "type": "object", + "description": "the name and value for the parameter", + "properties": { + "name": { + "type": "string", + "description": "field name" + }, + "value": { + "type": "string", + "description": "real value inferred by the context" + } + } } + }, + "return_field": { + "type": "object", + "description": "the name and alias for the return field", + "properties": { + "name": { + "type": "string", + "description": "field name" + }, + "alias": { + "type": "string", + "description": "meaningful field alias" + } + } + } }, - "required": ["sql_statement"] + "required": [ "sql_statement", "reason", "parameters", "return_field" ] } -}] \ No newline at end of file + }, + { + "name": "sql_select", + "description": "Get the specific value from table", + "parameters": { + "type": "object", + "properties": { + "sql_statement": { + "type": "string", + "description": "SQL statement with SELECT" + }, + "reason": { + "type": "string", + "description": "reason" + }, + "is_check_existence": { + "type": "boolean", + "description": "check record existence" + }, + "parameters": { + "type": "array", + "description": "data criteria for the query", + "items": { + "type": "object", + "description": "the name and value for the parameter", + "properties": { + "name": { + "type": "string", + "description": "field name" + }, + "value": { + "type": "string", + "description": "real value inferred by the context" + } + } + } + }, + "return_field": { + "type": "object", + "description": "the name and alias for the return field", + "properties": { + "name": { + "type": "string", + "description": "field in the table" + }, + "alias": { + "type": "string", + "description": "meaningful field alias" + } + } + } + }, + "required": [ "sql_statement", "reason", "is_check_existence", "parameters", "return_field" ] + } + } +] \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid index fca8fa07..20c74002 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid @@ -1 +1,22 @@ -You are a SQL Driver who knows how to convert human language to SQL statements. \ No newline at end of file +You're a SQL driver who knows how to translate text into SQL query. +Analyze the user requirement, think step by step, breakdown complex task into multiple steps. + +Your response must meet below requirements: +* DO NOT generate duplicated sql statements; +* The return field alias should be meaningful, it can be similar name of reference table column; +* Double check if the fields in the SQL query are correct; +* Use "Unique Index" to help check record existence; + +{% if return_variables and return_variables != empty -%} +===== +Below variables can be used by subsequent SQL: +{% for v in return_variables %} +- @{{ v }} +{% endfor %} +{%- endif %} + +{% if table_definition -%} +===== +Related table {{ related_table }} definition: +{{ table_definition }} +{%- endif %}