diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 697f94a5..e94cf839 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -33,7 +33,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 7e7e372e..85f41e57 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -12,6 +12,7 @@ public interface IConversationService Task GetConversation(string id); Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); + Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request); Task> GetLastConversations(); Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds); Task DeleteConversations(IEnumerable ids); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/UpdateMessageRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/UpdateMessageRequest.cs new file mode 100644 index 00000000..66a33031 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/UpdateMessageRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class UpdateMessageRequest +{ + public DialogElement Message { get; set; } = null!; + public int InnderIndex { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs index 34b9f76d..a5b79a93 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs @@ -5,6 +5,6 @@ public class InstructResult : ITrackableMessage [JsonPropertyName("message_id")] public string MessageId { get; set; } public string Text { get; set; } - public object Data { get; set; } - public Dictionary States { get; set; } + public object? Data { get; set; } + public Dictionary? States { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ExtractedKnowledge.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ExtractedKnowledge.cs index 0043c437..eecd56d1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ExtractedKnowledge.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ExtractedKnowledge.cs @@ -7,4 +7,7 @@ public class ExtractedKnowledge [JsonPropertyName("answer")] public string Answer { get; set; } = string.Empty; + + [JsonPropertyName("refined_collection")] + public string RefinedCollection { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 62888acc..dd2648d8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -72,6 +72,7 @@ public interface IBotSharpRepository Conversation GetConversation(string conversationId); PagedItems GetConversations(ConversationFilter filter); void UpdateConversationTitle(string conversationId, string title); + bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request); void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint); ConversationBreakpoint? GetConversationBreakpoint(string conversationId); List GetLastConversations(); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 8511b873..4d3ed7af 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -50,6 +50,13 @@ public partial class ConversationService : IConversationService var conversation = db.GetConversation(id); return conversation; } + + public async Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request) + { + var db = _services.GetRequiredService(); + return db.UpdateConversationMessage(conversationId, request); + } + public async Task GetConversation(string id) { var db = _services.GetRequiredService(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index ae3ae408..8e647694 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -156,22 +156,25 @@ public class BotSharpDbContext : Database, IBotSharpRepository => throw new NotImplementedException(); public void AppendConversationDialogs(string conversationId, List dialogs) - => new NotImplementedException(); + => throw new NotImplementedException(); public void UpdateConversationTitle(string conversationId, string title) - => new NotImplementedException(); + => throw new NotImplementedException(); + + public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) + => throw new NotImplementedException(); public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) - => new NotImplementedException(); + => throw new NotImplementedException(); public ConversationBreakpoint? GetConversationBreakpoint(string conversationId) => throw new NotImplementedException(); public void UpdateConversationStates(string conversationId, List states) - => new NotImplementedException(); + => throw new NotImplementedException(); public void UpdateConversationStatus(string conversationId, string status) - => new NotImplementedException(); + => throw new NotImplementedException(); public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 0a3a1045..d61fda41 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Models; using System.Globalization; using System.IO; +using System.Xml.Linq; namespace BotSharp.Core.Repository { @@ -133,6 +134,38 @@ namespace BotSharp.Core.Repository } } + public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var dialogs = GetConversationDialogs(conversationId); + var candidates = dialogs.Where(x => x.MetaData.MessageId == request.Message.MetaData.MessageId + && x.MetaData.Role == request.Message.MetaData.Role).ToList(); + + var found = candidates.Where((_, idx) => idx == request.InnderIndex).FirstOrDefault(); + if (found == null) return false; + + found.Content = request.Message.Content; + found.RichContent = request.Message.RichContent; + + if (!string.IsNullOrEmpty(found.SecondaryContent)) + { + found.SecondaryContent = request.Message.Content; + } + + if (!string.IsNullOrEmpty(found.SecondaryRichContent)) + { + found.SecondaryRichContent = request.Message.RichContent; + } + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var dialogFile = Path.Combine(convDir, DIALOG_FILE); + File.WriteAllText(dialogFile, JsonSerializer.Serialize(dialogs, _options)); + return true; + } + public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) { var convDir = FindConversationDirectory(conversationId); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs index 1ae18e84..a0e5412c 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs @@ -13,9 +13,12 @@ public class FirstStagePlan [JsonPropertyName("step")] public int Step { get; set; } = -1; - [JsonPropertyName("need_additional_information")] + [JsonPropertyName("need_breakdown_task")] public bool ContainMultipleSteps { get; set; } = false; + [JsonPropertyName("need_lookup_dictionary")] + public bool NeedLookupDictionary { get; set; } = false; + [JsonPropertyName("related_tables")] public string[] Tables { get; set; } = new string[0]; diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 5dc16b85..61fdd758 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -560,6 +560,11 @@ public class UserService : IUserService return false; } + if ((record.UserName.Substring(0, 3) == "+86" || record.FirstName.Substring(0, 3) == "+86") && phone.Substring(0, 3) != "+86") + { + phone = $"+86{phone}"; + } + db.UpdateUserPhone(record.Id, phone); return true; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index a020a490..600c0dec 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -221,6 +221,29 @@ public class ConversationController : ControllerBase return response != null; } + [HttpPut("/conversation/{conversationId}/update-message")] + public async Task UpdateConversationMessage([FromRoute] string conversationId, [FromBody] UpdateMessageModel model) + { + var conversationService = _services.GetRequiredService(); + var request = new UpdateMessageRequest + { + Message = new DialogElement + { + MetaData = new DialogMetaData + { + MessageId = model.Message.MessageId, + Role = model.Message.Sender?.Role + }, + Content = model.Message.Text, + RichContent = JsonSerializer.Serialize(model.Message.RichContent, _jsonOptions), + }, + InnderIndex = model.InnerIndex + }; + + return await conversationService.UpdateConversationMessage(conversationId, request); + } + + [HttpDelete("/conversation/{conversationId}")] public async Task DeleteConversation([FromRoute] string conversationId) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs new file mode 100644 index 00000000..8ae28dfd --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/UpdateMessageModel.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class UpdateMessageModel +{ + [JsonPropertyName("message")] + public ChatResponseModel Message { get; set; } = null!; + + [JsonPropertyName("inner_index")] + public int InnerIndex { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs index 544cc9ff..393bbb86 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs @@ -5,11 +5,11 @@ namespace BotSharp.OpenAPI.ViewModels.Users; public class UserViewModel { - public string Id { get; set; } = null!; + public string Id { get; set; } = string.Empty; [JsonPropertyName("user_name")] - public string UserName { get; set; } = null!; + public string UserName { get; set; } = string.Empty; [JsonPropertyName("first_name")] - public string FirstName { get; set; } = null!; + public string FirstName { get; set; } = string.Empty; [JsonPropertyName("last_name")] public string? LastName { get; set; } public string? Email { get; set; } @@ -18,7 +18,7 @@ public class UserViewModel public string Role { get; set; } = UserRole.User; [JsonPropertyName("full_name")] public string FullName => $"{FirstName} {LastName}".Trim(); - public string Source { get; set; } + public string? Source { get; set; } [JsonPropertyName("external_id")] public string? ExternalId { get; set; } public string Avatar { get; set; } = "/user/avatar"; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index fd0f9cc7..abbe89dd 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -41,13 +41,22 @@ public class WelcomeHook : ConversationHookBase }); var richContentService = _services.GetRequiredService(); var messages = richContentService.ConvertToMessages(content); + var guid = Guid.NewGuid().ToString(); foreach (var message in messages) { var richContent = new RichContent(message); + var dialog = new RoleDialogModel(AgentRole.Assistant, message.Text) + { + MessageId = guid, + CurrentAgentId = agent.Id, + RichContent = richContent + }; + var json = JsonSerializer.Serialize(new ChatResponseModel() { ConversationId = conversation.Id, + MessageId = dialog.MessageId, Text = message.Text, RichContent = richContent, Sender = new UserViewModel() @@ -60,12 +69,7 @@ public class WelcomeHook : ConversationHookBase await Task.Delay(300); - _storage.Append(conversation.Id, new RoleDialogModel(AgentRole.Assistant, message.Text) - { - MessageId = conversation.Id, - CurrentAgentId = agent.Id, - RichContent = richContent - }); + _storage.Append(conversation.Id, dialog); await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageReceivedFromAssistant", json); } diff --git a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs index 9ca55a66..7ba8aaf4 100644 --- a/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs +++ b/src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs @@ -215,7 +215,7 @@ namespace BotSharp.Plugin.ExcelHandler.Services } private string CreateDBTableSqlString(string tableName, List headerColumns, List? columnTypes = null, bool isMemory = false) { - _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes; + _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(128)").ToList() : columnTypes; /*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase))) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 42663445..1cd8144f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -21,6 +21,7 @@ + @@ -37,6 +38,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/GenerateKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/GenerateKnowledgeFn.cs new file mode 100644 index 00000000..cd43fca1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/GenerateKnowledgeFn.cs @@ -0,0 +1,65 @@ +using BotSharp.Abstraction.Templating; +using BotSharp.Core.Infrastructures; + +namespace BotSharp.Plugin.KnowledgeBase.Functions; + +public class GenerateKnowledgeFn : IFunctionCallback +{ + public string Name => "generate_knowledge"; + + public string Indication => "generating knowledge"; + + private readonly IServiceProvider _services; + private readonly KnowledgeBaseSettings _settings; + + public GenerateKnowledgeFn(IServiceProvider services, KnowledgeBaseSettings settings) + { + _services = services; + _settings = settings; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); + var agentService = _services.GetRequiredService(); + var llmAgent = await agentService.GetAgent(BuiltInAgentId.Planner); + var generateKnowledgePrompt = await GetGenerateKnowledgePrompt(args.Question, args.Answer); + var agent = new Agent + { + Id = message.CurrentAgentId ?? string.Empty, + Name = "sqlDriver_DictionarySearch", + Instruction = generateKnowledgePrompt, + LlmConfig = llmAgent.LlmConfig + }; + var response = await GetAiResponse(agent); + message.Data = response.Content.JsonArrayContent(); + message.Content = response.Content; + return true; + } + + private async Task GetGenerateKnowledgePrompt(string userQuestions, string sqlAnswer) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + + var agent = await agentService.GetAgent(BuiltInAgentId.Learner); + var template = agent.Templates.FirstOrDefault(x => x.Name == "knowledge.generation")?.Content ?? string.Empty; + + return render.Render(template, new Dictionary + { + { "user_questions", userQuestions }, + { "sql_answer", sqlAnswer }, + }); + } + private async Task GetAiResponse(Agent agent) + { + var text = "Generate question and answer pair"; + var message = new RoleDialogModel(AgentRole.User, text); + + var completion = CompletionProvider.GetChatCompletion(_services, + provider: agent.LlmConfig.Provider, + model: agent.LlmConfig.Model); + + return await completion.GetChatCompletions(agent, new List { message }); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 0af59975..e432abc1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -19,7 +19,9 @@ public class MemorizeKnowledgeFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + var collectionName = !string.IsNullOrEmpty(args.RefinedCollection) + ? args.RefinedCollection + : _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; var knowledgeService = _services.GetRequiredService(); var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/templates/knowledge.generation.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/templates/knowledge.generation.liquid new file mode 100644 index 00000000..82afe7e6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/templates/knowledge.generation.liquid @@ -0,0 +1,12 @@ +You are a knowledge generator for knowledge base. Extract the answer in "SQL Answer" to answer the User Questions +Replace alias with the actual table name. Output json array only, formatting as [{"question":"string", "answer":"string/sql statement"}]. +Skip the question/answer for tmp table. +Don't include tmp table in the answer. + +===== +User Questions: +{{ user_questions }} + +===== +SQL Answer: +{{ sql_answer }} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 9aed7265..f710a237 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -108,6 +108,42 @@ public partial class MongoRepository _dc.Conversations.UpdateOne(filterConv, updateConv); } + public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var filter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundDialog = _dc.ConversationDialogs.Find(filter).FirstOrDefault(); + if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) + { + return false; + } + + var dialogs = foundDialog.Dialogs; + var candidates = dialogs.Where(x => x.MetaData.MessageId == request.Message.MetaData.MessageId + && x.MetaData.Role == request.Message.MetaData.Role).ToList(); + + var found = candidates.Where((_, idx) => idx == request.InnderIndex).FirstOrDefault(); + if (found == null) return false; + + found.Content = request.Message.Content; + found.RichContent = request.Message.RichContent; + + if (!string.IsNullOrEmpty(found.SecondaryContent)) + { + found.SecondaryContent = request.Message.Content; + } + + if (!string.IsNullOrEmpty(found.SecondaryRichContent)) + { + found.SecondaryRichContent = request.Message.RichContent; + } + + var update = Builders.Update.Set(x => x.Dialogs, dialogs); + _dc.ConversationDialogs.UpdateOne(filter, update); + return true; + } + public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) { if (string.IsNullOrEmpty(conversationId)) return; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index ba7c7047..e228d607 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,4 +1,5 @@ using BotSharp.Plugin.Planner.TwoStaging.Models; +using System.Threading.Tasks; namespace BotSharp.Plugin.Planner.Functions; @@ -27,14 +28,18 @@ public class SecondaryStagePlanFn : IFunctionCallback var planPrimary = states.GetState("planning_result"); var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); - - // Search knowledgebase - var knowledges = await knowledgeService.SearchVectorKnowledge(taskSecondary.SolutionQuestion, collectionName, new VectorSearchOptions - { - Confidence = 0.6f - }); - var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); + // Search knowledgebase + var hooks = _services.GetServices(); + var knowledges = new List(); + foreach (var hook in hooks) + { + var k = await hook.GetRelevantKnowledges(message, taskSecondary.SolutionQuestion); + knowledges.AddRange(k); + } + knowledges = knowledges.Distinct().ToList(); + + var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges); // Get second stage planning prompt var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index fd464e23..e9fd3b37 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Planning; using BotSharp.Plugin.Planner.TwoStaging; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.EntityFrameworkCore.Metadata.Internal; namespace BotSharp.Plugin.Planner.Functions; @@ -54,7 +53,7 @@ public class SummaryPlanFn : IFunctionCallback ddlStatements += "\r\n" + msgCopy.Content; // Summarize and generate query - var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult); + var summaryPlanPrompt = await GetSummaryPlanPrompt(msgCopy, taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult); _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}"); var plannerAgent = new Agent @@ -74,10 +73,11 @@ public class SummaryPlanFn : IFunctionCallback return true; } - private async Task GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult) + private async Task GetSummaryPlanPrompt(RoleDialogModel message, string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult) { var agentService = _services.GetRequiredService(); var render = _services.GetRequiredService(); + var knowledgeHooks = _services.GetServices(); var agent = await agentService.GetAgent(BuiltInAgentId.Planner); var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; @@ -89,10 +89,18 @@ public class SummaryPlanFn : IFunctionCallback additionalRequirements.Add(requirement); }); + var globalKnowledges = new List(); + foreach (var hook in knowledgeHooks) + { + var k = await hook.GetGlobalKnowledges(message); + globalKnowledges.AddRange(k); + } + return render.Render(template, new Dictionary { { "task_description", taskDescription }, { "summary_requirements", string.Join("\r\n", additionalRequirements) }, + { "global_knowledges", globalKnowledges }, { "relevant_knowledges", relevantKnowledge }, { "dictionary_items", dictionaryItems }, { "table_structure", ddlStatement }, diff --git a/src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerAgentHook.cs b/src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerAgentHook.cs index 1a85e1c0..503de3d5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerAgentHook.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Hooks/PlannerAgentHook.cs @@ -2,13 +2,32 @@ namespace BotSharp.Plugin.Planner.Hooks; public class PlannerAgentHook : AgentHookBase { - public override string SelfId => string.Empty; + public override string SelfId => BuiltInAgentId.Planner; public PlannerAgentHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { } + public override bool OnInstructionLoaded(string template, Dictionary dict) + { + var knowledgeHooks = _services.GetServices(); + + // Get global knowledges + var Knowledges = new List(); + foreach (var hook in knowledgeHooks) + { + var k = hook.GetGlobalKnowledges(new RoleDialogModel(AgentRole.User, template) + { + CurrentAgentId = BuiltInAgentId.Planner + }).Result; + Knowledges.AddRange(k); + } + dict["global_knowledges"] = Knowledges; + + return true; + } + public override void OnAgentLoaded(Agent agent) { var conv = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs index 754e5308..9588b811 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs @@ -11,9 +11,12 @@ public class FirstStagePlan [JsonPropertyName("step")] public int Step { get; set; } = -1; - [JsonPropertyName("need_additional_information")] + [JsonPropertyName("need_breakdown_task")] public bool NeedAdditionalInformation { get; set; } = false; + [JsonPropertyName("need_lookup_dictionary")] + public bool NeedLookupDictionary { get; set; } = false; + [JsonPropertyName("related_tables")] public string[] Tables { get; set; } = []; diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs index 9076a6b4..652d4e7d 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs @@ -5,6 +5,9 @@ public class SecondStagePlan [JsonPropertyName("related_tables")] public string[] Tables { get; set; } = []; + [JsonPropertyName("need_lookup_dictionary")] + public bool NeedLookupDictionary { get; set; } = false; + [JsonPropertyName("description")] public string Description { get; set; } = ""; diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 2f4c7a41..d15329a8 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,9 +1,13 @@ +The user is dealing with a complex problem, and you need to break this complex problem into several small tasks to more easily solve the user's needs. Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. + 1. Call plan_primary_stage to generate the primary plan. -2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. -3. Repeat step 2 until you processed all the primary stages. -4. If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name. + If you've already got the plan to meet the user goal, directly go to step 5. +2. If need_lookup_dictionary is True, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name. If you no items retured, you can pull all the list and find the match. + If need_lookup_dictionary is False, skip calling verify_dictionary_term. +3. If need_breakdown_task is true, call plan_secondary_stage for the specific primary stage. +4. Repeat step 3 until you processed all the primary stages. 5. You must call plan_summary for you final planned output. *** IMPORTANT *** @@ -13,6 +17,7 @@ Don't run the planning process repeatedly if you have already got the result of {% if global_knowledges != empty -%} ===== Global Knowledge: +Current date time is: {{ "now" | date: "%Y-%m-%d %H:%M" }} {% for k in global_knowledges %} {{ k }} {% endfor %} diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index f6e8f557..656a6fe4 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -4,11 +4,12 @@ Thinking process: 1. Reference to "Task Knowledge" if there is relevant knowledge; 2. Breakdown task into subtasks. - The subtask should contain all needed parameters for subsequent steps. - - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true. - - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. - - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true. - - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_additional_information to true. + - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_breakdown_task to true. + - If there is extra knowledge or relationship needed between steps, set the need_breakdown_task to true for both steps. + - If the solution mentioned "related solutions" is needed, set the need_breakdown_task to true. + - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_breakdown_task to true. + - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_lookup_dictionary to true. + - Seperate the dictionary lookup and need additional information/knowledge into different subtask. 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@'; 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}] 5. You can NOT generate the final query before calling function plan_summary. diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index d4bd15b5..81e22fe8 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -3,10 +3,9 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown * The parameters can be extracted from the original task. * You need to list all the steps in detail. Finding relationships should also be a step. * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly. -* If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code. +* If need_lookup_dictionary is true, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name/code. * Output all the steps as much detail as possible in JSON: [{{ response_format }}] - Additional Requirements: * "output_results" is variable name that needed to be used in the next step. diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid index b4b28e0e..6b901eb8 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid @@ -7,6 +7,10 @@ Requirements: Task description: {{ task_description }} +===== +Global Knowledges: +{{ global_knowledges }} + ===== Relevant Knowledges: {{ relevant_knowledges }} diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid index 31aca793..1b2e95bd 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid @@ -1,2 +1 @@ -For every primary step, if need_additional_information is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step. -if need_lookup_dictionary is true, you have to call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code. \ No newline at end of file +For every primary step, if need_breakdown_task is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 4f93597a..d4d0a243 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -30,6 +30,7 @@ + @@ -69,6 +70,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 78ab8541..e3d8e57e 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -1,8 +1,10 @@ using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.SqlDriver.Models; using Dapper; using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; using MySqlConnector; namespace BotSharp.Plugin.SqlDriver.Functions; @@ -13,31 +15,47 @@ public class ExecuteQueryFn : IFunctionCallback public string Indication => "Performing data retrieval operation."; private readonly SqlDriverSetting _setting; private readonly IServiceProvider _services; + private readonly ILogger _logger; - public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting) + public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting, ILogger logger) { _services = services; _setting = setting; + _logger = logger; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - var settings = _services.GetRequiredService(); - var results = settings.DatabaseType switch - { - "MySql" => RunQueryInMySql(args.SqlStatements), - "SqlServer" => RunQueryInSqlServer(args.SqlStatements), - _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") - }; - - if (results.Count() == 0) - { - message.Content = "No record found"; - return true; - } - message.Content = JsonSerializer.Serialize(results); + var refinedArgs = await RefineSqlStatement(message, args); + + var settings = _services.GetRequiredService(); + + try + { + var results = settings.DatabaseType switch + { + "MySql" => RunQueryInMySql(refinedArgs.SqlStatements), + "SqlServer" => RunQueryInSqlServer(refinedArgs.SqlStatements), + _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") + }; + + if (results.Count() == 0) + { + message.Content = "No record found"; + return true; + } + + message.Content = JsonSerializer.Serialize(results); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error occurred while executing SQL query."); + message.Content = "Error occurred while retrieving information."; + message.StopCompletion = true; + return false; + } if (args.FormattingResult) { @@ -59,6 +77,7 @@ public class ExecuteQueryFn : IFunctionCallback }); message.Content = result.Content; + message.StopCompletion = true; } return true; @@ -77,4 +96,54 @@ public class ExecuteQueryFn : IFunctionCallback using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString); return connection.Query(string.Join("\r\n", sqlTexts)); } + + private async Task RefineSqlStatement(RoleDialogModel message, ExecuteQueryArgs args) + { + // get table DDL + var fn = _services.GetRequiredService(); + var msg = RoleDialogModel.From(message); + await fn.InvokeFunction("sql_table_definition", msg); + + // refine SQL + var agentService = _services.GetRequiredService(); + var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); + var dictionarySqlPrompt = await GetDictionarySQLPrompt(string.Join("\r\n\r\n", args.SqlStatements), msg.Content); + var agent = new Agent + { + Id = message.CurrentAgentId ?? string.Empty, + Name = "sqlDriver_ExecuteQuery", + Instruction = dictionarySqlPrompt, + TemplateDict = new Dictionary(), + LlmConfig = currentAgent.LlmConfig + }; + + var completion = CompletionProvider.GetChatCompletion(_services, + provider: agent.LlmConfig.Provider, + model: agent.LlmConfig.Model); + + var refinedMessage = await completion.GetChatCompletions(agent, new List + { + new RoleDialogModel(AgentRole.User, "Check and output the correct SQL statements") + }); + + return refinedMessage.Content.JsonContent(); + } + + private async Task GetDictionarySQLPrompt(string originalSql, string tableStructure) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var knowledgeHooks = _services.GetServices(); + + var agent = await agentService.GetAgent(BuiltInAgentId.SqlDriver); + var template = agent.Templates.FirstOrDefault(x => x.Name == "sql_statement_correctness")?.Content ?? string.Empty; + var responseFormat = JsonSerializer.Serialize(new ExecuteQueryArgs { }); + + return render.Render(template, new Dictionary + { + { "original_sql", originalSql }, + { "table_structure", tableStructure }, + { "response_format", responseFormat } + }); + } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs index ed1d9d4b..c94c12db 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs @@ -9,7 +9,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions; public class LookupDictionaryFn : IFunctionCallback { - public string Name => "sql_dictionary_lookup"; + public string Name => "verify_dictionary_term"; private readonly IServiceProvider _services; public LookupDictionaryFn(IServiceProvider services) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs index edaa0cf3..27af8e64 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs @@ -10,7 +10,7 @@ public class SqlDictionaryLookupHook : AgentHookBase, IAgentHook private const string SQL_EXECUTOR_TEMPLATE = "sql_dictionary_lookup.fn"; private IEnumerable _targetSqlExecutorFunctions = new List { - "sql_dictionary_lookup", + "verify_dictionary_term", }; public override string SelfId => BuiltInAgentId.Planner; diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index ac99c699..29160780 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -33,20 +33,25 @@ public class SqlDriverPlanningHook : IPlanningHook var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); wholeDialogs.Add(RoleDialogModel.From(msg)); - wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query")); - - var agent = await _services.GetRequiredService().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f"); + wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, $"call execute_sql to run query, set formatting_result as {settings.FormattingResult}")); + var agent = await _services.GetRequiredService().LoadAgent(BuiltInAgentId.SqlDriver); var completion = CompletionProvider.GetChatCompletion(_services, provider: agent.LlmConfig.Provider, model: agent.LlmConfig.Model); var response = await completion.GetChatCompletions(agent, wholeDialogs); + + // Invoke "execute_sql" var routing = _services.GetRequiredService(); await routing.InvokeFunction(response.FunctionName, response); msg.CurrentAgentId = agent.Id; msg.FunctionName = response.FunctionName; msg.FunctionArgs = response.FunctionArgs; msg.Content = response.Content; + msg.StopCompletion = response.StopCompletion; + + /*var routing = _services.GetRequiredService(); + await routing.InvokeAgent(BuiltInAgentId.SqlDriver, wholeDialogs);*/ } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs index ac2279ea..bc5c6fbd 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs @@ -7,8 +7,12 @@ public class ExecuteQueryArgs [JsonPropertyName("sql_statements")] public string[] SqlStatements { get; set; } = []; + [JsonPropertyName("tables")] + public string[] Tables { get; set; } = []; + /// /// Beautifying query result /// + [JsonPropertyName("formatting_result")] public bool FormattingResult { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs index 5815151c..3a4095a6 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs @@ -10,4 +10,5 @@ public class SqlDriverSetting public string SqlServerExecutionConnectionString { get; set; } = null!; public string SqlLiteConnectionString { get; set; } = null!; public bool ExecuteSqlSelectAutonomous { get; set; } = false; + public bool FormattingResult { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json index 66abe705..1be003a4 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json @@ -1,5 +1,5 @@ { - "name": "sql_dictionary_lookup", + "name": "verify_dictionary_term", "description": "Get id from dictionary table by keyword if tool or solution mentioned this approach", "parameters": { "type": "object", @@ -10,7 +10,7 @@ }, "reason": { "type": "string", - "description": "the reason why you need to call sql_dictionary_lookup" + "description": "the reason why you need to call verify_dictionary_term" }, "tables": { "type": "array", diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid index 73dcb58d..f5ceebdb 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid @@ -1,8 +1,11 @@ -Dictionary Lookup Rules: +Dictionary Verification Rules: ===== -Please call function sql_dictionary_lookup if user wants to get or retrieve dictionary/enum/term from data tables. -You must return the id and name/code. The table name must come from the planning in conversation. +1. The table name must come from the planning in conversation. +2. You must return the id and name/code. You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules. -Dictionary table pattern is table name starting with "data_". You can only query the dictionary table without join other non-dictionary tables. + +The dictionary table is identified by a name that begins with "data_". You are only allowed to query the dictionary table without joining it with other non-dictionary tables. + +IMPORTANT: Don't generate insert SQL. ===== \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 60309dff..eb79816f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -10,7 +10,7 @@ "profiles": [ "database" ], "llmConfig": { "provider": "openai", - "model": "gpt-4o-mini" + "model": "gpt-4o" }, "routingRules": [ { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json index 15e6d281..9cdafc04 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json @@ -11,8 +11,22 @@ "type": "string", "description": "sql statement" } + }, + + "formatting_result": { + "type": "boolean", + "description": "formatting the results" + }, + + "tables": { + "type": "array", + "description": "all related tables", + "items": { + "type": "string", + "description": "table name" + } } }, - "required": [ "sql_statement" ] + "required": [ "sql_statement", "tables", "formatting_result" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid index 7c40b11a..5d07a941 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/query_result_formatting.liquid @@ -1 +1,5 @@ -Output in human readable format. If there is large amount of information, shape it in tabular. \ No newline at end of file +Output in human readable format. If there is large amount of rows, shape it in tabular, otherwise, output in plain text. +Put user task description in the first line in the same language, for example, user is using Chinese, you have to output the result in Chinese. + +User Task Description: +{{ requirement_detail }} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid new file mode 100644 index 00000000..cf61eb1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_statement_correctness.liquid @@ -0,0 +1,11 @@ +You are a sql statement corrector. You will need to refer to the table structure and rewrite the original sql statement so it's using the correct information, e.g. column name. +Output the sql statement only without comments, in JSON format: {{ response_format }} +Make sure all the column names are defined in the Table Structure. + +===== +Original SQL statements: +{{ original_sql }} + +===== +Table Structure: +{{ table_structure }}