From e21a44ec3d9b6fcd5105634bc07e982dfd8a9e72 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 3 Sep 2024 15:41:12 -0500 Subject: [PATCH] refine db knowledge --- .../BotSharp.Logger/Hooks/VerboseLogHook.cs | 11 +- .../Controllers/KnowledgeBaseController.cs | 1 + .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../BotSharp.Plugin.SqlDriver.csproj | 3 +- .../Controllers/SqlDriverController.cs | 24 +++ .../Functions/GetTableDefinitionFn.cs | 31 ++-- .../Models/RequestBase.cs | 19 +++ .../Services/DbKnowledgeService.cs | 140 ++++++++++++++++++ .../SqlDriverPlugin.cs | 1 + 10 files changed, 221 insertions(+), 21 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index cf6985e9..afb8dd01 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -23,10 +23,13 @@ public class VerboseLogHook : IContentGeneratingHook { if (!_convSettings.ShowVerboseLog) return; - var dialog = conversations.Last(); - var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; - _logger.LogInformation(log); - + var dialog = conversations.LastOrDefault(); + if (dialog != null) + { + var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; + _logger.LogInformation(log); + } + await Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 260dc166..827c2d7c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -142,6 +142,7 @@ public class KnowledgeBaseController : ControllerBase } #endregion + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index af82b037..678ee82b 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = value.FunctionCall.FunctionName, FunctionArgs = value.FunctionCall.FunctionArguments }; @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments }; @@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 00f47149..57f7ac13 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = value.FunctionCall.FunctionName, FunctionArgs = value.FunctionCall.FunctionArguments }; @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Function, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, FunctionName = toolCall?.FunctionName, FunctionArgs = toolCall?.FunctionArguments }; @@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion responseMessage = new RoleDialogModel(AgentRole.Assistant, text) { CurrentAgentId = agent.Id, - MessageId = conversations.Last().MessageId + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, }; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 6d2b4b05..266abbec 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -53,6 +53,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs new file mode 100644 index 00000000..f863adb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs @@ -0,0 +1,24 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.SqlDriver.Controllers; + +[Authorize] +[ApiController] +public class SqlDriverController : ControllerBase +{ + private readonly IServiceProvider _services; + + public SqlDriverController(IServiceProvider services) + { + _services = services; + } + + [HttpPost("/knowledge/database/import")] + public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) + { + var dbKnowledge = _services.GetRequiredService(); + return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 47f64a1f..6fcee0bd 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -20,25 +20,36 @@ public class GetTableDefinitionFn : IFunctionCallback var settings = _services.GetRequiredService(); // Get table DDL from database - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); var tableDdls = new List(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); - foreach (var p in (List)message.Data) + foreach (var table in (List)message.Data) { - var escapedTableName = MySqlHelper.EscapeString(p); - dictionary["@" + "table_name"] = p; - dictionary["table_name"] = escapedTableName; + var escapedTableName = MySqlHelper.EscapeString(table); + + var sql = $"select * from information_schema.tables where table_name = @tableName"; + var result = connection.QueryFirstOrDefault(sql, new + { + tableName = escapedTableName + }); - var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'"; - var result = connection.QueryFirstOrDefault(sql: sql, dictionary); if (result == null) continue; sql = $"SHOW CREATE TABLE `{escapedTableName}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - tableDdls.Add(result); + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + tableDdls.Add(result); + } + + reader.Close(); + command.Dispose(); } + connection.Close(); message.Content = string.Join("\r\n", tableDdls); return true; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs new file mode 100644 index 00000000..48ba820c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs @@ -0,0 +1,19 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class RequestBase +{ + [JsonPropertyName("provider")] + public string? Provider { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } +} + + +public class ImportDbKnowledgeRequest : RequestBase +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs new file mode 100644 index 00000000..7807f87f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -0,0 +1,140 @@ +using static Dapper.SqlMapper; +using Microsoft.Extensions.Logging; +using BotSharp.Core.Infrastructures; +using MySqlConnector; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.SqlDriver.Services; + +public class DbKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public DbKnowledgeService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Import(string provider, string model, string schema) + { + var sqlDriverSettings = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var tables = new HashSet(); + using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString); + + var sql = $"select table_name from information_schema.tables where table_schema = @tableSchema"; + var results = connection.Query(sql, new + { + tableSchema = schema + }); + + foreach (var item in results) + { + if (item == null) continue; + + tables.Add(item.TABLE_NAME); + } + + foreach (var table in tables) + { + try + { + _logger.LogInformation($"Start processing table {table}\r\n"); + + var ddl = GetTableStructure(table); + if (string.IsNullOrEmpty(ddl)) continue; + + var prompt = await GetPrompt(ddl); + var response = await GetAiResponse(prompt, provider, model); + var knowledges = response.Content.JsonArrayContent(); + + if (knowledges.IsNullOrEmpty()) + { + _logger.LogInformation($"No knowledge for table {table}"); + continue; + } + + foreach (var item in knowledges) + { + await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + { + Text = item.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, item.Answer } + } + }); + + _logger.LogInformation($"Knowledge {table} is saved =>\r\nQuestion: {item.Question}\r\nAnswer: {item.Answer}\r\n"); + } + } + catch (Exception ex) + { + var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}"; + _logger.LogWarning(note); + } + } + + return true; + } + + private string GetTableStructure(string table) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); + + var result = string.Empty; + var escapedTableName = MySqlHelper.EscapeString(table); + var sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + } + + reader.Close(); + command.Dispose(); + connection.Close(); + return result; + } + + private async Task GetPrompt(string content) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; + + return render.Render(template, new Dictionary + { + { "table_structure", content } + }); + } + + private async Task GetAiResponse(string prompt, string provider, string model) + { + var agent = new Agent + { + Id = string.Empty, + Name = "Db knowledge", + Instruction = prompt, + }; + + var completion = CompletionProvider.GetChatCompletion(_services, provider, model); + return await completion.GetChatCompletions(agent, new List()); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index aaad1a0f..c059edfb 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -16,6 +16,7 @@ public class SqlDriverPlugin : IBotSharpPlugin }); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped();