refine db knowledge

src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs

@@ -23,10 +23,13 @@ public class VerboseLogHook : IContentGeneratingHook
     {
         if (!_convSettings.ShowVerboseLog) return;
 
-        var dialog = conversations.Last();
-        var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>";
-        _logger.LogInformation(log);
-
+        var dialog = conversations.LastOrDefault();
+        if (dialog != null)
+        {
+            var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>";
+            _logger.LogInformation(log);
+        }
+        
         await Task.CompletedTask;
     }
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -142,6 +142,7 @@ public class KnowledgeBaseController : ControllerBase
     }
     #endregion
 
+
     #region Knowledge
     [HttpPost("/knowledge/search")]
     public async Task<KnowledgeSearchViewModel> SearchKnowledge([FromBody] SearchKnowledgeRequest request)

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion
             responseMessage = new RoleDialogModel(AgentRole.Function, text)
             {
                 CurrentAgentId = agent.Id,
-                MessageId = conversations.Last().MessageId,
+                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
                 FunctionName = value.FunctionCall.FunctionName,
                 FunctionArgs = value.FunctionCall.FunctionArguments
             };
@@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion
             responseMessage = new RoleDialogModel(AgentRole.Function, text)
             {
                 CurrentAgentId = agent.Id,
-                MessageId = conversations.Last().MessageId,
+                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
                 FunctionName = toolCall?.FunctionName,
                 FunctionArgs = toolCall?.FunctionArguments
             };
@@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion
             responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
             {
                 CurrentAgentId = agent.Id,
-                MessageId = conversations.Last().MessageId
+                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
             };
         }
 

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -49,7 +49,7 @@ public class ChatCompletionProvider : IChatCompletion
             responseMessage = new RoleDialogModel(AgentRole.Function, text)
             {
                 CurrentAgentId = agent.Id,
-                MessageId = conversations.Last().MessageId,
+                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
                 FunctionName = value.FunctionCall.FunctionName,
                 FunctionArgs = value.FunctionCall.FunctionArguments
             };
@@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion
             responseMessage = new RoleDialogModel(AgentRole.Function, text)
             {
                 CurrentAgentId = agent.Id,
-                MessageId = conversations.Last().MessageId,
+                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
                 FunctionName = toolCall?.FunctionName,
                 FunctionArgs = toolCall?.FunctionArguments
             };
@@ -76,7 +76,7 @@ public class ChatCompletionProvider : IChatCompletion
             responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
             {
                 CurrentAgentId = agent.Id,
-                MessageId = conversations.Last().MessageId
+                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
             };
         }
 

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>
@@ -53,6 +53,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
     <PackageReference Include="MySqlConnector" Version="2.3.7" />
   </ItemGroup>
 

src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs

@@ -0,0 +1,24 @@
+using BotSharp.Plugin.SqlDriver.Models;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.SqlDriver.Controllers;
+
+[Authorize]
+[ApiController]
+public class SqlDriverController : ControllerBase
+{
+    private readonly IServiceProvider _services;
+
+    public SqlDriverController(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    [HttpPost("/knowledge/database/import")]
+    public async Task<bool> ImportDbKnowledge(ImportDbKnowledgeRequest request)
+    {
+        var dbKnowledge = _services.GetRequiredService<DbKnowledgeService>();
+        return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema);
+    }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs

@@ -20,25 +20,36 @@ public class GetTableDefinitionFn : IFunctionCallback
         var settings = _services.GetRequiredService<SqlDriverSetting>();
 
         // Get table DDL from database
-        using var connection = new MySqlConnection(settings.MySqlConnectionString);
-        var dictionary = new Dictionary<string, object>();
         var tableDdls = new List<string>();
+        using var connection = new MySqlConnection(settings.MySqlConnectionString);
+        connection.Open();
 
-        foreach (var p in (List<string>)message.Data)
+        foreach (var table in (List<string>)message.Data)
         {
-            var escapedTableName = MySqlHelper.EscapeString(p);
-            dictionary["@" + "table_name"] = p;
-            dictionary["table_name"] = escapedTableName;
+            var escapedTableName = MySqlHelper.EscapeString(table);
+
+            var sql = $"select * from information_schema.tables where table_name = @tableName";
+            var result = connection.QueryFirstOrDefault(sql, new
+            {
+                tableName = escapedTableName
+            });
 
-            var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'";
-            var result = connection.QueryFirstOrDefault(sql: sql, dictionary);
             if (result == null) continue;
 
             sql = $"SHOW CREATE TABLE `{escapedTableName}`";
-            result = connection.QueryFirstOrDefault(sql: sql, dictionary);
-            tableDdls.Add(result);
+            using var command = new MySqlCommand(sql, connection);
+            using var reader = command.ExecuteReader();
+            if (reader.Read())
+            {
+                result = reader.GetString(1);
+                tableDdls.Add(result);
+            }
+
+            reader.Close();
+            command.Dispose();
         }
 
+        connection.Close();
         message.Content = string.Join("\r\n", tableDdls);
         return true;
     }

src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs

@@ -0,0 +1,19 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.SqlDriver.Models;
+
+public class RequestBase
+{
+    [JsonPropertyName("provider")]
+    public string? Provider { get; set; }
+
+    [JsonPropertyName("model")]
+    public string? Model { get; set; }
+}
+
+
+public class ImportDbKnowledgeRequest : RequestBase
+{
+    [JsonPropertyName("schema")]
+    public string Schema { get; set; }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs

@@ -0,0 +1,140 @@
+using static Dapper.SqlMapper;
+using Microsoft.Extensions.Logging;
+using BotSharp.Core.Infrastructures;
+using MySqlConnector;
+using BotSharp.Abstraction.Agents.Enums;
+using BotSharp.Abstraction.Knowledges.Settings;
+using BotSharp.Abstraction.Knowledges.Enums;
+using BotSharp.Abstraction.VectorStorage.Models;
+
+namespace BotSharp.Plugin.SqlDriver.Services;
+
+public class DbKnowledgeService
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger<DbKnowledgeService> _logger;
+
+    public DbKnowledgeService(
+        IServiceProvider services,
+        ILogger<DbKnowledgeService> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Import(string provider, string model, string schema)
+    {
+        var sqlDriverSettings = _services.GetRequiredService<SqlDriverSetting>();
+        var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
+        var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
+        var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+
+        var tables = new HashSet<string>();
+        using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString);
+
+        var sql = $"select table_name from information_schema.tables where table_schema = @tableSchema";
+        var results = connection.Query(sql, new
+        {
+            tableSchema = schema 
+        });
+
+        foreach (var item in results)
+        {
+            if (item == null) continue;
+
+            tables.Add(item.TABLE_NAME);
+        }
+
+        foreach (var table in tables)
+        {
+            try
+            {
+                _logger.LogInformation($"Start processing table {table}\r\n");
+
+                var ddl = GetTableStructure(table);
+                if (string.IsNullOrEmpty(ddl)) continue;
+
+                var prompt = await GetPrompt(ddl);
+                var response = await GetAiResponse(prompt, provider, model);
+                var knowledges = response.Content.JsonArrayContent<ExtractedKnowledge>();
+
+                if (knowledges.IsNullOrEmpty())
+                {
+                    _logger.LogInformation($"No knowledge for table {table}");
+                    continue;
+                }
+
+                foreach (var item in knowledges)
+                {
+                    await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel
+                    {
+                        Text = item.Question,
+                        Payload = new Dictionary<string, string>
+                        {
+                            { KnowledgePayloadName.Answer, item.Answer }
+                        }
+                    });
+
+                    _logger.LogInformation($"Knowledge {table} is saved =>\r\nQuestion: {item.Question}\r\nAnswer: {item.Answer}\r\n");
+                }
+            }
+            catch (Exception ex)
+            {
+                var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}";
+                _logger.LogWarning(note);
+            }
+        }
+
+        return true;
+    }
+
+    private string GetTableStructure(string table)
+    {
+        var settings = _services.GetRequiredService<SqlDriverSetting>();
+        using var connection = new MySqlConnection(settings.MySqlConnectionString);
+        connection.Open();
+
+        var result = string.Empty;
+        var escapedTableName = MySqlHelper.EscapeString(table);
+        var sql = $"SHOW CREATE TABLE `{escapedTableName}`";
+
+        using var command = new MySqlCommand(sql, connection);
+        using var reader = command.ExecuteReader();
+        if (reader.Read())
+        {
+            result = reader.GetString(1);
+        }
+
+        reader.Close();
+        command.Dispose();
+        connection.Close();
+        return result;
+    }
+
+    private async Task<string> GetPrompt(string content)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+
+        var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant);
+        var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty;
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "table_structure", content }
+        });
+    }
+
+    private async Task<RoleDialogModel> GetAiResponse(string prompt, string provider, string model)
+    {
+        var agent = new Agent
+        {
+            Id = string.Empty,
+            Name = "Db knowledge",
+            Instruction = prompt,
+        };
+
+        var completion = CompletionProvider.GetChatCompletion(_services, provider, model);
+        return await completion.GetChatCompletions(agent, new List<RoleDialogModel>());
+    }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs

@@ -16,6 +16,7 @@ public class SqlDriverPlugin : IBotSharpPlugin
         });
 
         services.AddScoped<SqlDriverService>();
+        services.AddScoped<DbKnowledgeService>();
         services.AddScoped<IKnowledgeHook, SqlDriverKnowledgeHook>();
         services.AddScoped<IAgentHook, SqlExecutorHook>();
         services.AddScoped<IAgentUtilityHook, SqlExecutorUtilityHook>();