Merge branch 'SciSharp:master' into master

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -6,6 +6,7 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     #region Vector
+    Task<bool> ExistVectorCollection(string collectionName);
     Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
     Task<bool> DeleteVectorCollection(string collectionName);
     Task<IEnumerable<string>> GetVectorCollections(string type);

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -116,7 +116,6 @@ public interface IBotSharpRepository
     bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
     bool DeleteKnowledgeCollectionConfig(string collectionName);
     IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
-
     bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
     /// <summary>
     /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -5,7 +5,8 @@ namespace BotSharp.Abstraction.VectorStorage;
 public interface IVectorDb
 {
     string Provider { get; }
-    
+
+    Task<bool> DoesCollectionExist(string collectionName);
     Task<IEnumerable<string>> GetCollections();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
     Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

@@ -213,7 +213,7 @@ public partial class FileRepository
         
         return new PagedItems<KnowledgeDocMetaData>
         {
-            Items = records.Skip(filter.Offset).Take(filter.Size),
+            Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size),
             Count = records.Count
         };
     }

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Graph.Models;
-using BotSharp.Abstraction.Knowledges.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 using BotSharp.OpenAPI.ViewModels.Knowledges;
 
@@ -20,6 +19,12 @@ public class KnowledgeBaseController : ControllerBase
     }
 
     #region Vector
+    [HttpGet("knowledge/vector/{collection}/exist")]
+    public async Task<bool> ExistVectorCollection([FromRoute] string collection)
+    {
+        return await _knowledgeService.ExistVectorCollection(collection);
+    }
+
     [HttpGet("knowledge/vector/collections")]
     public async Task<IEnumerable<string>> GetVectorCollections([FromQuery] string type)
     {

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -10,6 +10,12 @@ public class MemoryVectorDb : IVectorDb
 
     public string Provider => "MemoryVector";
 
+
+    public async Task<bool> DoesCollectionExist(string collectionName)
+    {
+        return false;
+    }
+
     public async Task<bool> CreateCollection(string collectionName, int dimension)
     {
         _collections[collectionName] = dimension;

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -3,7 +3,6 @@ using BotSharp.Abstraction.Files.Models;
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Knowledges.Helpers;
 using BotSharp.Abstraction.VectorStorage.Enums;
-using System.Collections;
 using System.Net.Http;
 using System.Net.Mime;
 
@@ -13,13 +12,21 @@ public partial class KnowledgeService
 {
     public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
     {
+        var res = new UploadKnowledgeResponse
+        {
+            Success = [],
+            Failed = files?.Select(x => x.FileName) ?? new List<string>()
+        };
+
         if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
         {
-            return new UploadKnowledgeResponse
-            {
-                Success = [],
-                Failed = files?.Select(x => x.FileName) ?? new List<string>()
-            };
+            return res;
+        }
+
+        var exist = await ExistVectorCollection(collectionName);
+        if (!exist)
+        {
+            return res;
         }
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
@@ -103,6 +110,9 @@ public partial class KnowledgeService
 
         try
         {
+            var exist = await ExistVectorCollection(collectionName);
+            if (!exist) return false;
+
             var db = _services.GetRequiredService<IBotSharpRepository>();
             var userId = await GetUserId();
             var vectorStoreProvider = _settings.VectorDb.Provider;

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -7,6 +7,23 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 public partial class KnowledgeService
 {
     #region Collection
+    public async Task<bool> ExistVectorCollection(string collectionName)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var vectorDb = GetVectorDb();
+
+        var exist = await vectorDb.DoesCollectionExist(collectionName);
+        if (exist) return true;
+
+        var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
+        {
+            CollectionNames = [collectionName],
+            VectorStroageProviders = [_settings.VectorDb.Provider]
+        });
+
+        return !configs.IsNullOrEmpty();
+    }
+
     public async Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model)
     {
         try

src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs

@@ -33,7 +33,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
         {
             var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions
             {
-                Confidence = 0.2f
+                Confidence = 0.4f
             });
             knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));
             
@@ -56,7 +56,10 @@ public class PrimaryStagePlanFn : IFunctionCallback
             LlmConfig = currentAgent.LlmConfig
         };
         var response = await GetAiResponse(plannerAgent);
-        message.Content = response.Content; 
+        message.Content = response.Content;
+
+        var states = _services.GetRequiredService<IConversationStateService>();
+        states.SetState("planning_result", response.Content);
 
         return true;
     }

src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs

@@ -41,7 +41,7 @@ public class SecondaryStagePlanFn : IFunctionCallback
 
             var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions
             {
-                Confidence = 0.5f
+                Confidence = 0.6f
             });
             message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
         }
@@ -63,6 +63,9 @@ public class SecondaryStagePlanFn : IFunctionCallback
         var response = await GetAiResponse(plannerAgent);
         message.Content = response.Content;
         _logger.LogInformation(response.Content);
+
+        var states = _services.GetRequiredService<IConversationStateService>();
+        states.SetState("planning_result", response.Content);
         return true;
     }
 

src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs

@@ -30,10 +30,13 @@ public class SummaryPlanFn : IFunctionCallback
         var taskRequirement = state.GetState("requirement_detail");
 
         // Get table names
-        var steps = message.Content.JsonArrayContent<SecondStagePlan>();
+        var states = _services.GetRequiredService<IConversationStateService>();
+        var steps = states.GetState("planning_result").JsonArrayContent<SecondStagePlan>();
         var allTables = new List<string>();
         var ddlStatements = "";
-        var relevantKnowledge = message.Content;
+        var relevantKnowledge = states.GetState("planning_result");
+        relevantKnowledge += states.GetState("dictionary_items");
+
         foreach (var step in steps)
         {
             allTables.AddRange(step.Tables);

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondaryBreakdownTask.cs

@@ -7,4 +7,7 @@ public class SecondaryBreakdownTask
 
     [JsonPropertyName("solution_search_question")]
     public string SolutionQuestion { get; set; } = null!;
+
+    [JsonPropertyName("need_lookup_dictionary")]
+    public bool NeedLookupDictionary { get; set; }
 }

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_summary.json

@@ -4,7 +4,15 @@
   "parameters": {
     "type": "object",
     "properties": {
+      "related_tables": {
+        "type": "array",
+        "description": "table name in planning steps",
+        "items": {
+          "type": "string",
+          "description": "table name"
+        }
+      }
     },
-    "required": []
+    "required": [ "related_tables" ]
   }
 }

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid

@@ -1,8 +1,10 @@
 Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 
 1. Call plan_primary_stage to generate the primary plan.
 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage.
-3. You must call plan_summary to generate final planned steps.
-4. If you can't generate the final accurate planning steps due to missing some specific informations, please ask user for more information.
+3. Repeat step 2 until you processed all the primary stages.
+4. If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name.
+   If you no items retured, you can pull all the list and find the match.
+5. You must call plan_summary for you final planned output.
 
 *** IMPORTANT ***
 Don't run the planning process repeatedly if you have already got the result of user's request.

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid

@@ -8,6 +8,7 @@ Thinking process:
     - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps.
     - If the solution mentioned "related solutions" is needed, set the need_additional_information to true.
     - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true.
+    - If you need to verify or get the enum/term/dictionary value, set the need_additional_information to true.
 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]
 5. You can NOT generate the final query before calling function plan_summary.

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid

@@ -3,6 +3,7 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown
 * The parameters can be extracted from the original task.
 * You need to list all the steps in detail. Finding relationships should also be a step.
 * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly.
+* If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code.
 * Output all the steps as much detail as possible in JSON: [{{ response_format }}]
 
 

src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid

@@ -1 +1,2 @@
-For every primary step, if need_additional_information is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step.
+For every primary step, if need_additional_information is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step.
+if need_lookup_dictionary is true, you have to call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code.

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -39,16 +39,22 @@ public class QdrantDb : IVectorDb
         return _client;
     }
 
-    public async Task<bool> CreateCollection(string collectionName, int dimension)
+    public async Task<bool> DoesCollectionExist(string collectionName)
     {
         var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        return await client.CollectionExistsAsync(collectionName);
+    }
+
+    public async Task<bool> CreateCollection(string collectionName, int dimension)
+    {
+        var exist = await DoesCollectionExist(collectionName);
 
         if (exist) return false;
 
         try
         {
             // Create a new collection
+            var client = GetClient();
             await client.CreateCollectionAsync(collectionName, new VectorParams()
             {
                 Size = (ulong)dimension,
@@ -65,11 +71,11 @@ public class QdrantDb : IVectorDb
 
     public async Task<bool> DeleteCollection(string collectionName)
     {
-        var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        var exist = await DoesCollectionExist(collectionName);
 
         if (!exist) return false;
 
+        var client = GetClient();
         await client.DeleteCollectionAsync(collectionName);
         return true;
     }
@@ -83,8 +89,7 @@ public class QdrantDb : IVectorDb
 
     public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
     {
-        var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        var exist = await DoesCollectionExist(collectionName);
         if (!exist)
         {
             return new StringIdPagedItems<VectorCollectionData>();
@@ -126,6 +131,7 @@ public class QdrantDb : IVectorDb
             };
         }
 
+        var client = GetClient();
         var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
         var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, 
             offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
@@ -152,15 +158,18 @@ public class QdrantDb : IVectorDb
     public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
         bool withPayload = false, bool withVector = false)
     {
-        if (ids.IsNullOrEmpty()) return Enumerable.Empty<VectorCollectionData>();
-
-        var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        if (ids.IsNullOrEmpty())
+        {
+            return Enumerable.Empty<VectorCollectionData>();
+        }
+        
+        var exist = await DoesCollectionExist(collectionName);
         if (!exist)
         {
             return Enumerable.Empty<VectorCollectionData>();
         }
 
+        var client = GetClient();
         var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
         var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
         return points.Select(x => new VectorCollectionData
@@ -209,8 +218,7 @@ public class QdrantDb : IVectorDb
     {
         var results = new List<VectorCollectionData>();
 
-        var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        var exist = await DoesCollectionExist(collectionName);
         if (!exist)
         {
             return results;
@@ -221,7 +229,8 @@ public class QdrantDb : IVectorDb
         {
             payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } };
         }
-        
+
+        var client = GetClient();
         var points = await client.SearchAsync(collectionName,
                                             vector,
                                             limit: (ulong)limit,
@@ -244,33 +253,27 @@ public class QdrantDb : IVectorDb
     {
         if (ids.IsNullOrEmpty()) return false;
 
-        var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        var exist = await DoesCollectionExist(collectionName);
         if (!exist)
         {
             return false;
         }
 
+        var client = GetClient();
         var result = await client.DeleteAsync(collectionName, ids);
         return result.Status == UpdateStatus.Completed;
     }
 
     public async Task<bool> DeleteCollectionAllData(string collectionName)
     {
-        var client = GetClient();
-        var exist = await DoesCollectionExist(client, collectionName);
+        var exist = await DoesCollectionExist(collectionName);
         if (!exist)
         {
             return false;
         }
 
+        var client = GetClient();
         var result = await client.DeleteAsync(collectionName, new Filter());
         return result.Status == UpdateStatus.Completed;
     }
-
-
-    private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
-    {
-        return await client.CollectionExistsAsync(collectionName);
-    }
 }

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

@@ -25,6 +25,12 @@ namespace BotSharp.Plugin.SemanticKernel
 
         public string Provider => "SemanticKernel";
 
+
+        public async Task<bool> DoesCollectionExist(string collectionName)
+        {
+            return false;
+        }
+
         public async Task<bool> CreateCollection(string collectionName, int dimension)
         {
             await _memoryStore.CreateCollectionAsync(collectionName);

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -18,8 +18,10 @@
 
   <ItemGroup>
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\get_table_definition.json" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_dictionary_lookup.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_select.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_dictionary_lookup.fn.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_executor.fn.liquid" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json" />
@@ -27,10 +29,16 @@
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_select.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid" />
-    <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\lookup_dictionary.liquid" />
+    <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_dictionary_lookup.liquid" />
   </ItemGroup>
 
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_dictionary_lookup.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_dictionary_lookup.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\get_table_definition.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
@@ -46,10 +54,7 @@
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\lookup_dictionary.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\lookup_dictionary.json">
+    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_dictionary_lookup.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json">

src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs

@@ -3,4 +3,5 @@ namespace BotSharp.Plugin.SqlDriver.Enum;
 public  class Utility
 {
     public const string SqlExecutor = "sql-executor";
+    public const string SqlDictionaryLookup = "sql-dictionary-lookup";
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs

@@ -1,3 +1,4 @@
+using Azure;
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Core.Infrastructures;
@@ -9,7 +10,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions;
 
 public class LookupDictionaryFn : IFunctionCallback
 {
-    public string Name => "lookup_dictionary";
+    public string Name => "sql_dictionary_lookup";
     private readonly IServiceProvider _services;
 
     public LookupDictionaryFn(IServiceProvider services)
@@ -21,58 +22,24 @@ public class LookupDictionaryFn : IFunctionCallback
     {
         var args = JsonSerializer.Deserialize<LookupDictionary>(message.FunctionArgs);
 
+        // check if need to instantely
         var settings = _services.GetRequiredService<SqlDriverSetting>();
-        using var connection = new MySqlConnection(settings.MySqlConnectionString);
-        var dictionary = new Dictionary<string, object>();
-        var results = connection.Query($"SELECT * FROM {args.Table} LIMIT 10");
-        var items = new List<string>();
-        foreach(var item in results)
+        using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
+        var result = connection.Query(args.SqlStatement);
+
+        if (result == null)
         {
-            items.Add(JsonSerializer.Serialize(item));
+            message.Content = "Record not found";
         }
-
-        var agentService = _services.GetRequiredService<IAgentService>();
-        var agent = await agentService.LoadAgent(message.CurrentAgentId);
-        var prompt = GetPrompt(agent, items, args.Keyword);
-
-        // Ask LLM which one is the best
-        var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
-        var model = llmProviderService.GetProviderModel("azure-openai", "gpt-35-turbo");
-
-        // chat completion
-        var completion = CompletionProvider.GetChatCompletion(_services,
-            provider: "azure-openai",
-            model: model.Name);
-
-        var conversations = new List<RoleDialogModel>
+        else
         {
-            new RoleDialogModel(AgentRole.User, prompt)
-            {
-                CurrentAgentId = message.CurrentAgentId,
-                MessageId = message.MessageId,
-            } 
-        };
-
-        var response = await completion.GetChatCompletions(new Agent
-        {
-            Id = message.CurrentAgentId,
-            Instruction = ""
-        }, conversations);
-
-        message.Content = response.Content;
+            message.Content = JsonSerializer.Serialize(result);
+        }
+        var states = _services.GetRequiredService<IConversationStateService>();
+        var dictionaryItems = states.GetState("dictionary_items", "");
+        dictionaryItems += "\r\n\r\n" +  args.Reason + ":\r\n" + message.Content + "\r\n";
+        states.SetState("dictionary_items", dictionaryItems);
 
         return true;
     }
-
-    private string GetPrompt(Agent agent, List<string> task, string keyword)
-    {
-        var template = agent.Templates.First(x => x.Name == "lookup_dictionary").Content;
-
-        var render = _services.GetRequiredService<ITemplateRender>();
-        return render.Render(template, new Dictionary<string, object>
-        {
-            { "items", task },
-            { "keyword", keyword }
-        });
-    }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs

@@ -0,0 +1,85 @@
+using BotSharp.Abstraction.Agents.Enums;
+using BotSharp.Abstraction.Agents.Settings;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Repositories;
+using System.Collections.Generic;
+
+namespace BotSharp.Plugin.SqlDriver.Hooks;
+
+public class SqlDictionaryLookupHook : AgentHookBase, IAgentHook
+{
+    private const string SQL_EXECUTOR_TEMPLATE = "sql_dictionary_lookup.fn";
+    private IEnumerable<string> _targetSqlExecutorFunctions = new List<string>
+    {
+        "sql_dictionary_lookup",
+    };
+
+    public override string SelfId => BuiltInAgentId.Planner;
+
+    public SqlDictionaryLookupHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
+    {
+    }
+
+    public override void OnAgentLoaded(Agent agent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var isConvMode = conv.IsConversationMode();
+        var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.SqlDictionaryLookup);
+
+        if (isConvMode && isEnabled)
+        {
+            var (prompt, fns) = GetPromptAndFunctions();
+            if (!fns.IsNullOrEmpty())
+            {
+                if (!string.IsNullOrWhiteSpace(prompt))
+                {
+                    agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+                }
+
+                if (agent.Functions == null)
+                {
+                    agent.Functions = fns;
+                }
+                else
+                {
+                    agent.Functions.AddRange(fns);
+                }
+            }
+        }
+
+        base.OnAgentLoaded(agent);
+    }
+
+    private (string, List<FunctionDef>?) GetPromptAndFunctions()
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
+        var fns = agent?.Functions?.Where(x => _targetSqlExecutorFunctions.Contains(x.Name))?.ToList();
+
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(SQL_EXECUTOR_TEMPLATE))?.Content ?? string.Empty;
+        var dbType = GetDatabaseType();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        prompt = render.Render(prompt, new Dictionary<string, object>
+        {
+            { "db_type", dbType }
+        });
+
+        return (prompt, fns);
+    }
+
+    private string GetDatabaseType()
+    {
+        var settings = _services.GetRequiredService<SqlDriverSetting>();
+        var dbType = "MySQL";
+
+        if (!string.IsNullOrWhiteSpace(settings?.SqlServerConnectionString))
+        {
+            dbType = "SQL Server";
+        }
+        else if (!string.IsNullOrWhiteSpace(settings?.SqlLiteConnectionString))
+        {
+            dbType = "SQL Lite";
+        }
+        return dbType;
+    }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs

@@ -1,9 +1,10 @@
 namespace BotSharp.Plugin.SqlDriver.Hooks;
 
-public class SqlExecutorUtilityHook : IAgentUtilityHook
+public class SqlUtilityHook : IAgentUtilityHook
 {
     public void AddUtilities(List<string> utilities)
     {
         utilities.Add(Utility.SqlExecutor);
+        utilities.Add(Utility.SqlDictionaryLookup);
     }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs

@@ -4,15 +4,12 @@ namespace BotSharp.Plugin.SqlDriver.Models;
 
 public class LookupDictionary
 {
-    [JsonPropertyName("table")]
-    public string Table { get; set; }
-
-    [JsonPropertyName("keyword")]
-    public string Keyword { get; set; }
+    [JsonPropertyName("sql_statement")]
+    public string SqlStatement { get; set; }
 
     [JsonPropertyName("reason")]
     public string Reason { get; set; }
 
-    [JsonPropertyName("columns")]
-    public string[] Columns { get; set; }
+    [JsonPropertyName("table")]
+    public string Table { get; set; }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs

@@ -21,7 +21,8 @@ public class SqlDriverPlugin : IBotSharpPlugin
         services.AddScoped<DbKnowledgeService>();
         services.AddScoped<IKnowledgeHook, SqlDriverKnowledgeHook>();
         services.AddScoped<IAgentHook, SqlExecutorHook>();
-        services.AddScoped<IAgentUtilityHook, SqlExecutorUtilityHook>();
+        services.AddScoped<IAgentUtilityHook, SqlUtilityHook>();
         services.AddScoped<IPlanningHook, SqlDriverPlanningHook>();
+        services.AddScoped<IAgentHook, SqlDictionaryLookupHook>();
     }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json

@@ -0,0 +1,22 @@
+{
+  "name": "sql_dictionary_lookup",
+  "description": "Get id from dictionary table by keyword if tool or solution mentioned this approach",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "sql_statement": {
+        "type": "string",
+        "description": "sql text"
+      },
+      "reason": {
+        "type": "string",
+        "description": "the reason why you need to call sql_dictionary_lookup"
+      },
+      "table": {
+        "type": "string",
+        "description": "table name"
+      }
+    },
+    "required": [ "sql_statement", "reason", "table" ]
+  }
+}

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/lookup_dictionary.json

@@ -1,30 +0,0 @@
-{
-  "name": "lookup_dictionary",
-  "description": "Get id from dictionary table by keyword if tool or solution mentioned this approach",
-  "parameters": {
-    "type": "object",
-    "properties": {
-      "table": {
-        "type": "string",
-        "description": "table name"
-      },
-      "keyword": {
-        "type": "string",
-        "description": "table name"
-      },
-      "reason": {
-        "type": "string",
-        "description": "the reason why you need to call lookup_dictionary"
-      },
-      "columns": {
-        "type": "array",
-        "description": "columns",
-        "items": {
-          "type": "string",
-          "description": "column"
-        }
-      }
-    },
-    "required": [ "table", "keyword", "reason", "columns" ]
-  }
-}

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid

@@ -0,0 +1,8 @@
+Dictionary Lookup Rules:
+=====
+Please call function sql_dictionary_lookup if user wants to get or retrieve dictionary/enum/term from data tables. 
+You must return the id and name/code. 
+
+You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
+Dictionary table pattern is table name starting with "data_". You can only query the dictionary table without join other non-dictionary tables.
+=====

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_select.json

@@ -1,6 +1,6 @@
 {
   "name": "sql_select",
-  "description": "Get the specific value from table",
+  "description": "Execute the reporting related query in the database and get the result",
   "parameters": {
     "type": "object",
     "properties": {
@@ -11,46 +11,8 @@
       "reason": {
         "type": "string",
         "description": "reason"
-      },
-      "table": {
-        "type": "string",
-        "description": "related table"
-      },
-      "parameters": {
-        "type": "array",
-        "description": "data criteria for the query",
-        "items": {
-          "type": "object",
-          "description": "the name and value for the parameter",
-          "properties": {
-            "name": {
-              "type": "string",
-              "description": "field name"
-            },
-            "value": {
-              "type": "string",
-              "description": "real value inferred by the context"
-            }
-          },
-          "required": [ "name", "value" ]
-        }
-      },
-      "return_field": {
-        "type": "object",
-        "description": "the name and alias for the return field",
-        "properties": {
-          "name": {
-            "type": "string",
-            "description": "field in the table"
-          },
-          "alias": {
-            "type": "string",
-            "description": "meaningful field alias"
-          }
-        },
-        "required": [ "name", "value" ]
       }
     },
-    "required": [ "sql_statement", "reason", "table", "parameters", "return_field" ]
+    "required": [ "sql_statement", "reason" ]
   }
 }