diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 291af271..0b1c06b5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -6,6 +6,7 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { #region Vector + Task ExistVectorCollection(string collectionName); Task CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model); Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(string type); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index bfd757c1..ced839d0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -116,7 +116,6 @@ public interface IBotSharpRepository bool AddKnowledgeCollectionConfigs(List configs, bool reset = false); bool DeleteKnowledgeCollectionConfig(string collectionName); IEnumerable GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter); - bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData); /// /// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection. diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs index 4d32d241..347e7554 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs @@ -10,4 +10,5 @@ public interface IUserIdentity string FullName { get; } string? UserLanguage { get; } string? Phone { get; } + string? AffiliateId { get; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 42471d42..ae0828de 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -5,7 +5,8 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { string Provider { get; } - + + Task DoesCollectionExist(string collectionName); Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs index 53927629..b0a95027 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs @@ -213,7 +213,7 @@ public partial class FileRepository return new PagedItems { - Items = records.Skip(filter.Offset).Take(filter.Size), + Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size), Count = records.Count }; } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index ee3b5c75..531fd286 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -69,4 +69,7 @@ public class UserIdentity : IUserIdentity [JsonPropertyName("phone")] public string? Phone => _claims?.FirstOrDefault(x => x.Type == "phone")?.Value; + + [JsonPropertyName("affiliateId")] + public string? AffiliateId => _claims?.FirstOrDefault(x => x.Type == "affiliateId")?.Value; } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index e4f5fc31..f8df34dc 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -254,7 +254,8 @@ public class UserService : IUserService new Claim("type", user.Type ?? UserType.Client), new Claim("role", user.Role ?? UserRole.User), new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), - new Claim("phone", user.Phone ?? string.Empty) + new Claim("phone", user.Phone ?? string.Empty), + new Claim("affiliateId", user.AffiliateId ?? string.Empty) }; var validators = _services.GetServices(); @@ -280,14 +281,14 @@ public class UserService : IUserService }; var tokenHandler = new JwtSecurityTokenHandler(); var token = tokenHandler.CreateToken(tokenDescriptor); - SaveUserTokenExpiresCache(user.Id, expires).GetAwaiter().GetResult(); + SaveUserTokenExpiresCache(user.Id, expires, expireInMinutes).GetAwaiter().GetResult(); return tokenHandler.WriteToken(token); } - private async Task SaveUserTokenExpiresCache(string userId, DateTime expires) + private async Task SaveUserTokenExpiresCache(string userId, DateTime expires, int expireInMinutes) { var _cacheService = _services.GetRequiredService(); - await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, null); + await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, TimeSpan.FromMinutes(expireInMinutes)); } private string GetUserTokenExpiresCacheKey(string userId) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index af01a685..3ead41b0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -20,6 +19,12 @@ public class KnowledgeBaseController : ControllerBase } #region Vector + [HttpGet("knowledge/vector/{collection}/exist")] + public async Task ExistVectorCollection([FromRoute] string collection) + { + return await _knowledgeService.ExistVectorCollection(collection); + } + [HttpGet("knowledge/vector/collections")] public async Task> GetVectorCollections([FromQuery] string type) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 0ba4f3e9..41331c2b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -10,6 +10,12 @@ public class MemoryVectorDb : IVectorDb public string Provider => "MemoryVector"; + + public async Task DoesCollectionExist(string collectionName) + { + return false; + } + public async Task CreateCollection(string collectionName, int dimension) { _collections[collectionName] = dimension; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index bb804673..bd851eb8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Knowledges.Helpers; using BotSharp.Abstraction.VectorStorage.Enums; -using System.Collections; using System.Net.Http; using System.Net.Mime; @@ -13,13 +12,21 @@ public partial class KnowledgeService { public async Task UploadDocumentsToKnowledge(string collectionName, IEnumerable files) { + var res = new UploadKnowledgeResponse + { + Success = [], + Failed = files?.Select(x => x.FileName) ?? new List() + }; + if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty()) { - return new UploadKnowledgeResponse - { - Success = [], - Failed = files?.Select(x => x.FileName) ?? new List() - }; + return res; + } + + var exist = await ExistVectorCollection(collectionName); + if (!exist) + { + return res; } var db = _services.GetRequiredService(); @@ -103,6 +110,9 @@ public partial class KnowledgeService try { + var exist = await ExistVectorCollection(collectionName); + if (!exist) return false; + var db = _services.GetRequiredService(); var userId = await GetUserId(); var vectorStoreProvider = _settings.VectorDb.Provider; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index 0c2d6e65..ee46b13e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -7,6 +7,23 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { #region Collection + public async Task ExistVectorCollection(string collectionName) + { + var db = _services.GetRequiredService(); + var vectorDb = GetVectorDb(); + + var exist = await vectorDb.DoesCollectionExist(collectionName); + if (exist) return true; + + var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter + { + CollectionNames = [collectionName], + VectorStroageProviders = [_settings.VectorDb.Provider] + }); + + return !configs.IsNullOrEmpty(); + } + public async Task CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model) { try diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index f1a34513..02d4f3a5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -33,7 +33,7 @@ public class PrimaryStagePlanFn : IFunctionCallback { var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions { - Confidence = 0.2f + Confidence = 0.4f }); knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer()))); @@ -56,7 +56,10 @@ public class PrimaryStagePlanFn : IFunctionCallback LlmConfig = currentAgent.LlmConfig }; var response = await GetAiResponse(plannerAgent); - message.Content = response.Content; + message.Content = response.Content; + + var states = _services.GetRequiredService(); + states.SetState("planning_result", response.Content); return true; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index cfd150d9..2e7cdaaf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -41,7 +41,7 @@ public class SecondaryStagePlanFn : IFunctionCallback var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions { - Confidence = 0.5f + Confidence = 0.6f }); message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); } @@ -63,6 +63,9 @@ public class SecondaryStagePlanFn : IFunctionCallback var response = await GetAiResponse(plannerAgent); message.Content = response.Content; _logger.LogInformation(response.Content); + + var states = _services.GetRequiredService(); + states.SetState("planning_result", response.Content); return true; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 430caf19..1c53a6c5 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -30,10 +30,13 @@ public class SummaryPlanFn : IFunctionCallback var taskRequirement = state.GetState("requirement_detail"); // Get table names - var steps = message.Content.JsonArrayContent(); + var states = _services.GetRequiredService(); + var steps = states.GetState("planning_result").JsonArrayContent(); var allTables = new List(); var ddlStatements = ""; - var relevantKnowledge = message.Content; + var relevantKnowledge = states.GetState("planning_result"); + relevantKnowledge += states.GetState("dictionary_items"); + foreach (var step in steps) { allTables.AddRange(step.Tables); diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondaryBreakdownTask.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondaryBreakdownTask.cs index 9adcc2ff..b16b0bcd 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondaryBreakdownTask.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondaryBreakdownTask.cs @@ -7,4 +7,7 @@ public class SecondaryBreakdownTask [JsonPropertyName("solution_search_question")] public string SolutionQuestion { get; set; } = null!; + + [JsonPropertyName("need_lookup_dictionary")] + public bool NeedLookupDictionary { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_summary.json b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_summary.json index 0345b744..7e459503 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_summary.json +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_summary.json @@ -4,7 +4,15 @@ "parameters": { "type": "object", "properties": { + "related_tables": { + "type": "array", + "description": "table name in planning steps", + "items": { + "type": "string", + "description": "table name" + } + } }, - "required": [] + "required": [ "related_tables" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid index 8f8b29a5..2f4c7a41 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid @@ -1,8 +1,10 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 1. Call plan_primary_stage to generate the primary plan. 2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage. -3. You must call plan_summary to generate final planned steps. -4. If you can't generate the final accurate planning steps due to missing some specific informations, please ask user for more information. +3. Repeat step 2 until you processed all the primary stages. +4. If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name. + If you no items retured, you can pull all the list and find the match. +5. You must call plan_summary for you final planned output. *** IMPORTANT *** Don't run the planning process repeatedly if you have already got the result of user's request. diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 943c1b00..22cc3c5f 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -8,6 +8,7 @@ Thinking process: - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true. + - If you need to verify or get the enum/term/dictionary value, set the need_additional_information to true. 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@'; 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}] 5. You can NOT generate the final query before calling function plan_summary. diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index d5340e2e..5437d3be 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -3,6 +3,7 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown * The parameters can be extracted from the original task. * You need to list all the steps in detail. Finding relationships should also be a step. * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly. +* If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code. * Output all the steps as much detail as possible in JSON: [{{ response_format }}] diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid index 726fe329..31aca793 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid @@ -1 +1,2 @@ -For every primary step, if need_additional_information is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step. \ No newline at end of file +For every primary step, if need_additional_information is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step. +if need_lookup_dictionary is true, you have to call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 5664fa40..6c54f60b 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -39,16 +39,22 @@ public class QdrantDb : IVectorDb return _client; } - public async Task CreateCollection(string collectionName, int dimension) + public async Task DoesCollectionExist(string collectionName) { var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + return await client.CollectionExistsAsync(collectionName); + } + + public async Task CreateCollection(string collectionName, int dimension) + { + var exist = await DoesCollectionExist(collectionName); if (exist) return false; try { // Create a new collection + var client = GetClient(); await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dimension, @@ -65,11 +71,11 @@ public class QdrantDb : IVectorDb public async Task DeleteCollection(string collectionName) { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + var exist = await DoesCollectionExist(collectionName); if (!exist) return false; + var client = GetClient(); await client.DeleteCollectionAsync(collectionName); return true; } @@ -83,8 +89,7 @@ public class QdrantDb : IVectorDb public async Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + var exist = await DoesCollectionExist(collectionName); if (!exist) { return new StringIdPagedItems(); @@ -126,6 +131,7 @@ public class QdrantDb : IVectorDb }; } + var client = GetClient(); var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, @@ -152,15 +158,18 @@ public class QdrantDb : IVectorDb public async Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false) { - if (ids.IsNullOrEmpty()) return Enumerable.Empty(); - - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + if (ids.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + var exist = await DoesCollectionExist(collectionName); if (!exist) { return Enumerable.Empty(); } + var client = GetClient(); var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList(); var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector); return points.Select(x => new VectorCollectionData @@ -209,8 +218,7 @@ public class QdrantDb : IVectorDb { var results = new List(); - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + var exist = await DoesCollectionExist(collectionName); if (!exist) { return results; @@ -221,7 +229,8 @@ public class QdrantDb : IVectorDb { payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; } - + + var client = GetClient(); var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, @@ -244,33 +253,27 @@ public class QdrantDb : IVectorDb { if (ids.IsNullOrEmpty()) return false; - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } + var client = GetClient(); var result = await client.DeleteAsync(collectionName, ids); return result.Status == UpdateStatus.Completed; } public async Task DeleteCollectionAllData(string collectionName) { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); + var exist = await DoesCollectionExist(collectionName); if (!exist) { return false; } + var client = GetClient(); var result = await client.DeleteAsync(collectionName, new Filter()); return result.Status == UpdateStatus.Completed; } - - - private async Task DoesCollectionExist(QdrantClient client, string collectionName) - { - return await client.CollectionExistsAsync(collectionName); - } } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index bf086db9..5a5dd5a7 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -25,6 +25,12 @@ namespace BotSharp.Plugin.SemanticKernel public string Provider => "SemanticKernel"; + + public async Task DoesCollectionExist(string collectionName) + { + return false; + } + public async Task CreateCollection(string collectionName, int dimension) { await _memoryStore.CreateCollectionAsync(collectionName); diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 295fad70..90a645b8 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -18,8 +18,10 @@ + + @@ -27,10 +29,16 @@ - + + + PreserveNewest + + + PreserveNewest + PreserveNewest @@ -46,10 +54,7 @@ PreserveNewest - - PreserveNewest - - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs index 380d9f51..b3a4862a 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs @@ -3,4 +3,5 @@ namespace BotSharp.Plugin.SqlDriver.Enum; public class Utility { public const string SqlExecutor = "sql-executor"; + public const string SqlDictionaryLookup = "sql-dictionary-lookup"; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs index e09caa7b..3c8a8cc4 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs @@ -1,3 +1,4 @@ +using Azure; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.MLTasks; using BotSharp.Core.Infrastructures; @@ -9,7 +10,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions; public class LookupDictionaryFn : IFunctionCallback { - public string Name => "lookup_dictionary"; + public string Name => "sql_dictionary_lookup"; private readonly IServiceProvider _services; public LookupDictionaryFn(IServiceProvider services) @@ -21,58 +22,24 @@ public class LookupDictionaryFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); + // check if need to instantely var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); - var results = connection.Query($"SELECT * FROM {args.Table} LIMIT 10"); - var items = new List(); - foreach(var item in results) + using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString); + var result = connection.Query(args.SqlStatement); + + if (result == null) { - items.Add(JsonSerializer.Serialize(item)); + message.Content = "Record not found"; } - - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(message.CurrentAgentId); - var prompt = GetPrompt(agent, items, args.Keyword); - - // Ask LLM which one is the best - var llmProviderService = _services.GetRequiredService(); - var model = llmProviderService.GetProviderModel("azure-openai", "gpt-35-turbo"); - - // chat completion - var completion = CompletionProvider.GetChatCompletion(_services, - provider: "azure-openai", - model: model.Name); - - var conversations = new List + else { - new RoleDialogModel(AgentRole.User, prompt) - { - CurrentAgentId = message.CurrentAgentId, - MessageId = message.MessageId, - } - }; - - var response = await completion.GetChatCompletions(new Agent - { - Id = message.CurrentAgentId, - Instruction = "" - }, conversations); - - message.Content = response.Content; + message.Content = JsonSerializer.Serialize(result); + } + var states = _services.GetRequiredService(); + var dictionaryItems = states.GetState("dictionary_items", ""); + dictionaryItems += "\r\n\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n"; + states.SetState("dictionary_items", dictionaryItems); return true; } - - private string GetPrompt(Agent agent, List task, string keyword) - { - var template = agent.Templates.First(x => x.Name == "lookup_dictionary").Content; - - var render = _services.GetRequiredService(); - return render.Render(template, new Dictionary - { - { "items", task }, - { "keyword", keyword } - }); - } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs new file mode 100644 index 00000000..0c0c9f24 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs @@ -0,0 +1,85 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories; +using System.Collections.Generic; + +namespace BotSharp.Plugin.SqlDriver.Hooks; + +public class SqlDictionaryLookupHook : AgentHookBase, IAgentHook +{ + private const string SQL_EXECUTOR_TEMPLATE = "sql_dictionary_lookup.fn"; + private IEnumerable _targetSqlExecutorFunctions = new List + { + "sql_dictionary_lookup", + }; + + public override string SelfId => BuiltInAgentId.Planner; + + public SqlDictionaryLookupHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + { + } + + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(Utility.SqlDictionaryLookup); + + if (isConvMode && isEnabled) + { + var (prompt, fns) = GetPromptAndFunctions(); + if (!fns.IsNullOrEmpty()) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = fns; + } + else + { + agent.Functions.AddRange(fns); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, List?) GetPromptAndFunctions() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var fns = agent?.Functions?.Where(x => _targetSqlExecutorFunctions.Contains(x.Name))?.ToList(); + + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo(SQL_EXECUTOR_TEMPLATE))?.Content ?? string.Empty; + var dbType = GetDatabaseType(); + var render = _services.GetRequiredService(); + prompt = render.Render(prompt, new Dictionary + { + { "db_type", dbType } + }); + + return (prompt, fns); + } + + private string GetDatabaseType() + { + var settings = _services.GetRequiredService(); + var dbType = "MySQL"; + + if (!string.IsNullOrWhiteSpace(settings?.SqlServerConnectionString)) + { + dbType = "SQL Server"; + } + else if (!string.IsNullOrWhiteSpace(settings?.SqlLiteConnectionString)) + { + dbType = "SQL Lite"; + } + return dbType; + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorUtilityHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs similarity index 61% rename from src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorUtilityHook.cs rename to src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs index 47e0fedc..5ab8e723 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlExecutorUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs @@ -1,9 +1,10 @@ namespace BotSharp.Plugin.SqlDriver.Hooks; -public class SqlExecutorUtilityHook : IAgentUtilityHook +public class SqlUtilityHook : IAgentUtilityHook { public void AddUtilities(List utilities) { utilities.Add(Utility.SqlExecutor); + utilities.Add(Utility.SqlDictionaryLookup); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs index 4f6b6496..504ba9b8 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs @@ -4,15 +4,12 @@ namespace BotSharp.Plugin.SqlDriver.Models; public class LookupDictionary { - [JsonPropertyName("table")] - public string Table { get; set; } - - [JsonPropertyName("keyword")] - public string Keyword { get; set; } + [JsonPropertyName("sql_statement")] + public string SqlStatement { get; set; } [JsonPropertyName("reason")] public string Reason { get; set; } - [JsonPropertyName("columns")] - public string[] Columns { get; set; } + [JsonPropertyName("table")] + public string Table { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index f6aab398..a41490d4 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -21,7 +21,8 @@ public class SqlDriverPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json new file mode 100644 index 00000000..17daab18 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json @@ -0,0 +1,22 @@ +{ + "name": "sql_dictionary_lookup", + "description": "Get id from dictionary table by keyword if tool or solution mentioned this approach", + "parameters": { + "type": "object", + "properties": { + "sql_statement": { + "type": "string", + "description": "sql text" + }, + "reason": { + "type": "string", + "description": "the reason why you need to call sql_dictionary_lookup" + }, + "table": { + "type": "string", + "description": "table name" + } + }, + "required": [ "sql_statement", "reason", "table" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/lookup_dictionary.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/lookup_dictionary.json deleted file mode 100644 index 2123be32..00000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/lookup_dictionary.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "lookup_dictionary", - "description": "Get id from dictionary table by keyword if tool or solution mentioned this approach", - "parameters": { - "type": "object", - "properties": { - "table": { - "type": "string", - "description": "table name" - }, - "keyword": { - "type": "string", - "description": "table name" - }, - "reason": { - "type": "string", - "description": "the reason why you need to call lookup_dictionary" - }, - "columns": { - "type": "array", - "description": "columns", - "items": { - "type": "string", - "description": "column" - } - } - }, - "required": [ "table", "keyword", "reason", "columns" ] - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid new file mode 100644 index 00000000..d9d4c93b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid @@ -0,0 +1,8 @@ +Dictionary Lookup Rules: +===== +Please call function sql_dictionary_lookup if user wants to get or retrieve dictionary/enum/term from data tables. +You must return the id and name/code. + +You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules. +Dictionary table pattern is table name starting with "data_". You can only query the dictionary table without join other non-dictionary tables. +===== \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_select.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_select.json index 956af5ef..bd5b88ce 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_select.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/sql_select.json @@ -1,6 +1,6 @@ { "name": "sql_select", - "description": "Get the specific value from table", + "description": "Execute the reporting related query in the database and get the result", "parameters": { "type": "object", "properties": { @@ -11,46 +11,8 @@ "reason": { "type": "string", "description": "reason" - }, - "table": { - "type": "string", - "description": "related table" - }, - "parameters": { - "type": "array", - "description": "data criteria for the query", - "items": { - "type": "object", - "description": "the name and value for the parameter", - "properties": { - "name": { - "type": "string", - "description": "field name" - }, - "value": { - "type": "string", - "description": "real value inferred by the context" - } - }, - "required": [ "name", "value" ] - } - }, - "return_field": { - "type": "object", - "description": "the name and alias for the return field", - "properties": { - "name": { - "type": "string", - "description": "field in the table" - }, - "alias": { - "type": "string", - "description": "meaningful field alias" - } - }, - "required": [ "name", "value" ] } }, - "required": [ "sql_statement", "reason", "table", "parameters", "return_field" ] + "required": [ "sql_statement", "reason" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/lookup_dictionary.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/lookup_dictionary.liquid rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/templates/sql_dictionary_lookup.liquid