Merge branch 'master' into lida_dev

src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs

@@ -58,4 +58,20 @@ public static class FileUtility
 
         return contentType;
     }
+
+    public static List<string> GetMimeFileTypes(IEnumerable<string> fileTypes)
+    {
+        var provider = new FileExtensionContentTypeProvider();
+        var mimeTypes = provider.Mappings.Where(x => fileTypes.Any(type => x.Value.Contains(type))).Select(x => x.Key).ToList();
+
+        return mimeTypes;
+    }
+
+    public static List<string> GetContentFileTypes(IEnumerable<string> mimeTypes)
+    {
+        var provider = new FileExtensionContentTypeProvider();
+        var mappings = provider.Mappings.Where(x => mimeTypes.Any(type => x.Key.Contains(type))).Select(x => x.Value).ToList();
+
+        return mappings;
+    }
 }

src/Infrastructure/BotSharp.Abstraction/Infrastructures/ICacheService.cs

@@ -5,4 +5,5 @@ public interface ICacheService
     Task<T?> GetAsync<T>(string key);
     Task<object> GetAsync(string key, Type type);
     Task SetAsync<T>(string key, T value, TimeSpan? expiry);
+    Task RemoveAsync(string key);
 }

src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs

@@ -31,7 +31,13 @@ public class SharpCacheAttribute : MoAttribute
         var value = cache.GetAsync(key, context.TaskReturnType).Result;
         if (value != null)
         {
-            context.ReplaceReturnValue(this, value);
+            // check if the cache is out of date
+            var isOutOfDate = IsOutOfDate(context, value).Result;
+
+            if (!isOutOfDate)
+            {
+                context.ReplaceReturnValue(this, value);
+            }
         }
     }
 
@@ -58,6 +64,11 @@ public class SharpCacheAttribute : MoAttribute
         }
     }
 
+    public virtual Task<bool> IsOutOfDate(MethodContext context, object value)
+    {
+        return Task.FromResult(false);
+    }
+
     private string GetCacheKey(SharpCacheSettings settings, MethodContext context)
     {
         var key = settings.Prefix + "-" + context.Method.Name;

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -12,7 +12,7 @@ public interface IVectorDb
     Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
     Task<bool> CreateCollection(string collectionName, int dimension);
     Task<bool> DeleteCollection(string collectionName);
-    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
+    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
     Task<bool> DeleteCollectionAllData(string collectionName);

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs

@@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.VectorStorage.Models;
 public class VectorCollectionData
 {
     public string Id { get; set; }
-    public Dictionary<string, string> Data { get; set; } = new();
+    public Dictionary<string, object> Data { get; set; } = new();
     public double? Score { get; set; }
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs

@@ -6,5 +6,5 @@ public class VectorCreateModel
 {
     public string Text { get; set; }
     public string DataSource { get; set; } = VectorDataSource.Api;
-    public Dictionary<string, string>? Payload { get; set; }
+    public Dictionary<string, object>? Payload { get; set; }
 }

src/Infrastructure/BotSharp.Core/Infrastructures/MemoryCacheService.cs

@@ -32,4 +32,9 @@ public class MemoryCacheService : ICacheService
             AbsoluteExpirationRelativeToNow = expiry
         });
     }
+
+    public async Task RemoveAsync(string key)
+    {
+        _cache.Remove(key);
+    }
 }

src/Infrastructure/BotSharp.Core/Infrastructures/RedisCacheService.cs

@@ -76,4 +76,20 @@ public class RedisCacheService : ICacheService
         var db = redis.GetDatabase();
         await db.StringSetAsync(key, JsonConvert.SerializeObject(value), expiry);
     }
+
+    public async Task RemoveAsync(string key)
+    {
+        if (string.IsNullOrEmpty(_settings.Redis))
+        {
+            return;
+        }
+
+        if (redis == null)
+        {
+            redis = ConnectionMultiplexer.Connect(_settings.Redis);
+        }
+
+        var db = redis.GetDatabase();
+        await db.KeyDeleteAsync(key);
+    }
 }

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -291,8 +291,13 @@ public class UserService : IUserService
 
     private async Task SaveUserTokenExpiresCache(string userId, DateTime expires, int expireInMinutes)
     {
-        var _cacheService = _services.GetRequiredService<ICacheService>();
-        await _cacheService.SetAsync<DateTime>(GetUserTokenExpiresCacheKey(userId), expires, TimeSpan.FromMinutes(expireInMinutes));
+        var config = _services.GetService<IConfiguration>();
+        var enableSingleLogin = bool.Parse(config["Jwt:EnableSingleLogin"] ?? "false");
+        if (enableSingleLogin)
+        {
+            var _cacheService = _services.GetRequiredService<ICacheService>();
+            await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, TimeSpan.FromMinutes(expireInMinutes));
+        }
     }
 
     private string GetUserTokenExpiresCacheKey(string userId)

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs

@@ -11,5 +11,5 @@ public class VectorKnowledgeCreateRequest
     public string DataSource { get; set; } = VectorDataSource.Api;
 
     [JsonPropertyName("payload")]
-    public Dictionary<string, string>? Payload { get; set; }
+    public Dictionary<string, object>? Payload { get; set; }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs

@@ -9,7 +9,7 @@ public class VectorKnowledgeViewModel
     public string Id { get; set; }
 
     [JsonPropertyName("data")]
-    public IDictionary<string, string> Data { get; set; }
+    public IDictionary<string, object> Data { get; set; }
 
     [JsonPropertyName("score")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Files.Utilities;
 using OpenAI.Chat;
+using System.ClientModel;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat;
 
@@ -37,43 +38,67 @@ public class ChatCompletionProvider : IChatCompletion
         var chatClient = client.GetChatClient(_model);
         var (prompt, messages, options) = PrepareOptions(agent, conversations);
 
-        var response = chatClient.CompleteChat(messages, options);
-        var value = response.Value;
-        var reason = value.FinishReason;
-        var content = value.Content;
-        var text = content.FirstOrDefault()?.Text ?? string.Empty;
-
+        ChatCompletion value = default;
         RoleDialogModel responseMessage;
-        if (reason == ChatFinishReason.FunctionCall)
-        {
-            responseMessage = new RoleDialogModel(AgentRole.Function, text)
-            {
-                CurrentAgentId = agent.Id,
-                MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
-                FunctionName = value.FunctionCall.FunctionName,
-                FunctionArgs = value.FunctionCall.FunctionArguments
-            };
 
-            // Somethings LLM will generate a function name with agent name.
-            if (!string.IsNullOrEmpty(responseMessage.FunctionName))
+        try
+        {
+            var response = chatClient.CompleteChat(messages, options);
+            value = response.Value;
+
+            var reason = value.FinishReason;
+            var content = value.Content;
+            var text = content.FirstOrDefault()?.Text ?? string.Empty;
+
+            if (reason == ChatFinishReason.FunctionCall)
             {
-                responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
+                responseMessage = new RoleDialogModel(AgentRole.Function, text)
+                {
+                    CurrentAgentId = agent.Id,
+                    MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
+                    FunctionName = value.FunctionCall.FunctionName,
+                    FunctionArgs = value.FunctionCall.FunctionArguments
+                };
+
+                // Somethings LLM will generate a function name with agent name.
+                if (!string.IsNullOrEmpty(responseMessage.FunctionName))
+                {
+                    responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
+                }
+            }
+            else if (reason == ChatFinishReason.ToolCalls)
+            {
+                var toolCall = value.ToolCalls.FirstOrDefault();
+                responseMessage = new RoleDialogModel(AgentRole.Function, text)
+                {
+                    CurrentAgentId = agent.Id,
+                    MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
+                    FunctionName = toolCall?.FunctionName,
+                    FunctionArgs = toolCall?.FunctionArguments
+                };
+            }
+            else
+            {
+                responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
+                {
+                    CurrentAgentId = agent.Id,
+                    MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
+                };
             }
         }
-        else if (reason == ChatFinishReason.ToolCalls)
+        catch (ClientResultException ex)
         {
-            var toolCall = value.ToolCalls.FirstOrDefault();
-            responseMessage = new RoleDialogModel(AgentRole.Function, text)
+            _logger.LogError(ex, ex.Message);
+            responseMessage = new RoleDialogModel(AgentRole.Assistant, "The response was filtered due to the prompt triggering our content management policy. Please modify your prompt and retry.")
             {
                 CurrentAgentId = agent.Id,
                 MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
-                FunctionName = toolCall?.FunctionName,
-                FunctionArgs = toolCall?.FunctionArguments
             };
         }
-        else
+        catch (Exception ex)
         {
-            responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
+            _logger.LogError(ex, ex.Message);
+            responseMessage = new RoleDialogModel(AgentRole.Assistant, ex.Message)
             {
                 CurrentAgentId = agent.Id,
                 MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
@@ -88,8 +113,8 @@ public class ChatCompletionProvider : IChatCompletion
                 Prompt = prompt,
                 Provider = Provider,
                 Model = _model,
-                PromptCount = response.Value.Usage.InputTokens,
-                CompletionCount = response.Value.Usage.OutputTokens
+                PromptCount = value?.Usage?.InputTokens ?? 0,
+                CompletionCount = value?.Usage?.OutputTokens ?? 0
             });
         }
 

src/Plugins/BotSharp.Plugin.ExcelHandler/BotSharp.Plugin.ExcelHandler.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="Provider\**" />
+    <EmbeddedResource Remove="Provider\**" />
+    <None Remove="Provider\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.8" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
+    <PackageReference Include="NPOI" Version="2.7.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\" />
+    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
+    <ProjectReference Include="..\BotSharp.Plugin.SqlDriver\BotSharp.Plugin.SqlDriver.csproj" />
+  </ItemGroup>
+
+</Project>

src/Plugins/BotSharp.Plugin.ExcelHandler/Enums/UtilityName.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.ExcelHandler.Enums;
+
+public class UtilityName
+{
+    public const string ExcelHandler = "excel-handler";
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/ExcelHandlerPlugin.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Plugins;
+using BotSharp.Abstraction.Settings;
+using BotSharp.Plugin.ExcelHandler.Helpers;
+using BotSharp.Plugin.ExcelHandler.Hooks;
+using BotSharp.Plugin.ExcelHandler.Settings;
+using Microsoft.Extensions.Configuration;
+
+namespace BotSharp.Plugin.ExcelHandler;
+
+public class ExcelHandlerPlugin : IBotSharpPlugin
+{
+    public string Id => "c56a8e29-b16f-4d75-8766-8309342130cb";
+    public string Name => "Excel Handler";
+    public string Description => "Load data from excel file and transform it into a list of JSON format.";
+
+    public void RegisterDI(IServiceCollection services, IConfiguration config)
+    {
+        services.AddScoped(provider =>
+        {
+            var settingService = provider.GetRequiredService<ISettingService>();
+            return settingService.Bind<ExcelHandlerSettings>("ExcelHandler");
+        });
+
+        services.AddScoped<IAgentUtilityHook, ExcelHandlerUtilityHook>();
+        services.AddScoped<IAgentHook, ExcelHandlerHook>();
+        services.AddScoped<IDbHelpers, DbHelpers>();
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs

@@ -0,0 +1,427 @@
+using BotSharp.Abstraction.Files.Enums;
+using BotSharp.Abstraction.Files.Models;
+using BotSharp.Abstraction.Files.Utilities;
+using BotSharp.Abstraction.Utilities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Data.Sqlite;
+using NPOI.SS.UserModel;
+using NPOI.XSSF.UserModel;
+using static Microsoft.EntityFrameworkCore.DbLoggerCategory.Database;
+using Microsoft.Extensions.Primitives;
+using BotSharp.Plugin.ExcelHandler.Helpers;
+using System.Data.SqlTypes;
+using BotSharp.Plugin.ExcelHandler.Models;
+using NPOI.SS.Formula.Functions;
+using System.Linq.Dynamic.Core;
+using Microsoft.EntityFrameworkCore.Metadata.Internal;
+
+namespace BotSharp.Plugin.ExcelHandler.Functions;
+
+public class HandleExcelRequestFn : IFunctionCallback
+{
+    public string Name => "handle_excel_request";
+    public string Indication => "Handling excel request";
+
+    private readonly IServiceProvider _serviceProvider;
+    private readonly IFileStorageService _fileStorage;
+    private readonly ILogger<HandleExcelRequestFn> _logger;
+    private readonly BotSharpOptions _options;
+    private readonly IDbHelpers _dbHelpers;
+
+    private HashSet<string> _excelMimeTypes;
+    private double _excelRowSize = 0;
+    private double _excelColumnSize = 0;
+    private string _tableName = "tempTable";
+    private string _currentFileName = string.Empty;
+    private List<string> _headerColumns = new List<string>();
+    private List<string> _columnTypes = new List<string>();
+
+    public HandleExcelRequestFn(
+        IServiceProvider serviceProvider,
+        IFileStorageService fileStorage,
+        ILogger<HandleExcelRequestFn> logger,
+        BotSharpOptions options,
+        IDbHelpers dbHelpers
+        )
+    {
+        _serviceProvider = serviceProvider;
+        _fileStorage = fileStorage;
+        _logger = logger;
+        _options = options;
+        _dbHelpers = dbHelpers;
+    }
+
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, _options.JsonSerializerOptions);
+        var conv = _serviceProvider.GetRequiredService<IConversationService>();
+
+        if (_excelMimeTypes.IsNullOrEmpty())
+        {
+            _excelMimeTypes = FileUtility.GetMimeFileTypes(new List<string> { "excel", "spreadsheet" }).ToHashSet<string>();
+        }
+
+        var dialogs = conv.GetDialogHistory();
+        var isExcelExist = AssembleFiles(conv.ConversationId, dialogs);
+        if (!isExcelExist)
+        {
+            message.Content = "No excel files found in the conversation";
+            return true;
+        }
+        if (!DeleteTable())
+        {
+            message.Content = "Failed to clear existing tables. Please manually delete all existing tables";
+        }
+        else
+        {
+            var resultList = GetResponeFromDialogs(dialogs);
+            message.Content = GenerateSqlExecutionSummary(resultList);
+        }
+        message.StopCompletion = true;
+        return true;
+    }
+
+
+    #region Private Methods
+    private bool AssembleFiles(string convId, List<RoleDialogModel> dialogs)
+    {
+        if (dialogs.IsNullOrEmpty()) return false;
+
+        var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
+        var contentType = FileUtility.GetContentFileTypes(mimeTypes: _excelMimeTypes);
+        var excelMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, contentType);
+
+        if (excelMessageFiles.IsNullOrEmpty()) return false;
+
+        dialogs.ForEach(dialog => {
+            var found = excelMessageFiles.Where(y => y.MessageId == dialog.MessageId).ToList();
+            if (found.IsNullOrEmpty()) return;
+
+            dialog.Files = found.Select(x => new BotSharpFile
+            {
+                ContentType = x.ContentType,
+                FileUrl = x.FileUrl,
+                FileStorageUrl = x.FileStorageUrl
+            }).ToList();
+        });
+        return true;
+    }
+
+    private List<SqlContextOut> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
+    {
+        var dialog = dialogs.Last(x => !x.Files.IsNullOrEmpty());
+        var sqlCommandList = new List<SqlContextOut>();
+        foreach (var file in dialog.Files)
+        {
+            if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue;
+
+            string extension = Path.GetExtension(file.FileStorageUrl);
+            if (!_excelMimeTypes.Contains(extension)) continue;
+            
+            _currentFileName = Path.GetFileName(file.FileStorageUrl);
+
+            var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
+            var workbook = ConvertToWorkBook(bytes);
+            var currentCommandList = WriteExcelDataToDB(workbook);
+            sqlCommandList.AddRange(currentCommandList);
+        }
+        return sqlCommandList;
+    }
+
+    private List<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
+    {
+        var numTables = workbook.NumberOfSheets;
+        var commandList = new List<SqlContextOut>();
+
+        for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
+        {
+            var commandResult = new SqlContextOut();
+            ISheet sheet = workbook.GetSheetAt(sheetIdx);
+            var (isCreateSuccess, message) = SqlCreateTableFn(sheet);
+
+            if (!isCreateSuccess)
+            {
+                commandResult = new SqlContextOut
+                {
+                    isSuccessful = isCreateSuccess,
+                    Message = message,
+                    FileName = _currentFileName
+                };
+                commandList.Add(commandResult);
+                continue;
+            }
+            var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
+            commandResult = new SqlContextOut
+            {
+                isSuccessful = isInsertSuccess,
+                Message = insertMessage,
+                FileName = _currentFileName
+            };
+            commandList.Add(commandResult);
+        }
+        return commandList;
+    }
+
+    private bool DeleteTable()
+    {
+        try
+        {
+            DeleteTableSqlQuery();
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to delete table");
+            return false;
+        }
+    }
+
+    private (bool, string) SqlInsertDataFn(ISheet sheet)
+    {
+        try
+        {
+            string dataSql = ParseSheetData(sheet);
+            string insertDataSql = ProcessInsertSqlQuery(dataSql);
+            ExecuteSqlQueryForInsertion(insertDataSql);
+
+            return (true, $"{_currentFileName}: \r\n `**{_excelRowSize}**` data have been successfully stored into `{_tableName}` table");
+        }
+        catch (Exception ex)
+        {
+            return (false, $"{_currentFileName}: Failed to parse excel data into `{_tableName}` table. ####Error: {ex.Message}");
+        }
+    }
+
+    private string GenerateSqlExecutionSummary(List<SqlContextOut> messageList)
+    {
+        var stringBuilder = new StringBuilder();
+        if (messageList.Any(x => x.isSuccessful))
+        {
+            stringBuilder.Append("---Success---");
+            stringBuilder.Append("\r\n");
+            foreach (var message in messageList.Where(x => x.isSuccessful))
+            {
+                stringBuilder.Append(message.Message);
+                string tableSchemaInfo = GenerateTableSchema();
+                stringBuilder.Append(tableSchemaInfo);
+                stringBuilder.Append("\r\n\r\n");
+            }
+        }
+        if (messageList.Any(x => !x.isSuccessful))
+        {
+            stringBuilder.Append("---Failed---");
+            stringBuilder.Append("\r\n");
+            foreach (var message in messageList.Where(x => !x.isSuccessful))
+            {
+                stringBuilder.Append(message.Message);
+                stringBuilder.Append("\r\n");
+            }
+        }
+        return stringBuilder.ToString();
+    }
+
+    private string GenerateTableSchema()
+    {
+        var sb = new StringBuilder();
+        sb.Append($"\nTable Schema for `{_tableName}`:");
+        sb.Append("\n");
+        sb.Append($"cid | name       | type      ");
+        sb.Append("\n");
+        //sb.Append("----|------------|------------");
+        for (int i = 0; i < _excelColumnSize; i++)
+        {
+            sb.Append($"{i,-4}   | {_headerColumns[i],-10} | {_columnTypes[i],-10}");
+            sb.Append("\n");
+        }
+        return sb.ToString();
+    }
+
+    private IWorkbook ConvertToWorkBook(byte[] bytes)
+    {
+        IWorkbook workbook;
+        using (var fileStream = new MemoryStream(bytes))
+        {
+            workbook = new XSSFWorkbook(fileStream);
+        }
+        return workbook;
+    }
+
+    private (bool, string) SqlCreateTableFn(ISheet sheet)
+    {
+        try
+        {
+            _tableName = sheet.SheetName;
+            _headerColumns = ParseSheetColumn(sheet);
+            string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
+            ExecuteSqlQueryForInsertion(createTableSql);
+            return (true, $"{_tableName} has been successfully created.");
+        }
+        catch (Exception ex)
+        {
+            return (false, ex.Message);
+        }
+    }
+
+    private List<string> ParseSheetColumn(ISheet sheet)
+    {
+        if (sheet.PhysicalNumberOfRows < 2)
+            throw new Exception("No data found in the excel file");
+
+        _excelRowSize = sheet.PhysicalNumberOfRows - 1;
+        var headerRow = sheet.GetRow(0);
+        var headerColumn = headerRow.Cells.Select(x => x.StringCellValue.Replace(" ", "_")).ToList();
+        _excelColumnSize = headerColumn.Count;
+        return headerColumn;
+    }
+
+    private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null)
+    {
+        var createTableSql = $"CREATE TABLE if not exists {tableName} ( Id INTEGER PRIMARY KEY AUTOINCREMENT, ";
+
+        _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "TEXT").ToList() : columnTypes;
+
+        headerColumns = headerColumns.Select((x, i) => $"`{x.Replace(" ", "_")}`" + $" {_columnTypes[i]}").ToList();
+        createTableSql += string.Join(", ", headerColumns);
+        createTableSql += ");";
+        return createTableSql;
+    }
+
+
+    private void ExecuteSqlQueryForInsertion(string query)
+    {
+        var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
+        var inMemoryDbConnection = _dbHelpers.GetInMemoryDbConnection();
+
+        physicalDbConnection.BackupDatabase(inMemoryDbConnection, "main", "main");
+        physicalDbConnection.Close();
+
+        using (var command = new SqliteCommand())
+        {
+            command.CommandText = query;
+            command.Connection = inMemoryDbConnection;
+            command.ExecuteNonQuery();
+        }
+        inMemoryDbConnection.BackupDatabase(physicalDbConnection);
+    }
+
+    private void DeleteTableSqlQuery()
+    {
+        string deleteTableSql = @"
+            SELECT
+                name
+            FROM
+                sqlite_schema
+            WHERE
+                type = 'table' AND
+                name NOT LIKE 'sqlite_%'
+        ";
+        var physicalDbConnection = _dbHelpers.GetPhysicalDbConnection();
+        using var selectCmd = new SqliteCommand(deleteTableSql, physicalDbConnection);
+        using var reader = selectCmd.ExecuteReader();
+        if (reader.HasRows)
+        {
+            var dropTableQueries = new List<string>();
+            while (reader.Read())
+            {
+                string tableName = reader.GetString(0);
+                var dropTableSql = $"DROP TABLE IF EXISTS '{tableName}'";
+                dropTableQueries.Add(dropTableSql);
+            }
+            dropTableQueries.ForEach(query =>
+            {
+                using var dropTableCommand = new SqliteCommand(query, physicalDbConnection);
+                dropTableCommand.ExecuteNonQuery();
+            });
+        }
+        physicalDbConnection.Close();
+    }
+
+    private string ParseSheetData(ISheet singleSheet)
+    {
+        var stringBuilder = new StringBuilder();
+
+        for (int rowIdx = 1; rowIdx < _excelRowSize + 1; rowIdx++)
+        {
+            IRow row = singleSheet.GetRow(rowIdx);
+            stringBuilder.Append('(');
+            for (int colIdx = 0; colIdx < _excelColumnSize; colIdx++)
+            {
+                var cell = row.GetCell(colIdx, MissingCellPolicy.CREATE_NULL_AS_BLANK);
+
+                switch (cell.CellType)
+                {
+                    case CellType.String:
+                        //if (cell.DateCellValue == null || cell.DateCellValue == DateTime.MinValue)
+                        //{
+                        //    sb.Append($"{cell.DateCellValue}");
+                        //    break;
+                        //}
+                        stringBuilder.Append($"'{cell.StringCellValue.Replace("'", "''")}'");
+                        break;
+                    case CellType.Numeric:
+                        stringBuilder.Append($"{cell.NumericCellValue}");
+                        break;
+                    case CellType.Blank:
+                        stringBuilder.Append($"null");
+                        break;
+                    default:
+                        stringBuilder.Append($"'{cell.StringCellValue}'");
+                        break;
+                }
+
+                if (colIdx != (_excelColumnSize - 1))
+                {
+                    stringBuilder.Append(", ");
+                }
+            }
+            stringBuilder.Append(')');
+            stringBuilder.Append(rowIdx == _excelRowSize ? ';' : ", \r\n");
+        }
+        return stringBuilder.ToString();
+    }
+
+    private string ProcessInsertSqlQuery(string dataSql)
+    {
+        var wrapUpCols = _headerColumns.Select(x => $"`{x}`").ToList();
+        var transferedCols = '('+ string.Join(',', wrapUpCols) + ')';
+        string insertSqlQuery = $"Insert into {_tableName} {transferedCols} Values {dataSql}";
+        return insertSqlQuery;
+    }
+
+
+    [Obsolete("This method is not used anymore", true)]
+    private (bool, string) ParseExcelDataToSqlString(ISheet sheet)
+    {
+        try
+        {
+            if (_headerColumns.IsNullOrEmpty())
+            {
+                _headerColumns = ParseSheetColumn(sheet);
+                string createTableSql = CreateDBTableSqlString(_tableName, _headerColumns, null);
+                ExecuteSqlQueryForInsertion(createTableSql);
+            }
+
+            string dataSql = ParseSheetData(sheet);
+            string insertDataSql = ProcessInsertSqlQuery(dataSql);
+            ExecuteSqlQueryForInsertion(insertDataSql);
+            return (true, $"{_currentFileName}: {_excelRowSize} data have been successfully stored into {_tableName}");
+        }
+        catch (Exception ex)
+        {
+            return (false, $"{_currentFileName}: Failed to parse excel data to sql string. Error: {ex.Message}");
+        }
+    }
+
+    [Obsolete("This method is not used anymore", true)]
+    private bool IsHeaderColumnEqual(List<string> headerColumn)
+    {
+        if (_headerColumns.IsNullOrEmpty() || _headerColumns.Count != headerColumn.Count)
+        {
+            return false;
+        }
+
+        return new HashSet<string>(headerColumn).SetEquals(_headerColumns);
+    }
+    #endregion
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/DbHelpers.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Data.Sqlite;
+using BotSharp.Plugin.SqlDriver.Models;
+using BotSharp.Plugin.SqlHero.Settings;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers;
+
+public class DbHelpers : IDbHelpers
+{
+    private string _dbFilePath = string.Empty;
+    private SqliteConnection inMemoryDbConnection = null;
+
+    private readonly IServiceProvider _services;
+
+    public DbHelpers(IServiceProvider service)
+    {
+        _services = service;
+    }
+
+    public SqliteConnection GetInMemoryDbConnection()
+    {
+        if (inMemoryDbConnection == null)
+        {
+            inMemoryDbConnection = new SqliteConnection("Data Source=:memory:;Mode=ReadWrite");
+            inMemoryDbConnection.Open();
+            return inMemoryDbConnection;
+        }
+        return inMemoryDbConnection;
+    }
+
+    public SqliteConnection GetPhysicalDbConnection()
+    {
+        if (string.IsNullOrEmpty(_dbFilePath))
+        {
+            var settingService = _services.GetRequiredService<SqlDriverSetting>();
+            _dbFilePath = settingService.SqlLiteConnectionString;
+        }
+
+        var dbConnection = new SqliteConnection($"Data Source={_dbFilePath};Mode=ReadWrite");
+        dbConnection.Open();
+        return dbConnection;
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Helpers/IDbHelpers.cs

@@ -0,0 +1,9 @@
+using Microsoft.Data.Sqlite;
+
+namespace BotSharp.Plugin.ExcelHandler.Helpers;
+
+public interface IDbHelpers
+{
+    SqliteConnection GetPhysicalDbConnection();
+    SqliteConnection GetInMemoryDbConnection();
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerHook.cs

@@ -0,0 +1,58 @@
+namespace BotSharp.Plugin.ExcelHandler.Hooks;
+
+public class ExcelHandlerHook : AgentHookBase, IAgentHook
+{
+    private const string HANDLER_EXCEL = "handle_excel_request";
+
+    public override string SelfId => string.Empty;
+
+    public ExcelHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
+    {
+    }
+
+    public override void OnAgentLoaded(Agent agent)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+        var isConvMode = conv.IsConversationMode();
+        var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.ExcelHandler);
+
+        if (isEnabled && isConvMode)
+        {
+            AddUtility(agent, HANDLER_EXCEL);
+        }
+
+        base.OnAgentLoaded(agent);
+    }
+
+    private void AddUtility(Agent agent, string functionName)
+    {
+        var (prompt, fn) = GetPromptAndFunction(functionName);
+
+        if (fn != null)
+        {
+            if (!string.IsNullOrWhiteSpace(prompt))
+            {
+                agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
+            }
+
+            if (agent.Functions == null)
+            {
+                agent.Functions = new List<FunctionDef> { fn };
+            }
+            else
+            {
+                agent.Functions.Add(fn);
+            }
+        }
+    }
+
+    private (string, FunctionDef?) GetPromptAndFunction(string functionName)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
+        var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
+        var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
+        return (prompt, loadAttachmentFn);
+    }
+}
+

src/Plugins/BotSharp.Plugin.ExcelHandler/Hooks/ExcelHandlerUtilityHook.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Plugin.ExcelHandler.Hooks;
+
+public class ExcelHandlerUtilityHook : IAgentUtilityHook
+{
+    public void AddUtilities(List<string> utilities)
+    {
+        utilities.Add(UtilityName.ExcelHandler);
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextIn.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.ExcelHandler.LlmContexts
+{
+    public class LlmContextIn
+    {
+        [JsonPropertyName("user_request")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+        public string? UserRequest { get; set; }
+
+        [JsonPropertyName("is_need_processing")]
+        public bool IsNeedProcessing { get; set; }
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/LlmContexts/LlmContextOut.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Plugin.ExcelHandler.LlmContexts;
+
+public class LlmContextOut
+{
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Models/SqlContextOut.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Plugin.ExcelHandler.Models;
+
+public class SqlContextOut
+{
+    public bool isSuccessful { get; set; } 
+    public string Message { get; set; }
+    public string FileName { get; set; }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Settings/ExcelHandlerSettings.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.ExcelHandler.Settings;
+
+public class ExcelHandlerSettings
+{
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/Using.cs

@@ -0,0 +1,26 @@
+global using System;
+global using System.Collections.Generic;
+global using System.Linq;
+global using System.Text;
+global using System.Text.Json;
+global using System.Threading.Tasks;
+
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.Functions;
+global using BotSharp.Abstraction.Options;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.Agents.Settings;
+global using BotSharp.Abstraction.Functions.Models;
+global using BotSharp.Abstraction.Repositories;
+global using BotSharp.Abstraction.Utilities;
+
+global using BotSharp.Plugin.ExcelHandler.Enums;
+global using BotSharp.Plugin.ExcelHandler.LlmContexts;
+
+global using Microsoft.Extensions.Logging;
+global using Microsoft.Extensions.DependencyInjection;
+

src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_excel_request.json

@@ -0,0 +1,22 @@
+{
+    "name": "handle_excel_request",
+    "description": "If the user requests to read/load data from excel/csv files, you need to call this function to load the data from excel/csv files and transform into JSON format data",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "user_request": {
+                "type": "string",
+                "description": "The request posted by user, which is related to read/load data based on the inputted excel/csv file"
+            },
+            "is_need_processing": {
+                "type": "boolean",
+                "description": "If the user request is to do some processing on the data, set this value to true, otherwise, set it to false"
+            },
+            "table_name": {
+                "type": "string",
+                "description": "if the user request to store data into Database table, assign the table name to this value"
+            }
+        },
+        "required": [ "user_request" ]
+    }
+}

src/Plugins/BotSharp.Plugin.ExcelHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_excel_request.fn.liquid

@@ -0,0 +1 @@
+Please call handle_excel_request if user wants to load the data from a excel/csv file.

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs

@@ -24,7 +24,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback
         var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel
         {
             Text = args.Question,
-            Payload = new Dictionary<string, string>
+            Payload = new Dictionary<string, object>
             {
                 { KnowledgePayloadName.Answer, args.Answer }
             }

src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

@@ -59,7 +59,7 @@ public class MemoryVectorDb : IVectorDb
                         .Take(limit)
                         .Select(i => new VectorCollectionData
                         {
-                            Data = new Dictionary<string, string> { { "text", _vectors[collectionName][i].Text } },
+                            Data = new Dictionary<string, object> { { "text", _vectors[collectionName][i].Text } },
                             Score = similarities[i],
                             Vector = withVector ? _vectors[collectionName][i].Vector : null,
                         })
@@ -68,7 +68,7 @@ public class MemoryVectorDb : IVectorDb
         return await Task.FromResult(results);
     }
 
-    public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
+    public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
     {
         _vectors[collectionName].Add(new VecRecord
         {

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

@@ -398,7 +398,7 @@ public partial class KnowledgeService
         var vectorDb = GetVectorDb();
         var textEmbedding = GetTextEmbedding(collectionName);
 
-        var payload = new Dictionary<string, string>
+        var payload = new Dictionary<string, object>
         {
             { KnowledgePayloadName.DataSource, vectorDataSource },
             { KnowledgePayloadName.FileId, fileId.ToString() },

src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

@@ -196,7 +196,7 @@ public partial class KnowledgeService
                 withPayload: true);
             if (!found.IsNullOrEmpty())
             {
-                if (found.First().Data["text"] == update.Text)
+                if (found.First().Data["text"].ToString() == update.Text)
                 {
                     // Only update payload
                     return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload);

src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs

@@ -19,23 +19,23 @@ public class PrimaryStagePlanFn : IFunctionCallback
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var state = _services.GetRequiredService<IConversationStateService>();
-        var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
-        var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
+        // var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
+        // var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
 
         state.SetState("max_tokens", "4096");
         var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
-        var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        // var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
 
         // Get knowledge from vectordb
         var hooks = _services.GetServices<IKnowledgeHook>();
         var knowledges = new List<string>();
         foreach (var question in task.Questions)
         {
-            var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions
+            /*var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions
             {
                 Confidence = 0.4f
             });
-            knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));
+            knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));*/
             
             foreach (var hook in hooks)
             {
@@ -43,6 +43,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
                 knowledges.AddRange(k);
             }
         }
+        knowledges = knowledges.Distinct().ToList();
 
         // Get first stage planning prompt
         var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);

src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Qdrant.Client" Version="1.10.0" />
+    <PackageReference Include="Qdrant.Client" Version="1.11.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -142,7 +142,13 @@ public class QdrantDb : IVectorDb
         var points = response?.Result?.Select(x => new VectorCollectionData
         {
             Id = x.Id?.Uuid ?? string.Empty,
-            Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue),
+            Data = x.Payload.ToDictionary(p => p.Key, p => p.Value.KindCase switch
+            {
+                Value.KindOneofCase.StringValue => p.Value.StringValue,
+                Value.KindOneofCase.BoolValue => p.Value.BoolValue,
+                Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                _ => new object()
+            }),
             Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
         })?.ToList() ?? new List<VectorCollectionData>();
 
@@ -175,12 +181,18 @@ public class QdrantDb : IVectorDb
         return points.Select(x => new VectorCollectionData
         {
             Id = x.Id?.Uuid ?? string.Empty,
-            Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(),
+            Data = x.Payload?.ToDictionary(p => p.Key, p => p.Value.KindCase switch 
+            { 
+                Value.KindOneofCase.StringValue => p.Value.StringValue,
+                Value.KindOneofCase.BoolValue => p.Value.BoolValue,
+                Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                _ => new object()
+            }) ?? new(),
             Vector = x.Vectors?.Vector?.Data?.ToArray()
         });
     }
 
-    public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
+    public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
     {
         // Insert vectors
         var point = new PointStruct()
@@ -200,7 +212,42 @@ public class QdrantDb : IVectorDb
         {
             foreach (var item in payload)
             {
-                point.Payload[item.Key] = item.Value;
+                if (item.Value is string str)
+                {
+                    point.Payload[item.Key] = str;
+                }
+                else if (item.Value is bool b)
+                {
+                    point.Payload[item.Key] = b;
+                }
+                else if (item.Value is byte int8)
+                {
+                    point.Payload[item.Key] = int8;
+                }
+                else if (item.Value is short int16)
+                {
+                    point.Payload[item.Key] = int16;
+                }
+                else if (item.Value is int int32)
+                {
+                    point.Payload[item.Key] = int32;
+                }
+                else if (item.Value is long int64)
+                {
+                    point.Payload[item.Key] = int64;
+                }
+                else if (item.Value is float f32)
+                {
+                    point.Payload[item.Key] = f32;
+                }
+                else if (item.Value is double f64)
+                {
+                    point.Payload[item.Key] = f64;
+                }
+                else if (item.Value is DateTime dt)
+                {
+                    point.Payload[item.Key] = dt.ToUniversalTime().ToString("o");
+                }
             }
         }
 
@@ -241,7 +288,13 @@ public class QdrantDb : IVectorDb
         results = points.Select(x => new VectorCollectionData
         {
             Id = x.Id.Uuid,
-            Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue),
+            Data = x.Payload.ToDictionary(p => p.Key, p => p.Value.KindCase switch
+            {
+                Value.KindOneofCase.StringValue => p.Value.StringValue,
+                Value.KindOneofCase.BoolValue => p.Value.BoolValue,
+                Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
+                _ => new object()
+            }),
             Score = x.Score,
             Vector = x.Vectors?.Vector?.Data?.ToArray()
         }).ToList();

src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs

@@ -69,7 +69,7 @@ public class DbKnowledgeService
                     await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel
                     {
                         Text = item.Question,
-                        Payload = new Dictionary<string, string>
+                        Payload = new Dictionary<string, object>
                         {
                             { KnowledgePayloadName.Answer, item.Answer }
                         }