From 57bd04bf7b48dc31204c3c21f7109132b0eaa959 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 16 Sep 2025 19:20:53 -0500 Subject: [PATCH] refine file handling --- .../Conversations/Models/RoleDialogModel.cs | 7 ++ .../Files/FileCoreSettings.cs | 7 -- .../Files/IFileStorageService.cs | 4 +- .../Files/Models/MessageFileOptions.cs | 6 ++ .../Settings/KnowledgeBaseSettings.cs | 5 -- .../Models/LlmConfigBase.cs | 25 +++--- .../Settings/SettingBase.cs | 6 ++ .../BotSharp.Abstraction/Using.cs | 3 +- .../FileInstructService.SelectFile.cs | 29 ++++--- .../LocalFileStorageService.Conversation.cs | 86 ++++++------------- .../Functions/EditImageFn.cs | 7 +- .../Functions/GenerateImageFn.cs | 12 +-- .../Functions/ReadImageFn.cs | 23 +++-- .../Functions/ReadPdfFn.cs | 55 ++++++++++-- .../Settings/FileHandlerSettings.cs | 24 +++--- .../ImageCompletionProvider.Generation.cs | 18 ---- .../TencentCosService.Conversation.cs | 74 ++++++---------- src/WebStarter/appsettings.json | 25 +++--- .../Core/NullFileStorageService.cs | 2 +- 19 files changed, 199 insertions(+), 219 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 79ba3c64..33678ecc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -110,6 +110,7 @@ public class RoleDialogModel : ITrackableMessage /// /// Files to be used in conversation /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Files { get; set; } /// @@ -133,6 +134,12 @@ public class RoleDialogModel : ITrackableMessage public bool IsStreaming { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool IsFromUser => Role == AgentRole.User; + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model; + public RoleDialogModel() { } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index a1a7730f..1d66c80a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,12 +5,5 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public SettingBase? Pdf2TextConverter { get; set; } - public SettingBase? Pdf2ImageConverter { get; set; } public SettingBase? ImageConverter { get; set; } -} - -public class SettingBase -{ - public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index db6062af..716b86e2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -19,12 +19,12 @@ public interface IFileStorageService #region Conversation /// - /// Get the message file screenshots for specific content types, e.g., pdf + /// Get the message file screenshots for pdf /// /// /// /// - Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs index 98c2c4df..a647ad61 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs @@ -12,3 +12,9 @@ public class MessageFileOptions /// public IEnumerable? ContentTypes { get; set; } } + + +public class MessageFileScreenshotOptions : MessageFileOptions +{ + public string ImageConvertProvider { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index b589e5a6..b9c01612 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -20,9 +20,4 @@ public class KnowledgeTextEmbeddingSetting : SettingBase { public string Model { get; set; } public int Dimension { get; set; } -} - -public class SettingBase -{ - public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs index 59edd20d..1d88942c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -1,17 +1,7 @@ namespace BotSharp.Abstraction.Models; -public class LlmConfigBase +public class LlmConfigBase : LlmBase { - /// - /// Llm provider - /// - public string? LlmProvider { get; set; } - - /// - /// Llm model - /// - public string? LlmModel { get; set; } - /// /// Llm maximum output tokens /// @@ -22,3 +12,16 @@ public class LlmConfigBase /// public string? ReasoningEffortLevel { get; set; } } + +public class LlmBase +{ + /// + /// Llm provider + /// + public string? LlmProvider { get; set; } + + /// + /// Llm model + /// + public string? LlmModel { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs b/src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs new file mode 100644 index 00000000..7def8d3f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Settings; + +public class SettingBase +{ + public string Provider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 38a6dca0..d2077537 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -20,4 +20,5 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.Crontab.Models; -global using BotSharp.Abstraction.MCP.Models; \ No newline at end of file +global using BotSharp.Abstraction.MCP.Models; +global using BotSharp.Abstraction.Settings; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 18b8d0b3..a2de1940 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -86,19 +86,18 @@ public partial class FileInstructService var render = _services.GetRequiredService(); var db = _services.GetRequiredService(); + // Handle dialogs and files + var innerDialogs = (dialogs ?? []).ToList(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; + innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); + + if (options.IsAttachFiles) + { + AssembleMessageFiles(innerDialogs, files, options); + } + try { - // Handle dialogs and files - var innerDialogs = (dialogs ?? []).ToList(); - var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; - innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); - - if (options.IsAttachFiles) - { - AssembleMessageFiles(innerDialogs, files, options); - } - - // Handle instruction var promptMessages = innerDialogs.Select(x => { @@ -176,6 +175,10 @@ public partial class FileInstructService _logger.LogWarning(ex, $"Error when selecting files."); return []; } + finally + { + innerDialogs.ForEach(x => x.Files = null); + } } private void AssembleMessageFiles(IEnumerable dialogs, IEnumerable files, SelectFileOptions options) @@ -196,7 +199,7 @@ public partial class FileInstructService continue; } - var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User); + var userMsg = group.FirstOrDefault(x => x.IsFromUser); if (userMsg != null) { var userFiles = found.Where(x => x.FileSource == FileSource.User); @@ -211,7 +214,7 @@ public partial class FileInstructService }).ToList(); } - var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant); + var botMsg = group.LastOrDefault(x => x.IsFromAssistant); if (botMsg != null) { var botFiles = found.Where(x => x.FileSource == FileSource.Bot); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index f695a5b9..6050edcf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,16 +6,17 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) + || messageIds.IsNullOrEmpty() + || options.Sources.IsNullOrEmpty()) { return files; } - var source = FileSource.User; - var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); + var baseUrl = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { @@ -24,27 +25,30 @@ public partial class LocalFileStorageService continue; } - var dir = Path.Combine(pathPrefix, messageId, FileSource.User); - if (!ExistDirectory(dir)) + foreach (var source in options.Sources) { - continue; - } - - foreach (var subDir in Directory.GetDirectories(dir)) - { - var file = Directory.GetFiles(subDir).FirstOrDefault(); - if (file == null) + var dir = Path.Combine(baseUrl, messageId, source); + if (!ExistDirectory(dir)) { continue; } - var screenshots = await GetScreenshots(file, subDir, messageId, source); - if (screenshots.IsNullOrEmpty()) + foreach (var subDir in Directory.GetDirectories(dir)) { - continue; - } + var file = Directory.GetFiles(subDir).FirstOrDefault(); + if (file == null) + { + continue; + } - files.AddRange(screenshots); + var screenshots = await GetScreenshotsAsync(file, subDir, messageId, source, options); + if (screenshots.IsNullOrEmpty()) + { + continue; + } + + files.AddRange(screenshots); + } } } return files; @@ -283,40 +287,9 @@ public partial class LocalFileStorageService return dir; } - private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) + private async Task> ConvertPdfToImagesAsync(string pdfLoc, string imageLoc, MessageFileScreenshotOptions options) { - if (dialogs.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - if (offset.HasValue && offset < 1) - { - offset = 1; - } - - var messageIds = new List(); - if (offset.HasValue) - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - } - - return messageIds; - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) - { - var converters = _services.GetServices(); - if (converters.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options.ImageConvertProvider); if (converter == null) { return Enumerable.Empty(); @@ -325,14 +298,7 @@ public partial class LocalFileStorageService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IImageConverter? GetPdf2ImageConverter() - { - var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); - return converter; - } - - private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + private async Task> GetScreenshotsAsync(string file, string parentDir, string messageId, string source, MessageFileScreenshotOptions options) { var files = new List(); @@ -362,7 +328,7 @@ public partial class LocalFileStorageService } else if (contentType == MediaTypeNames.Application.Pdf) { - var images = await ConvertPdfToImages(file, screenshotDir); + var images = await ConvertPdfToImagesAsync(file, screenshotDir, options); foreach (var image in images) { var fileName = Path.GetFileNameWithoutExtension(image); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 4388485b..670261ac 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -150,7 +150,6 @@ public class EditImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_edit_llm_provider"); var model = state.GetState("image_edit_llm_provider"); @@ -160,8 +159,8 @@ public class EditImageFn : IFunctionCallback return (provider, model); } - provider = fileSettings?.Image?.Edit?.LlmProvider; - model = fileSettings?.Image?.Edit?.LlmModel; + provider = _settings?.Image?.Edit?.LlmProvider; + model = _settings?.Image?.Edit?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -197,7 +196,7 @@ public class EditImageFn : IFunctionCallback private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { - var provider = _settings?.ImageConverter?.Provider; + var provider = _settings?.Image?.Edit?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 2e13a0c0..33fa8e81 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; - namespace BotSharp.Plugin.FileHandler.Functions; public class GenerateImageFn : IFunctionCallback @@ -9,6 +7,7 @@ public class GenerateImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; private Agent _agent; private string _conversationId; @@ -16,10 +15,12 @@ public class GenerateImageFn : IFunctionCallback public GenerateImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -113,7 +114,6 @@ public class GenerateImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_generate_llm_provider"); var model = state.GetState("image_generate_llm_model"); @@ -123,8 +123,8 @@ public class GenerateImageFn : IFunctionCallback return (provider, model); } - provider = fileSettings?.Image?.Generation?.LlmProvider; - model = fileSettings?.Image?.Generation?.LlmModel; + provider = _settings?.Image?.Generation?.LlmProvider; + model = _settings?.Image?.Generation?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 29d6ca14..05848c99 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -9,6 +9,7 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; private readonly IEnumerable _imageContentTypes = new List { @@ -18,10 +19,12 @@ public class ReadImageFn : IFunctionCallback public ReadImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -53,6 +56,7 @@ public class ReadImageFn : IFunctionCallback var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs); var response = await GetChatCompletion(agent, dialogs); + dialogs.ForEach(x => x.Files = null); message.Content = response; return true; } @@ -77,7 +81,17 @@ public class ReadImageFn : IFunctionCallback var found = images.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; - dialog.Files = found.Select(x => new BotSharpFile + var targets = found; + if (dialog.Role == AgentRole.User) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.User)).ToList(); + } + else if (dialog.Role == AgentRole.Assistant || dialog.Role == AgentRole.Model) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)).ToList(); + } + + dialog.Files = targets.Select(x => new BotSharpFile { ContentType = x.ContentType, FileUrl = x.FileUrl, @@ -121,7 +135,6 @@ public class ReadImageFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_read_llm_provider"); var model = state.GetState("image_read_llm_model"); @@ -131,8 +144,8 @@ public class ReadImageFn : IFunctionCallback return (provider, model); } - provider = fileSettings?.Image?.Reading?.LlmProvider; - model = fileSettings?.Image?.Reading?.LlmModel; + provider = _settings?.Image?.Reading?.LlmProvider; + model = _settings?.Image?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index d6f8d74b..a2c2f4e5 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -9,6 +9,7 @@ public class ReadPdfFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; private readonly IEnumerable _pdfContentTypes = new List { @@ -17,10 +18,12 @@ public class ReadPdfFn : IFunctionCallback public ReadPdfFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -52,6 +55,7 @@ public class ReadPdfFn : IFunctionCallback var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); var response = await GetChatCompletion(agent, dialogs); + dialogs.ForEach(x => x.Files = null); message.Content = response; return true; } @@ -65,16 +69,47 @@ public class ReadPdfFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); - if (screenshots.IsNullOrEmpty()) return dialogs; + IEnumerable files; + if (_settings.Pdf?.Reading?.ConvertToImage == true) + { + files = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds, options: new() + { + Sources = [FileSource.User], + ImageConvertProvider = _settings.Pdf?.Reading?.ImageConverter?.Provider + }); + } + else + { + files = fileStorage.GetMessageFiles(conversationId, messageIds, options: new() + { + Sources = [FileSource.User], + ContentTypes = _pdfContentTypes + }); + } + + + if (files.IsNullOrEmpty()) + { + return dialogs; + } foreach (var dialog in dialogs) { - var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; - dialog.Files = found.Select(x => new BotSharpFile + var targets = found; + if (dialog.IsFromUser) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.User)).ToList(); + } + else if (dialog.IsFromAssistant) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)).ToList(); + } + + dialog.Files = targets.Select(x => new BotSharpFile { ContentType = x.ContentType, FileUrl = x.FileUrl, @@ -107,7 +142,6 @@ public class ReadPdfFn : IFunctionCallback { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("pdf_read_llm_provider"); var model = state.GetState("pdf_read_llm_model"); @@ -117,8 +151,8 @@ public class ReadPdfFn : IFunctionCallback return (provider, model); } - provider = fileSettings?.Image?.Reading?.LlmProvider; - model = fileSettings?.Image?.Reading?.LlmModel; + provider = _settings?.Pdf?.Reading?.LlmProvider; + model = _settings?.Pdf?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -133,6 +167,11 @@ public class ReadPdfFn : IFunctionCallback private void SetImageDetailLevel() { + if (_settings.Pdf?.Reading?.ConvertToImage != true) + { + return; + } + var state = _services.GetRequiredService(); var fileSettings = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index cf1b8590..9eb4c9ac 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -1,10 +1,11 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } - public SettingBase? ImageConverter { get; set; } } #region Image @@ -16,22 +17,22 @@ public class ImageSettings public ImageVariationSettings? Variation { get; set; } } -public class ImageReadSettings : FileLlmSettingBase +public class ImageReadSettings : LlmBase { public string? ImageDetailLevel { get; set; } } -public class ImageGenerationSettings : FileLlmSettingBase +public class ImageGenerationSettings : LlmBase { } -public class ImageEditSettings : FileLlmSettingBase +public class ImageEditSettings : LlmBase { - + public SettingBase? ImageConverter { get; set; } } -public class ImageVariationSettings : FileLlmSettingBase +public class ImageVariationSettings : LlmBase { } @@ -43,15 +44,10 @@ public class PdfSettings public PdfReadSettings? Reading { get; set; } } -public class PdfReadSettings : FileLlmSettingBase +public class PdfReadSettings : LlmBase { public bool ConvertToImage { get; set; } public string? ImageDetailLevel { get; set; } + public SettingBase? ImageConverter { get; set; } } -#endregion - -public class FileLlmSettingBase -{ - public string? LlmProvider { get; set; } - public string? LlmModel { get; set; } -} \ No newline at end of file +#endregion \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index acaea6d6..722f50d9 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -8,18 +8,10 @@ public partial class ImageCompletionProvider { public async Task GetImageGeneration(Agent agent, RoleDialogModel message) { - var hooks = _services.GetHooks(agent.Id); - var client = ProviderHelper.GetClient(Provider, _model, _services); var (prompt, imageCount, options) = PrepareGenerationOptions(message); var imageClient = client.GetImageClient(_model); - // Before generation - foreach (var hook in hooks) - { - await hook.BeforeGenerating(agent, [new RoleDialogModel(AgentRole.User, prompt)]); - } - var response = imageClient.GenerateImages(prompt, imageCount, options); var images = response.Value; @@ -32,16 +24,6 @@ public partial class ImageCompletionProvider GeneratedImages = generatedImages }; - // After generation - var usage = response.Value.Usage; - foreach (var hook in hooks) - { - await hook.AfterGenerated(responseMessage, new TokenStatsModel - { - - }); - } - return await Task.FromResult(responseMessage); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index db88dc52..0c774dd8 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; -using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; using System.Net.Mime; @@ -8,28 +6,37 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) + || messageIds.IsNullOrEmpty() + || options.Sources.IsNullOrEmpty()) { return files; } - var source = FileSource.User; - var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; + var baseDir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; foreach (var messageId in messageIds) { - var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + if (string.IsNullOrWhiteSpace(messageId)) { - var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); - if (file == null) continue; + continue; + } - var screenshots = await GetScreenshots(file, subDir, messageId, source); - if (screenshots.IsNullOrEmpty()) continue; + foreach (var source in options.Sources) + { + var dir = $"{baseDir}/{messageId}/{source}"; + foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + { + var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); + if (file == null) continue; - files.AddRange(screenshots); + var screenshots = await GetScreenshotsAsync(file, subDir, messageId, source, options); + if (screenshots.IsNullOrEmpty()) continue; + + files.AddRange(screenshots); + } } } @@ -214,35 +221,9 @@ public partial class TencentCosService return dir; } - private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) + private async Task> ConvertPdfToImagesAsync(string pdfLoc, string imageLoc, MessageFileScreenshotOptions options) { - if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - - if (offset.HasValue && offset < 1) - { - offset = 1; - } - - var messageIds = new List(); - if (offset.HasValue) - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - } - - return messageIds; - } - - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) - { - var converters = _services.GetServices(); - if (converters.IsNullOrEmpty()) return Enumerable.Empty(); - - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options.ImageConvertProvider); if (converter == null) { return Enumerable.Empty(); @@ -250,19 +231,12 @@ public partial class TencentCosService return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IImageConverter? GetPdf2ImageConverter() - { - var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); - return converter; - } - private string BuilFileUrl(string file) { return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; } - private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + private async Task> GetScreenshotsAsync(string file, string parentDir, string messageId, string source, MessageFileScreenshotOptions options) { var files = new List(); @@ -293,7 +267,7 @@ public partial class TencentCosService } else if (contentType == MediaTypeNames.Application.Pdf) { - var images = await ConvertPdfToImages(file, screenshotDir); + var images = await ConvertPdfToImagesAsync(file, screenshotDir, options); foreach (var image in images) { var fileName = Path.GetFileNameWithoutExtension(image); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index e43f6eb0..0ce8ce31 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -439,12 +439,6 @@ "FileCore": { "Storage": "LocalFileStorage", - "Pdf2TextConverter": { - "Provider": "" - }, - "Pdf2ImageConverter": { - "Provider": "" - }, "ImageConverter": { "Provider": "" } @@ -463,7 +457,10 @@ }, "Edit": { "LlmProvider": "openai", - "LlmModel": "gpt-image-1" + "LlmModel": "gpt-image-1", + "ImageConverter": { + "Provider": "file-handler" + } }, "Variation": { "LlmProvider": "", @@ -472,14 +469,14 @@ }, "Pdf": { "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", - "ConvertToImage": true, - "ImageDetailLevel": "auto" + "LlmProvider": "google-ai", + "LlmModel": "gemini-2.0-flash", + "ConvertToImage": false, + "ImageDetailLevel": "auto", + "ImageConverter": { + "Provider": null + } } - }, - "ImageConverter": { - "Provider": "file-handler" } }, diff --git a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs index fcc3aaf7..0e1be86c 100644 --- a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs +++ b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs @@ -54,7 +54,7 @@ namespace BotSharp.Plugin.Google.Core return string.Join("/", segments); } - public Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) + public Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options) { return Task.FromResult>(new List {