SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1164 from iceljc/features/refine-model-settings

Browse source 
Features/refine model settings
This commit is contained in:
iceljc 2025-09-16 19:29:49 -05:00 committed by GitHub
parent 611d5caf96 57bd04bf7b
commit a39beec920
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 367 additions and 362 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -110,6 +110,7 @@ public class RoleDialogModel : ITrackableMessage
/// <summary>
/// Files to be used in conversation
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<BotSharpFile>? Files { get; set; }
/// <summary>
@ -133,6 +134,12 @@ public class RoleDialogModel : ITrackableMessage
public bool IsStreaming { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public bool IsFromUser => Role == AgentRole.User;
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model;
public RoleDialogModel()
{
}

8
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs
View file

@ -5,13 +5,19 @@ public class TokenStatsModel
public string Provider { get; set; }
public string Model { get; set; }
public string Prompt { get; set; }
#region Input
public int TextInputTokens { get; set; }
public int CachedTextInputTokens { get; set; }
public int AudioInputTokens { get; set; }
public int CachedAudioInputTokens { get; set; }
#endregion
#region Output
public int TextOutputTokens { get; set; }
public int AudioOutputTokens { get; set; }
public AgentLlmConfig LlmConfig { get; set; }
#endregion
public int TotalInputTokens => TextInputTokens + CachedTextInputTokens + AudioInputTokens + CachedAudioInputTokens;
public int TotalOutputTokens => TextOutputTokens + AudioOutputTokens;

2
src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSourceType.cs → src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSource.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Files.Enums;
public static class FileSourceType
public static class FileSource
{
public const string User = "user";
public const string Bot = "bot";

7
src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs
View file

@ -5,12 +5,5 @@ namespace BotSharp.Abstraction.Files;
public class FileCoreSettings
{
public string Storage { get; set; } = FileStorageEnum.LocalFileStorage;
public SettingBase? Pdf2TextConverter { get; set; }
public SettingBase? Pdf2ImageConverter { get; set; }
public SettingBase? ImageConverter { get; set; }
}
public class SettingBase
{
public string Provider { get; set; }
}

10
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -19,24 +19,22 @@ public interface IFileStorageService
#region Conversation
/// <summary>
/// Get the message file screenshots for specific content types, e.g., pdf
/// Get the message file screenshots for pdf
/// </summary>
/// <param name="conversationId"></param>
/// <param name="messageIds"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds);
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds, MessageFileScreenshotOptions options);
/// <summary>
/// Get the files that have been uploaded in the chat. No screenshot images are included.
/// </summary>
/// <param name="conversationId"></param>
/// <param name="messageIds"></param>
/// <param name="source"></param>
/// <param name="imageOnly"></param>
/// <param name="options"></param>
/// <returns></returns>
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, MessageFileOptions? options = null);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files);
/// <summary>

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -6,7 +6,7 @@ public class MessageFileModel : FileInformation
public string MessageId { get; set; }
[JsonPropertyName("file_source")]
public string FileSource { get; set; } = FileSourceType.User;
public string FileSource { get; set; } = Enums.FileSource.User;
[JsonPropertyName("file_index")]
public string FileIndex { get; set; } = string.Empty;

20
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs Normal file
View file

@ -0,0 +1,20 @@
namespace BotSharp.Abstraction.Files.Models;
public class MessageFileOptions
{
/// <summary>
/// File sources: user, bot
/// </summary>
public IEnumerable<string>? Sources { get; set; }
/// <summary>
/// File content types
/// </summary>
public IEnumerable<string>? ContentTypes { get; set; }
}
public class MessageFileScreenshotOptions : MessageFileOptions
{
public string ImageConvertProvider { get; set; }
}

5
src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs
View file

@ -21,8 +21,3 @@ public class KnowledgeTextEmbeddingSetting : SettingBase
public string Model { get; set; }
public int Dimension { get; set; }
}
public class SettingBase
{
public string Provider { get; set; }
}

25
src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs
View file

@ -1,17 +1,7 @@
namespace BotSharp.Abstraction.Models;
public class LlmConfigBase
public class LlmConfigBase : LlmBase
{
/// <summary>
/// Llm provider
/// </summary>
public string? LlmProvider { get; set; }
/// <summary>
/// Llm model
/// </summary>
public string? LlmModel { get; set; }
/// <summary>
/// Llm maximum output tokens
/// </summary>
@ -22,3 +12,16 @@ public class LlmConfigBase
/// </summary>
public string? ReasoningEffortLevel { get; set; }
}
public class LlmBase
{
/// <summary>
/// Llm provider
/// </summary>
public string? LlmProvider { get; set; }
/// <summary>
/// Llm model
/// </summary>
public string? LlmModel { get; set; }
}

6
src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs Normal file
View file

@ -0,0 +1,6 @@
namespace BotSharp.Abstraction.Settings;
public class SettingBase
{
public string Provider { get; set; }
}

1
src/Infrastructure/BotSharp.Abstraction/Using.cs
View file

@ -21,3 +21,4 @@ global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.Abstraction.Crontab.Models;
global using BotSharp.Abstraction.MCP.Models;
global using BotSharp.Abstraction.Settings;

51
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs
View file

@ -28,12 +28,12 @@ public partial class FileInstructService
}
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes);
if (options.IsIncludeBotFiles)
var files = _fileStorage.GetMessageFiles(conversationId, messageIds, options: new()
{
var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes);
files = MergeMessageFiles(messageIds, files, botFiles);
}
Sources = options.IsIncludeBotFiles ?[FileSource.User, FileSource.Bot] : [FileSource.User],
ContentTypes = options.ContentTypes
});
files = MergeMessageFiles(messageIds, files);
if (files.IsNullOrEmpty())
{
@ -43,19 +43,31 @@ public partial class FileInstructService
return await SelectFiles(files, dialogs, options);
}
private IEnumerable<MessageFileModel> MergeMessageFiles(IEnumerable<string> messageIds, IEnumerable<MessageFileModel> userFiles, IEnumerable<MessageFileModel> botFiles)
private IEnumerable<MessageFileModel> MergeMessageFiles(IEnumerable<string> messageIds, IEnumerable<MessageFileModel> files)
{
var files = new List<MessageFileModel>();
var mergedFiles = new List<MessageFileModel>();
if (messageIds.IsNullOrEmpty()) return files;
if (messageIds.IsNullOrEmpty())
{
return mergedFiles;
}
var userFiles = files.Where(x => x.FileSource.IsEqualTo(FileSource.User));
var botFiles = files.Where(x => x.FileSource.IsEqualTo(FileSource.Bot));
foreach (var messageId in messageIds)
{
var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList();
if (!users.IsNullOrEmpty()) files.AddRange(users);
if (!bots.IsNullOrEmpty()) files.AddRange(bots);
if (!users.IsNullOrEmpty())
{
mergedFiles.AddRange(users);
}
if (!bots.IsNullOrEmpty())
{
mergedFiles.AddRange(bots);
}
}
return files;
@ -74,8 +86,6 @@ public partial class FileInstructService
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
// Handle dialogs and files
var innerDialogs = (dialogs ?? []).ToList();
var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s).";
@ -86,13 +96,14 @@ public partial class FileInstructService
AssembleMessageFiles(innerDialogs, files, options);
}
try
{
// Handle instruction
var promptMessages = innerDialogs.Select(x =>
{
var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}";
var fileDescs = x.Files?.Select((f, fidx) => $"- message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " +
$"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'");
$"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSource.User : FileSource.Bot)}'");
var desc = string.Empty;
if (!fileDescs.IsNullOrEmpty())
@ -164,6 +175,10 @@ public partial class FileInstructService
_logger.LogWarning(ex, $"Error when selecting files.");
return [];
}
finally
{
innerDialogs.ForEach(x => x.Files = null);
}
}
private void AssembleMessageFiles(IEnumerable<RoleDialogModel> dialogs, IEnumerable<MessageFileModel> files, SelectFileOptions options)
@ -184,10 +199,10 @@ public partial class FileInstructService
continue;
}
var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User);
var userMsg = group.FirstOrDefault(x => x.IsFromUser);
if (userMsg != null)
{
var userFiles = found.Where(x => x.FileSource == FileSourceType.User);
var userFiles = found.Where(x => x.FileSource == FileSource.User);
userMsg.Files = userFiles.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
@ -199,10 +214,10 @@ public partial class FileInstructService
}).ToList();
}
var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant);
var botMsg = group.LastOrDefault(x => x.IsFromAssistant);
if (botMsg != null)
{
var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot);
var botFiles = found.Where(x => x.FileSource == FileSource.Bot);
botMsg.Files = botFiles.Select(x => new BotSharpFile
{
ContentType = x.ContentType,

128
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -6,16 +6,17 @@ namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds, MessageFileScreenshotOptions options)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId)
|| messageIds.IsNullOrEmpty()
|| options.Sources.IsNullOrEmpty())
{
return files;
}
var source = FileSourceType.User;
var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
var baseUrl = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
foreach (var messageId in messageIds)
{
@ -24,7 +25,9 @@ public partial class LocalFileStorageService
continue;
}
var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User);
foreach (var source in options.Sources)
{
var dir = Path.Combine(baseUrl, messageId, source);
if (!ExistDirectory(dir))
{
continue;
@ -38,7 +41,7 @@ public partial class LocalFileStorageService
continue;
}
var screenshots = await GetScreenshots(file, subDir, messageId, source);
var screenshots = await GetScreenshotsAsync(file, subDir, messageId, source, options);
if (screenshots.IsNullOrEmpty())
{
continue;
@ -47,12 +50,12 @@ public partial class LocalFileStorageService
files.AddRange(screenshots);
}
}
}
return files;
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds,
string source, IEnumerable<string>? contentTypes = null)
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, MessageFileOptions? options = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty())
@ -67,7 +70,23 @@ public partial class LocalFileStorageService
continue;
}
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId, source);
var baseDir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId);
if (!ExistDirectory(baseDir))
{
continue;
}
var sources = options?.Sources != null
? options.Sources
: Directory.GetDirectories(baseDir).Select(x => x.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries).Last());
if (sources.IsNullOrEmpty())
{
continue;
}
foreach (var source in sources)
{
var dir = Path.Combine(baseDir, source);
if (!ExistDirectory(dir))
{
continue;
@ -75,34 +94,35 @@ public partial class LocalFileStorageService
foreach (var subDir in Directory.GetDirectories(dir))
{
var index = subDir.Split(Path.DirectorySeparatorChar).Last();
var fileIndex = subDir.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries).Last();
foreach (var file in Directory.GetFiles(subDir))
{
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
if (options?.ContentTypes != null && !options.ContentTypes.Contains(contentType))
{
continue;
}
var fileName = Path.GetFileNameWithoutExtension(file);
var fileExtension = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
var model = new MessageFileModel
{
MessageId = messageId,
FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}",
FileDownloadUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}/download",
FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{fileIndex}/{fileName}",
FileDownloadUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{fileIndex}/{fileName}/download",
FileStorageUrl = file,
FileName = fileName,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source,
FileIndex = index
FileIndex = fileIndex
};
files.Add(model);
}
}
}
}
return files;
}
@ -126,38 +146,6 @@ public partial class LocalFileStorageService
return found;
}
public IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds)
{
var foundMsgs = new List<MessageFileModel>();
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty())
{
return foundMsgs;
}
foreach (var messageId in messageIds)
{
if (string.IsNullOrWhiteSpace(messageId))
{
continue;
}
var prefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId);
var userDir = Path.Combine(prefix, FileSourceType.User);
if (ExistDirectory(userDir))
{
foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User });
}
var botDir = Path.Combine(prefix, FileSourceType.Bot);
if (ExistDirectory(botDir))
{
foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot });
}
}
return foundMsgs;
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files)
{
if (string.IsNullOrWhiteSpace(conversationId)
@ -299,40 +287,9 @@ public partial class LocalFileStorageService
return dir;
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> dialogs, int? offset = null)
private async Task<IEnumerable<string>> ConvertPdfToImagesAsync(string pdfLoc, string imageLoc, MessageFileScreenshotOptions options)
{
if (dialogs.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
}
if (offset.HasValue && offset < 1)
{
offset = 1;
}
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
}
return messageIds;
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IImageConverter>();
if (converters.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
}
var converter = GetPdf2ImageConverter();
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == options.ImageConvertProvider);
if (converter == null)
{
return Enumerable.Empty<string>();
@ -341,14 +298,7 @@ public partial class LocalFileStorageService
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}
private async Task<IEnumerable<MessageFileModel>> GetScreenshots(string file, string parentDir, string messageId, string source)
private async Task<IEnumerable<MessageFileModel>> GetScreenshotsAsync(string file, string parentDir, string messageId, string source, MessageFileScreenshotOptions options)
{
var files = new List<MessageFileModel>();
@ -378,7 +328,7 @@ public partial class LocalFileStorageService
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenshotDir);
var images = await ConvertPdfToImagesAsync(file, screenshotDir, options);
foreach (var image in images)
{
var fileName = Path.GetFileNameWithoutExtension(image);

12
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -96,7 +96,7 @@ public class ConversationController : ControllerBase
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = history.Select(x => x.MessageId).Distinct().ToList();
var fileMessages = fileStorage.GetMessagesWithFile(conversationId, messageIds);
var files = fileStorage.GetMessageFiles(conversationId, messageIds, options: new() { Sources = [FileSource.User, FileSource.Bot] });
var dialogs = new List<ChatResponseModel>();
foreach (var message in history)
@ -115,7 +115,7 @@ public class ConversationController : ControllerBase
Data = message.Data,
Sender = UserDto.FromUser(user),
Payload = message.Payload,
HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.User)
HasMessageFiles = files.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSource.User)
});
}
else if (message.Role == AgentRole.Assistant)
@ -136,7 +136,7 @@ public class ConversationController : ControllerBase
Role = message.Role,
},
RichContent = message.SecondaryRichContent ?? message.RichContent,
HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.Bot)
HasMessageFiles = files.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSource.Bot)
});
}
}
@ -490,7 +490,7 @@ public class ConversationController : ControllerBase
var conv = await convService.GetConversationRecordOrCreateNew(agentId);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageId = Guid.NewGuid().ToString();
var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files);
var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSource.User, input.Files);
return isSaved ? messageId : string.Empty;
}
@ -498,8 +498,8 @@ public class ConversationController : ControllerBase
public IEnumerable<MessageFileViewModel> GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var files = fileStorage.GetMessageFiles(conversationId, new List<string> { messageId }, source);
return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List<MessageFileViewModel>();
var files = fileStorage.GetMessageFiles(conversationId, [messageId], options: new() { Sources = [source] });
return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? [];
}
[HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")]

8
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -13,7 +13,7 @@ public class HandleAudioRequestFn : IFunctionCallback
private readonly ILogger<HandleAudioRequestFn> _logger;
private readonly BotSharpOptions _options;
private readonly IEnumerable<string> _audioContentType = new List<string>
private readonly IEnumerable<string> _audioContentTypes = new List<string>
{
AudioType.mp3.ToFileType(),
AudioType.wav.ToFileType(),
@ -52,7 +52,11 @@ public class HandleAudioRequestFn : IFunctionCallback
}
var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList();
var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType);
var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new()
{
Sources = [FileSource.User],
ContentTypes = _audioContentTypes
});
audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList();

9
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -150,7 +150,6 @@ public class EditImageFn : IFunctionCallback
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("image_edit_llm_provider");
var model = state.GetState("image_edit_llm_provider");
@ -160,8 +159,8 @@ public class EditImageFn : IFunctionCallback
return (provider, model);
}
provider = fileSettings?.Image?.Edit?.LlmProvider;
model = fileSettings?.Image?.Edit?.LlmModel;
provider = _settings?.Image?.Edit?.LlmProvider;
model = _settings?.Image?.Edit?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
@ -191,13 +190,13 @@ public class EditImageFn : IFunctionCallback
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files);
return files.Select(x => x.FileName);
}
private async Task<BinaryData> ConvertImageToPngWithRgba(BinaryData binaryFile)
{
var provider = _settings?.ImageConverter?.Provider;
var provider = _settings?.Image?.Edit?.ImageConverter?.Provider;
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
if (converter == null)
{

14
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Agents.Models;
namespace BotSharp.Plugin.FileHandler.Functions;
public class GenerateImageFn : IFunctionCallback
@ -9,6 +7,7 @@ public class GenerateImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<GenerateImageFn> _logger;
private readonly FileHandlerSettings _settings;
private Agent _agent;
private string _conversationId;
@ -16,10 +15,12 @@ public class GenerateImageFn : IFunctionCallback
public GenerateImageFn(
IServiceProvider services,
ILogger<GenerateImageFn> logger)
ILogger<GenerateImageFn> logger,
FileHandlerSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -113,7 +114,6 @@ public class GenerateImageFn : IFunctionCallback
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("image_generate_llm_provider");
var model = state.GetState("image_generate_llm_model");
@ -123,8 +123,8 @@ public class GenerateImageFn : IFunctionCallback
return (provider, model);
}
provider = fileSettings?.Image?.Generation?.LlmProvider;
model = fileSettings?.Image?.Generation?.LlmModel;
provider = _settings?.Image?.Generation?.LlmProvider;
model = _settings?.Image?.Generation?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
@ -151,7 +151,7 @@ public class GenerateImageFn : IFunctionCallback
}).ToList();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files);
return files.Select(x => x.FileName);
}
}

43
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -9,13 +9,22 @@ public class ReadImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<ReadImageFn> _logger;
private readonly FileHandlerSettings _settings;
private readonly IEnumerable<string> _imageContentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
};
public ReadImageFn(
IServiceProvider services,
ILogger<ReadImageFn> logger)
ILogger<ReadImageFn> logger,
FileHandlerSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -47,6 +56,7 @@ public class ReadImageFn : IFunctionCallback
var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs);
var response = await GetChatCompletion(agent, dialogs);
dialogs.ForEach(x => x.Files = null);
message.Content = response;
return true;
}
@ -58,24 +68,30 @@ public class ReadImageFn : IFunctionCallback
return new List<RoleDialogModel>();
}
var contentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes);
var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes);
var images = userImages.Concat(botImages);
var images = fileStorage.GetMessageFiles(conversationId, messageIds, options: new()
{
Sources = [FileSource.User, FileSource.Bot],
ContentTypes = _imageContentTypes
});
foreach (var dialog in dialogs)
{
var found = images.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
var targets = found;
if (dialog.Role == AgentRole.User)
{
targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.User)).ToList();
}
else if (dialog.Role == AgentRole.Assistant || dialog.Role == AgentRole.Model)
{
targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)).ToList();
}
dialog.Files = targets.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
@ -119,7 +135,6 @@ public class ReadImageFn : IFunctionCallback
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("image_read_llm_provider");
var model = state.GetState("image_read_llm_model");
@ -129,8 +144,8 @@ public class ReadImageFn : IFunctionCallback
return (provider, model);
}
provider = fileSettings?.Image?.Reading?.LlmProvider;
model = fileSettings?.Image?.Reading?.LlmModel;
provider = _settings?.Image?.Reading?.LlmProvider;
model = _settings?.Image?.Reading?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{

55
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -9,6 +9,7 @@ public class ReadPdfFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<ReadPdfFn> _logger;
private readonly FileHandlerSettings _settings;
private readonly IEnumerable<string> _pdfContentTypes = new List<string>
{
@ -17,10 +18,12 @@ public class ReadPdfFn : IFunctionCallback
public ReadPdfFn(
IServiceProvider services,
ILogger<ReadPdfFn> logger)
ILogger<ReadPdfFn> logger,
FileHandlerSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
@ -52,6 +55,7 @@ public class ReadPdfFn : IFunctionCallback
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetChatCompletion(agent, dialogs);
dialogs.ForEach(x => x.Files = null);
message.Content = response;
return true;
}
@ -65,16 +69,47 @@ public class ReadPdfFn : IFunctionCallback
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds);
if (screenshots.IsNullOrEmpty()) return dialogs;
IEnumerable<MessageFileModel> files;
if (_settings.Pdf?.Reading?.ConvertToImage == true)
{
files = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds, options: new()
{
Sources = [FileSource.User],
ImageConvertProvider = _settings.Pdf?.Reading?.ImageConverter?.Provider
});
}
else
{
files = fileStorage.GetMessageFiles(conversationId, messageIds, options: new()
{
Sources = [FileSource.User],
ContentTypes = _pdfContentTypes
});
}
if (files.IsNullOrEmpty())
{
return dialogs;
}
foreach (var dialog in dialogs)
{
var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList();
var found = files.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
var targets = found;
if (dialog.IsFromUser)
{
targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.User)).ToList();
}
else if (dialog.IsFromAssistant)
{
targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)).ToList();
}
dialog.Files = targets.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
@ -107,7 +142,6 @@ public class ReadPdfFn : IFunctionCallback
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();
var provider = state.GetState("pdf_read_llm_provider");
var model = state.GetState("pdf_read_llm_model");
@ -117,8 +151,8 @@ public class ReadPdfFn : IFunctionCallback
return (provider, model);
}
provider = fileSettings?.Image?.Reading?.LlmProvider;
model = fileSettings?.Image?.Reading?.LlmModel;
provider = _settings?.Pdf?.Reading?.LlmProvider;
model = _settings?.Pdf?.Reading?.LlmModel;
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
@ -133,6 +167,11 @@ public class ReadPdfFn : IFunctionCallback
private void SetImageDetailLevel()
{
if (_settings.Pdf?.Reading?.ConvertToImage != true)
{
return;
}
var state = _services.GetRequiredService<IConversationStateService>();
var fileSettings = _services.GetRequiredService<FileHandlerSettings>();

22
src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs
View file

@ -1,10 +1,11 @@
using BotSharp.Abstraction.Models;
namespace BotSharp.Plugin.FileHandler.Settings;
public class FileHandlerSettings
{
public ImageSettings? Image { get; set; }
public PdfSettings? Pdf { get; set; }
public SettingBase? ImageConverter { get; set; }
}
#region Image
@ -16,22 +17,22 @@ public class ImageSettings
public ImageVariationSettings? Variation { get; set; }
}
public class ImageReadSettings : FileLlmSettingBase
public class ImageReadSettings : LlmBase
{
public string? ImageDetailLevel { get; set; }
}
public class ImageGenerationSettings : FileLlmSettingBase
public class ImageGenerationSettings : LlmBase
{
}
public class ImageEditSettings : FileLlmSettingBase
public class ImageEditSettings : LlmBase
{
public SettingBase? ImageConverter { get; set; }
}
public class ImageVariationSettings : FileLlmSettingBase
public class ImageVariationSettings : LlmBase
{
}
@ -43,15 +44,10 @@ public class PdfSettings
public PdfReadSettings? Reading { get; set; }
}
public class PdfReadSettings : FileLlmSettingBase
public class PdfReadSettings : LlmBase
{
public bool ConvertToImage { get; set; }
public string? ImageDetailLevel { get; set; }
public SettingBase? ImageConverter { get; set; }
}
#endregion
public class FileLlmSettingBase
{
public string? LlmProvider { get; set; }
public string? LlmModel { get; set; }
}

1
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs
View file

@ -1,4 +1,5 @@
#pragma warning disable OPENAI001
using BotSharp.Abstraction.Hooks;
using OpenAI.Images;
namespace BotSharp.Plugin.OpenAI.Providers.Image;

122
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -1,6 +1,4 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Converters;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Utilities;
using System.Net.Mime;
@ -8,43 +6,70 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds, MessageFileScreenshotOptions options)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId)
|| messageIds.IsNullOrEmpty()
|| options.Sources.IsNullOrEmpty())
{
return files;
}
var source = FileSourceType.User;
var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}";
var baseDir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}";
foreach (var messageId in messageIds)
{
var dir = $"{pathPrefix}/{messageId}/{source}";
if (string.IsNullOrWhiteSpace(messageId))
{
continue;
}
foreach (var source in options.Sources)
{
var dir = $"{baseDir}/{messageId}/{source}";
foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir))
{
var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault();
if (file == null) continue;
var screenshots = await GetScreenshots(file, subDir, messageId, source);
var screenshots = await GetScreenshotsAsync(file, subDir, messageId, source, options);
if (screenshots.IsNullOrEmpty()) continue;
files.AddRange(screenshots);
}
}
}
return files;
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds,
string source, IEnumerable<string>? contentTypes = null)
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, MessageFileOptions? options = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files;
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty())
{
return files;
}
foreach (var messageId in messageIds)
{
var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}/{source}";
var baseDir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}";
if (!ExistDirectory(baseDir))
{
continue;
}
var sources = options?.Sources != null
? options.Sources
: _cosClient.BucketClient.GetDirectories(baseDir).Select(x => x.Split("/", StringSplitOptions.RemoveEmptyEntries).Last());
if (sources.IsNullOrEmpty())
{
continue;
}
foreach (var source in sources)
{
var dir = Path.Combine(baseDir, source);
if (!ExistDirectory(dir))
{
continue;
@ -52,17 +77,18 @@ public partial class TencentCosService
foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir))
{
var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty;
foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir))
{
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
if (options?.ContentTypes != null && !options.ContentTypes.Contains(contentType))
{
continue;
}
var fileName = Path.GetFileNameWithoutExtension(file);
var fileExtension = Path.GetExtension(file).Substring(1);
var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty;
var model = new MessageFileModel()
{
MessageId = messageId,
@ -79,6 +105,7 @@ public partial class TencentCosService
}
}
}
}
return files;
}
@ -87,37 +114,13 @@ public partial class TencentCosService
public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName)
{
var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/";
var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}";
var fileList = _cosClient.BucketClient.GetDirFiles(dir);
var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName));
return found;
}
public IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds)
{
var foundMsgs = new List<MessageFileModel>();
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return foundMsgs;
foreach (var messageId in messageIds)
{
var prefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}";
var userDir = $"{prefix}/{FileSourceType.User}/";
if (ExistDirectory(userDir))
{
foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User });
}
var botDir = $"{prefix}/{FileSourceType.Bot}";
if (ExistDirectory(botDir))
{
foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot });
}
}
return foundMsgs;
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files)
{
if (files.IsNullOrEmpty()) return false;
@ -218,35 +221,9 @@ public partial class TencentCosService
return dir;
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> dialogs, int? offset = null)
private async Task<IEnumerable<string>> ConvertPdfToImagesAsync(string pdfLoc, string imageLoc, MessageFileScreenshotOptions options)
{
if (dialogs.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset.HasValue && offset < 1)
{
offset = 1;
}
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
}
return messageIds;
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IImageConverter>();
if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
var converter = GetPdf2ImageConverter();
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == options.ImageConvertProvider);
if (converter == null)
{
return Enumerable.Empty<string>();
@ -254,19 +231,12 @@ public partial class TencentCosService
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
}
private IImageConverter? GetPdf2ImageConverter()
{
var settings = _services.GetRequiredService<FileCoreSettings>();
var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider);
return converter;
}
private string BuilFileUrl(string file)
{
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}";
}
private async Task<IEnumerable<MessageFileModel>> GetScreenshots(string file, string parentDir, string messageId, string source)
private async Task<IEnumerable<MessageFileModel>> GetScreenshotsAsync(string file, string parentDir, string messageId, string source, MessageFileScreenshotOptions options)
{
var files = new List<MessageFileModel>();
@ -297,7 +267,7 @@ public partial class TencentCosService
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenshotDir);
var images = await ConvertPdfToImagesAsync(file, screenshotDir, options);
foreach (var image in images)
{
var fileName = Path.GetFileNameWithoutExtension(image);

25
src/WebStarter/appsettings.json
View file

@ -439,12 +439,6 @@
"FileCore": {
"Storage": "LocalFileStorage",
"Pdf2TextConverter": {
"Provider": ""
},
"Pdf2ImageConverter": {
"Provider": ""
},
"ImageConverter": {
"Provider": ""
}
@ -463,7 +457,10 @@
},
"Edit": {
"LlmProvider": "openai",
"LlmModel": "gpt-image-1"
"LlmModel": "gpt-image-1",
"ImageConverter": {
"Provider": "file-handler"
}
},
"Variation": {
"LlmProvider": "",
@ -472,14 +469,14 @@
},
"Pdf": {
"Reading": {
"LlmProvider": "openai",
"LlmModel": "gpt-5-mini",
"ConvertToImage": true,
"ImageDetailLevel": "auto"
}
},
"LlmProvider": "google-ai",
"LlmModel": "gemini-2.0-flash",
"ConvertToImage": false,
"ImageDetailLevel": "auto",
"ImageConverter": {
"Provider": "file-handler"
"Provider": null
}
}
}
},

14
tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs
View file

@ -54,7 +54,7 @@ namespace BotSharp.Plugin.Google.Core
return string.Join("/", segments);
}
public Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds)
public Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds, MessageFileScreenshotOptions options)
{
return Task.FromResult<IEnumerable<MessageFileModel>>(new List<MessageFileModel>
{
@ -63,8 +63,7 @@ namespace BotSharp.Plugin.Google.Core
});
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source,
IEnumerable<string>? contentTypes = null)
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, MessageFileOptions? options = null)
{
return new List<MessageFileModel>
{
@ -78,15 +77,6 @@ namespace BotSharp.Plugin.Google.Core
return $"FakePath/{fileName}";
}
public IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds)
{
return new List<MessageFileModel>
{
new MessageFileModel { FileName = "MessageFile1.jpg" },
new MessageFileModel { FileName = "MessageFile2.png" }
};
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<FileDataModel> files)
{
return true;