diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index 1da42cc4..76570c0e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Files; public interface IBotSharpFileService { string GetDirectory(string conversationId); - IEnumerable GetChatImages(string conversationId, string source, List conversations, int? offset = null); + Task> GetChatImages(string conversationId, string source, IEnumerable fileTypes, List conversations, int? offset = null); IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); bool HasConversationUserFiles(string conversationId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs new file mode 100644 index 00000000..538f45e8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class LlmFileContext +{ + [JsonPropertyName("user_request")] + public string UserRequest { get; set; } + + [JsonPropertyName("file_types")] + public string FileTypes { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index 212dba8b..7c626ddf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -8,12 +8,13 @@ namespace BotSharp.Core.Files; public partial class BotSharpFileService { - public IEnumerable GetChatImages(string conversationId, string source, List conversations, int? offset = null) + public async Task> GetChatImages(string conversationId, string source, IEnumerable fileTypes, + List conversations, int? offset = null) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) { - return files; + return new List(); } if (offset <= 0) @@ -35,7 +36,98 @@ public partial class BotSharpFileService messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); } - files = GetMessageFiles(conversationId, messageIds, source, imageOnly: true).ToList(); + files = await GetMessageFiles(conversationId, messageIds, source, fileTypes); + return files; + } + + private async Task> GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable fileTypes) + { + var files = new List(); + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty() || fileTypes.IsNullOrEmpty()) return files; + + var isNeedScreenShot = fileTypes.Any(x => _allowScreenShotTypes.Contains(x)); + var onlyScreenShot = fileTypes.All(x => _allowScreenShotTypes.Contains(x)); + + try + { + var contextId = string.Empty; + var web = _services.GetRequiredService(); + var preFixPath = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); + + if (isNeedScreenShot) + { + contextId = Guid.NewGuid().ToString(); + await web.LaunchBrowser(contextId, string.Empty); + } + + foreach (var messageId in messageIds) + { + var dir = Path.Combine(preFixPath, messageId, source); + if (!ExistDirectory(dir)) continue; + + foreach (var subDir in Directory.GetDirectories(dir)) + { + var file = Directory.GetFiles(subDir).FirstOrDefault(); + if (file == null) continue; + + var index = subDir.Split(Path.DirectorySeparatorChar).Last(); + var contentType = GetFileContentType(file); + + if ((!isNeedScreenShot || (isNeedScreenShot && !onlyScreenShot)) && _allowedImageTypes.Contains(contentType)) + { + var model = new MessageFileModel() + { + MessageId = messageId, + FileStorageUrl = file, + ContentType = contentType + }; + files.Add(model); + } + else if ((isNeedScreenShot && !onlyScreenShot || onlyScreenShot) && !_allowedImageTypes.Contains(contentType)) + { + var screenShotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER); + if (ExistDirectory(screenShotDir) && Directory.GetFiles(screenShotDir).Any()) + { + file = Directory.GetFiles(screenShotDir).First(); + contentType = GetFileContentType(file); + + var model = new MessageFileModel() + { + MessageId = messageId, + FileStorageUrl = file, + ContentType = contentType + }; + files.Add(model); + } + else + { + await web.GoToPage(contextId, file); + var path = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER, $"{Guid.NewGuid()}.png"); + await web.ScreenshotAsync(contextId, path); + contentType = GetFileContentType(path); + + var model = new MessageFileModel() + { + MessageId = messageId, + FileStorageUrl = path, + ContentType = contentType + }; + files.Add(model); + } + } + } + } + + if (isNeedScreenShot) + { + await web.CloseBrowser(contextId); + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when reading conversation ({conversationId}) files: {ex.Message}"); + } + return files; } @@ -102,7 +194,7 @@ public partial class BotSharpFileService { if (string.IsNullOrEmpty(conversationId)) return false; - var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, USER_FILE_FOLDER); + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); if (!ExistDirectory(dir)) return false; return Directory.GetDirectories(dir).Count() > 0; diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index a195af01..f06bc0a3 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -13,11 +13,12 @@ public partial class BotSharpFileService : IBotSharpFileService private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _allowedImageTypes = new List { "image/png", "image/jpeg" }; - private readonly IEnumerable _allowScreenShotTypes = new List { ".pdf" }; + private readonly IEnumerable _allowScreenShotTypes = new List { "pdf" }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; + private const string SCREENSHOT_FILE_FOLDER = "screenshot"; private const string BOT_FILE_FOLDER = "bot"; private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs index 3db691e2..8dfb94b5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs @@ -11,6 +11,7 @@ public class LoadAttachmentFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a"; + private readonly IEnumerable _imageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; public LoadAttachmentFn( IServiceProvider services, @@ -22,17 +23,19 @@ public class LoadAttachmentFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { + var args = JsonSerializer.Deserialize(message.FunctionArgs); var conv = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); + var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List(); + var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); var agent = await agentService.LoadAgent(AIAssistant); var fileAgent = new Agent { Id = agent.Id, Name = agent.Name, - Instruction = "Please describe the images.", + Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.", TemplateDict = new Dictionary() }; @@ -42,15 +45,16 @@ public class LoadAttachmentFn : IFunctionCallback return true; } - private List AssembleFiles(string conversationId, List dialogs) + private async Task> AssembleFiles(string conversationId, List dialogs, List fileTypes) { if (dialogs.IsNullOrEmpty()) { return new List(); } + var parsedTypes = ParseFileTypes(fileTypes); var fileService = _services.GetRequiredService(); - var files = fileService.GetChatImages(conversationId, FileSourceType.User, dialogs); + var files = await fileService.GetChatImages(conversationId, FileSourceType.User, parsedTypes, dialogs); foreach (var dialog in dialogs) { @@ -59,7 +63,6 @@ public class LoadAttachmentFn : IFunctionCallback dialog.Files = found.Select(x => new BotSharpFile { - FileName = x.FileName, ContentType = x.ContentType, FileStorageUrl = x.FileStorageUrl }).ToList(); @@ -68,6 +71,38 @@ public class LoadAttachmentFn : IFunctionCallback return dialogs; } + private IEnumerable ParseFileTypes(IEnumerable fileTypes) + { + var imageType = "image"; + var pdfType = "pdf"; + var parsed = new List(); + + if (fileTypes.IsNullOrEmpty()) + { + return new List { imageType }; + } + + foreach (var fileType in fileTypes) + { + var type = fileType?.Trim(); + if (string.IsNullOrWhiteSpace(type) || _imageTypes.Any(x => type.IsEqualTo(x))) + { + parsed.Add(imageType); + } + else if (type.IsEqualTo("pdf")) + { + parsed.Add(pdfType); + } + } + + if (parsed.IsNullOrEmpty()) + { + parsed.Add(imageType); + } + + return parsed.Distinct(); + } + private async Task GetChatCompletion(Agent agent, List dialogs) { try diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index b8062acc..0a1d74b4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -24,27 +24,33 @@ public class AttachmentProcessingHook : AgentHookBase { var json = JsonSerializer.Serialize(new { - user_question = new + user_request = new { type = "string", - description = $"The question asked by user, which is related to analyzing images or other files." + description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." + }, + file_types = new + { + type = "string", + description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'" } }); functions.Add(new FunctionDef { Name = "load_attachment", - Description = $"If the user's request is related to analyzing files, you can call this function to analyze files.", + Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", Parameters = { Properties = JsonSerializer.Deserialize(json), Required = new List { - "user_question" + "user_request", + "file_types" } } }); } - return true; + return base.OnFunctionsLoaded(functions); ; } }