diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index 4a137070..65db27d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -3,7 +3,9 @@ namespace BotSharp.Abstraction.Files; public interface IBotSharpFileService { string GetDirectory(string conversationId); - Task> GetChatImages(string conversationId, string source, IEnumerable fileTypes, List conversations, int? offset = null); + Task> GetChatImages(string conversationId, string source, + IEnumerable conversations, IEnumerable contentTypes, + bool includeScreenShot = false, int? offset = null); IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs index d8a2d7c6..694bccb0 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs @@ -1,124 +1,47 @@ using AspectInjector.Broker; +using Azure.Core; using BotSharp.Abstraction.Files.Converters; using Microsoft.EntityFrameworkCore; using System.IO; +using System.Net.Mime; using System.Threading; namespace BotSharp.Core.Files.Services; public partial class BotSharpFileService { - public async Task> GetChatImages(string conversationId, string source, IEnumerable fileTypes, - List conversations, int? offset = null) + public async Task> GetChatImages(string conversationId, string source, + IEnumerable conversations, IEnumerable contentTypes, + bool includeScreenShot = false, int? offset = null) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) { - return new List(); + return files; } - if (offset <= 0) + var messageIds = GetMessageIds(conversations, offset); + var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); + + foreach (var messageId in messageIds) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; - } + var dir = Path.Combine(pathPrefix, messageId, source); + if (!ExistDirectory(dir)) continue; - var messageIds = new List(); - if (offset.HasValue) - { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); - } - - files = await GetMessageFiles(conversationId, messageIds, source, fileTypes); - return files; - } - - private async Task> GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable fileTypes) - { - var files = new List(); - if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty() || fileTypes.IsNullOrEmpty()) return files; - - var isNeedScreenShot = fileTypes.Any(x => _allowScreenShotTypes.Contains(x)); - var onlyScreenShot = fileTypes.All(x => _allowScreenShotTypes.Contains(x)); - - try - { - var preFixPath = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); - - foreach (var messageId in messageIds) + foreach (var subDir in Directory.GetDirectories(dir)) { - var dir = Path.Combine(preFixPath, messageId, source); - if (!ExistDirectory(dir)) continue; + var file = Directory.GetFiles(subDir).FirstOrDefault(); + if (file == null) continue; - foreach (var subDir in Directory.GetDirectories(dir)) - { - var file = Directory.GetFiles(subDir).FirstOrDefault(); - if (file == null) continue; + var contentType = GetFileContentType(file); + if (contentTypes?.Contains(contentType) != true) continue; - var index = subDir.Split(Path.DirectorySeparatorChar).Last(); - var contentType = GetFileContentType(file); + var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); + if (foundFiles.IsNullOrEmpty()) continue; - if ((!isNeedScreenShot || (isNeedScreenShot && !onlyScreenShot)) && _allowedImageTypes.Contains(contentType)) - { - var model = new MessageFileModel() - { - MessageId = messageId, - FileStorageUrl = file, - ContentType = contentType - }; - files.Add(model); - } - else if ((isNeedScreenShot && !onlyScreenShot || onlyScreenShot) && !_allowedImageTypes.Contains(contentType)) - { - var screenShotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER); - if (ExistDirectory(screenShotDir) && Directory.GetFiles(screenShotDir).Any()) - { - foreach (var screenShot in Directory.GetFiles(screenShotDir)) - { - contentType = GetFileContentType(screenShot); - if (!_allowedImageTypes.Contains(contentType)) continue; - - var model = new MessageFileModel() - { - MessageId = messageId, - FileStorageUrl = screenShot, - ContentType = contentType - }; - files.Add(model); - } - } - else - { - var screenShotPath = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER); - var images = await ConvertPdfToImages(file, screenShotPath); - - foreach (var image in images) - { - contentType = GetFileContentType(image); - var model = new MessageFileModel() - { - MessageId = messageId, - FileStorageUrl = image, - ContentType = contentType - }; - files.Add(model); - } - } - } - } + files.AddRange(foundFiles); } } - catch (Exception ex) - { - _logger.LogWarning($"Error when reading conversation ({conversationId}) files: {ex.Message}\r\n{ex.InnerException}\r\n{ex.StackTrace}"); - } return files; } @@ -144,7 +67,7 @@ public partial class BotSharpFileService foreach (var file in Directory.GetFiles(subDir)) { var contentType = GetFileContentType(file); - if (imageOnly && !_allowedImageTypes.Contains(contentType)) + if (imageOnly && !_imageTypes.Contains(contentType)) { continue; } @@ -327,6 +250,102 @@ public partial class BotSharpFileService return dir; } + private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + { + if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + + if (offset <= 0) + { + offset = MIN_OFFSET; + } + else if (offset > MAX_OFFSET) + { + offset = MAX_OFFSET; + } + + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + + return messageIds; + } + + + private async Task> GetMessageFiles(string file, string fileDir, string contentType, + string messageId, string source, bool includeScreenShot) + { + var files = new List(); + + try + { + if (!_imageTypes.Contains(contentType) && includeScreenShot) + { + var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER); + if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty()) + { + foreach (var screenShot in Directory.GetFiles(screenShotDir)) + { + contentType = GetFileContentType(screenShot); + if (!_imageTypes.Contains(contentType)) continue; + + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = Path.GetFileName(screenShot), + FileStorageUrl = screenShot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenShotDir); + foreach (var image in images) + { + contentType = GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = Path.GetFileName(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + } + else + { + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = Path.GetFileName(file), + FileStorageUrl = file, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } + + private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs index 602e4316..f915b51e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.StaticFiles; using System.IO; +using System.Net.Mime; namespace BotSharp.Core.Files.Services; @@ -10,8 +11,11 @@ public partial class BotSharpFileService : IBotSharpFileService private readonly IUserIdentity _user; private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _allowedImageTypes = new List { "image/png", "image/jpeg" }; - private readonly IEnumerable _allowScreenShotTypes = new List { "pdf" }; + private readonly IEnumerable _imageTypes = new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; diff --git a/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs index 83b3a6c5..9543b9fa 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs @@ -5,6 +5,7 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.MLTasks; using Microsoft.Extensions.Logging; using BotSharp.Core.Infrastructures; +using System.Net.Mime; namespace BotSharp.Plugin.FileHandler.Functions; @@ -16,8 +17,11 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private const string DEFAULT_IMAGE = "image"; - private readonly IEnumerable _targetImageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; + private readonly IEnumerable _imageContentTypes = new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg, + }; public ReadImageFn( IServiceProvider services, @@ -57,7 +61,7 @@ public class ReadImageFn : IFunctionCallback } var fileService = _services.GetRequiredService(); - var images = await fileService.GetChatImages(conversationId, FileSourceType.User, new List { "image" }, dialogs); + var images = await fileService.GetChatImages(conversationId, FileSourceType.User, dialogs, _imageContentTypes); foreach (var dialog in dialogs) { @@ -88,7 +92,7 @@ public class ReadImageFn : IFunctionCallback catch (Exception ex) { var error = $"Error when analyzing images."; - _logger.LogWarning($"{error} {ex.Message}"); + _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}"); return error; } } diff --git a/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs index 00cf8ded..29c63033 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs @@ -5,6 +5,7 @@ using BotSharp.Abstraction.Files; using BotSharp.Abstraction.MLTasks; using BotSharp.Core.Infrastructures; using Microsoft.Extensions.Logging; +using System.Net.Mime; namespace BotSharp.Plugin.FileHandler.Functions; @@ -16,7 +17,10 @@ public class ReadPdfFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private const string DEFAULT_PDF = "pdf"; + private readonly IEnumerable _pdfContentTypes = new List + { + MediaTypeNames.Application.Pdf + }; public ReadPdfFn( IServiceProvider services, @@ -56,7 +60,7 @@ public class ReadPdfFn : IFunctionCallback } var fileService = _services.GetRequiredService(); - var files = await fileService.GetChatImages(conversationId, FileSourceType.User, new List { "pdf" }, dialogs); + var files = await fileService.GetChatImages(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); foreach (var dialog in dialogs) { @@ -87,7 +91,7 @@ public class ReadPdfFn : IFunctionCallback catch (Exception ex) { var error = $"Error when analyzing pdf file(s)."; - _logger.LogWarning($"{error} {ex.Message}"); + _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}"); return error; } } diff --git a/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs b/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs index 435518e3..5a0b8adf 100644 --- a/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs @@ -30,20 +30,6 @@ public class FileHandlerHook : AgentHookBase, IAgentHook base.OnAgentLoaded(agent); } - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - - private (string, FunctionDef?) GetPromptAndFunction(string functionName) - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); - return (prompt, loadAttachmentFn); - } - private void AddUtility(Agent agent, string utility, string functionName) { if (!IsEnableUtility(agent, utility)) return; @@ -66,4 +52,18 @@ public class FileHandlerHook : AgentHookBase, IAgentHook } } } + + private bool IsEnableUtility(Agent agent, string utility) + { + return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); + } + + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs index ef3ce916..20d2d012 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs @@ -1,4 +1,5 @@ using System.Net.Http; +using System.Net.Mime; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -150,7 +151,7 @@ public class HandleHttpRequestFn : IFunctionCallback _logger.LogWarning($"Error when build http content: {content}\n(Error: {ex.Message})"); } - return new StringContent(str, Encoding.UTF8, "application/json"); + return new StringContent(str, Encoding.UTF8, MediaTypeNames.Application.Json); } private string BuildQuery(string url, string? content)