diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 570f5c92..51e18819 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -25,6 +25,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs similarity index 67% rename from src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index dd91a2bb..4a985950 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Abstraction.Files; -public interface IBotSharpFileService +public interface IFileBasicService { #region Conversation /// @@ -28,7 +28,7 @@ public interface IBotSharpFileService /// /// /// - IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); bool SaveMessageFiles(string conversationId, string messageId, string source, List files); @@ -45,38 +45,18 @@ public interface IBotSharpFileService bool DeleteConversationFiles(IEnumerable conversationIds); #endregion - #region Image - Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); - #endregion - - #region Pdf - /// - /// Take screenshots of pdf pages and get response from llm - /// - /// - /// Pdf files - /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); - #endregion - #region User string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion #region Common - /// - /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" - /// - /// - /// - (string, byte[]) GetFileInfoFromData(string data); string GetDirectory(string conversationId); - string GetFileContentType(string filePath); byte[] GetFileBytes(string fileStorageUrl); bool SavefileToPath(string filePath, Stream stream); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs new file mode 100644 index 00000000..7d717fd0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Abstraction.Files; + +public interface IFileInstructService +{ + #region Image + Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task GenerateImage(string? provider, string? model, string text); + Task VaryImage(string? provider, string? model, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + #endregion + + #region Pdf + /// + /// Take screenshots of pdf pages and get response from llm + /// + /// + /// Pdf files + /// + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + #endregion + + #region Select file + Task> SelectMessageFiles(string conversationId, + string? agentId = null, string? template = null, bool includeBotFile = false, bool fromBreakpoint = false, + int? offset = null, IEnumerable? contentTypes = null); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs new file mode 100644 index 00000000..d13b4f1e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileSelectContext +{ + [JsonPropertyName("selected_ids")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Selecteds { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs new file mode 100644 index 00000000..df33906d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -0,0 +1,40 @@ +using Microsoft.AspNetCore.StaticFiles; + +namespace BotSharp.Abstraction.Files.Utilities; + +public static class FileUtility +{ + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + public static (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + + public static string GetFileContentType(string filePath) + { + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) + { + contentType = string.Empty; + } + + return contentType; + } +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 667e7e71..dec4d9f6 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -45,6 +45,15 @@ 1701;1702 + + + + + + + + + @@ -69,6 +78,7 @@ + @@ -155,6 +165,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest @@ -172,7 +185,6 @@ - @@ -181,9 +193,4 @@ - - - - - diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 451cdeed..3d6cc79b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -5,7 +5,7 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true); fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 87beba41..74b49d3d 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -37,7 +37,7 @@ public partial class ConversationService : IConversationService public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var isDeleted = db.DeleteConversations(ids); fileService.DeleteConversationFiles(ids); return await Task.FromResult(isDeleted); diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index 90397b93..d429ca28 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -20,7 +20,8 @@ public class FilePlugin : IBotSharpPlugin if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) { - services.AddScoped(); + services.AddScoped(); } + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs new file mode 100644 index 00000000..df0417fb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs @@ -0,0 +1,54 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileBasicService +{ + public string GetDirectory(string conversationId) + { + var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return dir; + } + + public byte[] GetFileBytes(string fileStorageUrl) + { + using var stream = File.OpenRead(fileStorageUrl); + var bytes = new byte[stream.Length]; + stream.Read(bytes, 0, (int)stream.Length); + return bytes; + } + + public bool SavefileToPath(string filePath, Stream stream) + { + using (var fileStream = new FileStream(filePath, FileMode.Create)) + { + stream.CopyTo(fileStream); + } + return true; + } + + public string BuildDirectory(params string[] segments) + { + var relativePath = Path.Combine(segments); + return Path.Combine(_baseDir, relativePath); + } + + public void CreateDirectory(string dir) + { + Directory.CreateDirectory(dir); + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); + } + + public void DeleteDirectory(string dir) + { + Directory.Delete(dir, true); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs similarity index 93% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index 1b98f0d9..f2624fda 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -4,7 +4,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileBasicService { public async Task> GetChatFiles(string conversationId, string source, IEnumerable conversations, IEnumerable contentTypes, @@ -29,7 +29,7 @@ public partial class BotSharpFileService var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); + var contentType = FileUtility.GetFileContentType(file); if (contentTypes?.Contains(contentType) != true) continue; var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); @@ -43,7 +43,7 @@ public partial class BotSharpFileService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -62,8 +62,8 @@ public partial class BotSharpFileService foreach (var file in Directory.GetFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) { continue; } @@ -141,7 +141,7 @@ public partial class BotSharpFileService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { @@ -180,7 +180,7 @@ public partial class BotSharpFileService { if (ExistDirectory(newDir)) { - Directory.Delete(newDir, true); + DeleteDirectory(newDir); } Directory.Move(prevDir, newDir); @@ -189,7 +189,7 @@ public partial class BotSharpFileService var botDir = Path.Combine(newDir, BOT_FILE_FOLDER); if (ExistDirectory(botDir)) { - Directory.Delete(botDir, true); + DeleteDirectory(botDir); } } } @@ -200,7 +200,7 @@ public partial class BotSharpFileService if (!ExistDirectory(dir)) continue; Thread.Sleep(100); - Directory.Delete(dir, true); + DeleteDirectory(dir); } return true; @@ -215,7 +215,7 @@ public partial class BotSharpFileService var convDir = GetConversationDirectory(conversationId); if (!ExistDirectory(convDir)) continue; - Directory.Delete(convDir, true); + DeleteDirectory(convDir); } return true; } @@ -248,13 +248,9 @@ public partial class BotSharpFileService { if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset.HasValue && offset < 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); @@ -285,7 +281,7 @@ public partial class BotSharpFileService { foreach (var screenShot in Directory.GetFiles(screenShotDir)) { - contentType = GetFileContentType(screenShot); + contentType = FileUtility.GetFileContentType(screenShot); if (!_imageTypes.Contains(contentType)) continue; var fileName = Path.GetFileNameWithoutExtension(screenShot); @@ -307,7 +303,7 @@ public partial class BotSharpFileService var images = await ConvertPdfToImages(file, screenShotDir); foreach (var image in images) { - contentType = GetFileContentType(image); + contentType = FileUtility.GetFileContentType(image); var fileName = Path.GetFileNameWithoutExtension(image); var fileType = Path.GetExtension(image).Substring(1); var model = new MessageFileModel() diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs index fa99b13f..f26763c9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileBasicService { public string GetUserAvatar() { @@ -30,11 +30,11 @@ public partial class BotSharpFileService if (Directory.Exists(dir)) { - Directory.Delete(dir, true); + DeleteDirectory(dir); } dir = GetUserAvatarDir(user?.Id, createNewDir: true); - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes); return true; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs similarity index 70% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs index b7c9a946..1d2079b9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs @@ -1,14 +1,13 @@ -using Microsoft.AspNetCore.StaticFiles; using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService : IBotSharpFileService +public partial class FileBasicService : IFileBasicService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _imageTypes = new List { @@ -25,13 +24,10 @@ public partial class BotSharpFileService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - public BotSharpFileService( + public FileBasicService( BotSharpDatabaseSettings dbSettings, IUserIdentity user, - ILogger logger, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; @@ -40,11 +36,4 @@ public partial class BotSharpFileService : IBotSharpFileService _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); - } - #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs deleted file mode 100644 index be5f3180..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Microsoft.AspNetCore.StaticFiles; -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ - public string GetDirectory(string conversationId) - { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - return dir; - } - - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - - public byte[] GetFileBytes(string fileStorageUrl) - { - using var stream = File.OpenRead(fileStorageUrl); - var bytes = new byte[stream.Length]; - stream.Read(bytes, 0, (int)stream.Length); - return bytes; - } - - public bool SavefileToPath(string filePath, Stream stream) - { - using (var fileStream = new FileStream(filePath, FileMode.Create)) - { - stream.CopyTo(fileStream); - } - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs similarity index 84% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 619360d2..c9d35cb7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -2,8 +2,24 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileInstructService { + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + { + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, text) + { + Files = images?.ToList() ?? new List() + } + }); + return message; + } + public async Task GenerateImage(string? provider, string? model, string text) { var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); @@ -31,7 +47,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -53,7 +69,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -82,7 +98,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - + imageStream.Close(); maskStream.Close(); return message; @@ -100,7 +116,7 @@ public partial class BotSharpFileService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } return bytes; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs similarity index 80% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index daca7711..4fbe01aa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,8 +1,9 @@ +using BotSharp.Abstraction.Files.Converters; using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileInstructService { public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { @@ -14,11 +15,9 @@ public partial class BotSharpFileService } var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); - if (!ExistDirectory(sessionDir)) - { - Directory.CreateDirectory(sessionDir); - } + + var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid); + DeleteIfExistDirectory(sessionDir); try { @@ -38,9 +37,7 @@ public partial class BotSharpFileService Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } }); - - content = message.Content; - return content; + return message.Content; } catch (Exception ex) { @@ -49,17 +46,11 @@ public partial class BotSharpFileService } finally { - Directory.Delete(sessionDir, true); + _fileBasic.DeleteDirectory(sessionDir); } } #region Private methods - private string GetSessionDirectory(string id) - { - var dir = Path.Combine(_baseDir, SESSION_FOLDER, id); - return dir; - } - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) @@ -81,19 +72,16 @@ public partial class BotSharpFileService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } if (!bytes.IsNullOrEmpty()) { var guid = Guid.NewGuid().ToString(); - var fileDir = Path.Combine(dir, guid); - if (!ExistDirectory(fileDir)) - { - Directory.CreateDirectory(fileDir); - } + var fileDir = _fileBasic.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir); - var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}"); + var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}"); using (var fs = new FileStream(pdfDir, FileMode.Create)) { fs.Write(bytes, 0, bytes.Length); @@ -115,7 +103,7 @@ public partial class BotSharpFileService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(); if (converter == null || files.IsNullOrEmpty()) { return images; @@ -127,7 +115,7 @@ public partial class BotSharpFileService { var segs = file.Split(Path.DirectorySeparatorChar); var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); + var folder = _fileBasic.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs new file mode 100644 index 00000000..e4602b9f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -0,0 +1,106 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task> SelectMessageFiles(string conversationId, + string? agentId = null, string? template = null, bool includeBotFile = false, bool fromBreakpoint = false, + int? offset = null, IEnumerable? contentTypes = null) + { + if (string.IsNullOrEmpty(conversationId)) + { + return Enumerable.Empty(); + } + + var convService = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint); + var messageIds = GetMessageIds(dialogs, offset); + + var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); + if (includeBotFile) + { + var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + files = files.Concat(botFiles); + } + + if (files.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + return await SelectFiles(agentId, template, files, dialogs); + } + + private async Task> SelectFiles(string? agentId, string? template, IEnumerable files, List dialogs) + { + if (files.IsNullOrEmpty()) return new List(); + + var llmProviderService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + try + { + var promptFiles = files.Select((x, idx) => + { + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + }).ToList(); + + agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant; + template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt"; + + var foundAgent = db.GetAgent(agentId); + var prompt = db.GetAgentTemplate(agentId, template); + prompt = render.Render(prompt, new Dictionary + { + { "file_list", promptFiles } + }); + + var agent = new Agent + { + Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = foundAgent?.Name ?? "Utility Assistant", + Instruction = prompt + }; + + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var latest = dialogs.Last(); + var response = await completion.GetChatCompletions(agent, new List { latest }); + var content = response?.Content ?? string.Empty; + var selecteds = JsonSerializer.Deserialize(content); + var fids = selecteds?.Selecteds ?? new List(); + return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when selecting files. {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + + private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + { + if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + + if (offset.HasValue && offset < 1) + { + offset = 1; + } + + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + + return messageIds; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs new file mode 100644 index 00000000..f5d7ede1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService : IFileInstructService +{ + private readonly IFileBasicService _fileBasic; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private const string SESSION_FOLDER = "sessions"; + + public FileInstructService( + IFileBasicService fileBasic, + ILogger logger, + IServiceProvider services) + { + _fileBasic = fileBasic; + _logger = logger; + _services = services; + } + + private void DeleteIfExistDirectory(string? dir) + { + if (_fileBasic.ExistDirectory(dir)) + { + _fileBasic.DeleteDirectory(dir); + } + else + { + _fileBasic.CreateDirectory(dir); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 88293d97..fa3e94d2 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -29,6 +29,7 @@ global using BotSharp.Abstraction.Translation; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Core.Repository; @@ -37,4 +38,4 @@ global using BotSharp.Core.Agents.Services; global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; -global using Aspects.Cache; +global using Aspects.Cache; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid new file mode 100644 index 00000000..f1267212 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid @@ -0,0 +1,15 @@ +Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. + +** Ensure the output is only in JSON format without any additional text. +** If no files are selected, you must output an empty list []. +** You may need to look at the file_name as a reference to find the correct file id or ids. + +Here is the JSON format to use: +{ + "selected_ids": a list of id selected from the [FILES] section +} + +[FILES] +{% for file in file_list -%} +{{ file }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 261eb04b..a0d65f8a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -81,7 +81,7 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); @@ -349,7 +349,7 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var dir = fileService.GetDirectory(conversationId); foreach (var file in files) { @@ -372,7 +372,7 @@ public class ConversationController : ControllerBase var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); var conv = await convService.GetConversationRecordOrCreateNew(agentId); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); return isSaved ? messageId : string.Empty; @@ -381,15 +381,15 @@ public class ConversationController : ControllerBase [HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")] public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { - var fileService = _services.GetRequiredService(); - var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source, imageOnly: false); + var fileService = _services.GetRequiredService(); + var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source); return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName); if (string.IsNullOrEmpty(file)) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 2189af2f..48fdafe1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using NetTopologySuite.IO; namespace BotSharp.OpenAPI.Controllers; @@ -87,18 +86,8 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - model: input.Model ?? "gpt-4o", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, input.Text) - { - Files = input.Files - } - }); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); return message.Content; } catch (Exception ex) @@ -114,14 +103,14 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-generation")] public async Task ImageGeneration([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -140,7 +129,6 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-variation")] public async Task ImageVariation([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -152,7 +140,9 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.VaryImage(input.Provider, input.Model, image); + + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -169,7 +159,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -181,7 +171,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -198,7 +188,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -211,7 +201,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image, mask); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -236,8 +226,8 @@ public class InstructModeController : ControllerBase try { - var fileService = _services.GetRequiredService(); - var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); viewModel.Content = content; return viewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 966f51c9..74db3bd0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -137,14 +137,14 @@ public class UserController : ControllerBase [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] BotSharpFile file) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); return fileService.SaveUserAvatar(file); } [HttpGet("/user/avatar")] public IActionResult GetUserAvatar() { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var file = fileService.GetUserAvatar(); if (string.IsNullOrEmpty(file)) { @@ -158,7 +158,7 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var bytes = fileService.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 16e2841a..72034317 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat; @@ -196,7 +197,6 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -270,13 +270,13 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); using var stream = File.OpenRead(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 45dc65be..072c22d8 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -74,13 +74,11 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var conversationId = convService.ConversationId; - var dialogs = convService.GetDialogHistory(fromBreakpoint: false); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); - var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot); - return await SelectFiles(userFiles.Concat(botFiles), dialogs); + + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true); + return selecteds; } private async Task> SelectFiles(IEnumerable files, List dialogs) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 63938c5e..52cb1249 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -28,7 +28,7 @@ public class EditImageFn : IFunctionCallback Init(message); SetImageOptions(); - var image = await SelectConversationImage(descrpition); + var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; return true; @@ -48,64 +48,11 @@ public class EditImageFn : IFunctionCallback state.SetState("image_count", "1"); } - private async Task SelectConversationImage(string? description) + private async Task SelectImage(string? description) { - var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); - return await SelectImage(userImages, dialogs.LastOrDefault(), description); - } - - private async Task SelectImage(IEnumerable images, RoleDialogModel message, string? description) - { - if (images.IsNullOrEmpty()) return null; - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) => - { - return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}"; - }).ToList(); - - if (promptImages.IsNullOrEmpty()) return null; - - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "image_list", promptImages } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - - var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; - var dialog = RoleDialogModel.From(message, AgentRole.User, text); - - var response = await completion.GetChatCompletions(agent, new List { dialog }); - var content = response?.Content ?? string.Empty; - var selected = JsonSerializer.Deserialize(content); - var fid = selected?.Selected ?? -1; - return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}"); - return null; - } + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, contentTypes: new List { MediaTypeNames.Image.Png }); + return selecteds?.FirstOrDefault(); } private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) @@ -154,7 +101,7 @@ public class EditImageFn : IFunctionCallback } }; - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4d53f880..3869a419 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" }).ToList(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 66f69353..cdff6cf4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -51,7 +51,7 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); foreach (var dialog in dialogs) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 85c2afbc..d3c21737 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -50,7 +50,7 @@ public class ReadPdfFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); foreach (var dialog in dialogs) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 23084ead..f12e8b34 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -197,7 +198,6 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -271,13 +271,13 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); using var stream = File.OpenRead(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e9631786..61b58c6a 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,5 +1,3 @@ -using Microsoft.AspNetCore.StaticFiles; - namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -9,41 +7,11 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - public byte[] GetFileBytes(string fileStorageUrl) { try { var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); - return fileData; } catch (Exception ex) @@ -67,4 +35,24 @@ public partial class TencentCosService return false; } } + + public string BuildDirectory(params string[] segments) + { + return string.Join("/", segments); + } + + public void CreateDirectory(string dir) + { + + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); + } + + public void DeleteDirectory(string dir) + { + _cosClient.BucketClient.DeleteDir(dir); + } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index df39e0a9..207fb464 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Utilities; using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; @@ -28,7 +29,7 @@ public partial class TencentCosService var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); + var contentType = FileUtility.GetFileContentType(file); if (contentTypes?.Contains(contentType) != true) continue; var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); @@ -42,7 +43,7 @@ public partial class TencentCosService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -59,8 +60,8 @@ public partial class TencentCosService { foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) { continue; } @@ -135,7 +136,7 @@ public partial class TencentCosService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = $"{dir}/{source}/{i + 1}"; @@ -225,13 +226,9 @@ public partial class TencentCosService { if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset <= 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); @@ -264,7 +261,7 @@ public partial class TencentCosService { foreach (var screenShot in fileList) { - contentType = GetFileContentType(screenShot); + contentType = FileUtility.GetFileContentType(screenShot); if (!_imageTypes.Contains(contentType)) continue; var fileName = Path.GetFileNameWithoutExtension(screenShot); @@ -286,7 +283,7 @@ public partial class TencentCosService var images = await ConvertPdfToImages(file, screenShotDir); foreach (var image in images) { - contentType = GetFileContentType(image); + contentType = FileUtility.GetFileContentType(image); var fileName = Path.GetFileNameWithoutExtension(image); var fileType = Path.GetExtension(image).Substring(1); var model = new MessageFileModel() diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs deleted file mode 100644 index e8628ce3..00000000 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace BotSharp.Plugin.TencentCos.Services; - -public partial class TencentCosService -{ - public async Task GenerateImage(string? provider, string? model, string text) - { - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); - var message = await completion.GetImageGeneration(new Agent() - { - Id = Guid.Empty.ToString(), - }, new RoleDialogModel(AgentRole.User, text)); - return message; - } - - public async Task VaryImage(string? provider, string? model, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageVariation(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) - { - if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || - (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) - { - throw new ArgumentException($"Cannot find image/mask url or data"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var imageBytes = await DownloadFile(image); - var maskBytes = await DownloadFile(mask); - - using var imageStream = new MemoryStream(); - imageStream.Write(imageBytes, 0, imageBytes.Length); - imageStream.Position = 0; - - using var maskStream = new MemoryStream(); - maskStream.Write(maskBytes, 0, maskBytes.Length); - maskStream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - - imageStream.Close(); - maskStream.Close(); - return message; - } - - #region Private methods - private async Task DownloadFile(BotSharpFile file) - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - return bytes; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs deleted file mode 100644 index 1efbad6d..00000000 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs +++ /dev/null @@ -1,130 +0,0 @@ -namespace BotSharp.Plugin.TencentCos.Services; - -public partial class TencentCosService -{ - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) - { - var content = string.Empty; - - if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) - { - return content; - } - - var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); - - try - { - var pdfFiles = await DownloadFiles(sessionDir, files); - var images = await ConvertPdfToImages(pdfFiles); - if (images.IsNullOrEmpty()) return content; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", - model: model, modelId: modelId ?? "gpt-4", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, prompt) - { - Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() - } - }); - - content = message.Content; - return content; - } - catch (Exception ex) - { - _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}"); - return content; - } - finally - { - Directory.Delete(sessionDir, true); - } - } - - #region Private methods - private string GetSessionDirectory(string id) - { - var dir = $"{SESSION_FOLDER}/{id}"; - return dir; - } - - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") - { - if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var locs = new List(); - foreach (var file in files) - { - try - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - if (!bytes.IsNullOrEmpty()) - { - var guid = Guid.NewGuid().ToString(); - var fileDir = $"{dir}/{guid}"; - - var pdfDir = $"{fileDir}/{guid}.{extension}"; - - - _cosClient.BucketClient.UploadBytes(pdfDir, bytes); - locs.Add(pdfDir); - } - } - catch (Exception ex) - { - _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return locs; - } - - private async Task> ConvertPdfToImages(IEnumerable files) - { - var images = new List(); - var converter = GetPdf2ImageConverter(); - if (converter == null || files.IsNullOrEmpty()) - { - return images; - } - - foreach (var file in files) - { - try - { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); - var urls = await converter.ConvertPdfToImages(file, folder); - images.AddRange(urls); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return images; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 23ce68c0..55e26d81 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Files.Utilities; + namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -26,10 +28,8 @@ public partial class TencentCosService if (string.IsNullOrEmpty(dir)) return false; - var (_, bytes) = GetFileInfoFromData(file.FileData); - + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var extension = Path.GetExtension(file.FileName); - var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 80b8fd78..78c6bcc9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -5,8 +5,9 @@ using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; -public partial class TencentCosService : IBotSharpFileService +public partial class TencentCosService : IFileBasicService { + private readonly TencentCosClient _cosClient; private readonly TencentCosSettings _settings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; @@ -27,10 +28,6 @@ public partial class TencentCosService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - private readonly TencentCosClient _cosClient; public TencentCosService( TencentCosSettings settings, @@ -46,11 +43,4 @@ public partial class TencentCosService : IBotSharpFileService _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; _cosClient = cosClient; } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); - } - #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 93a99c14..25cdb277 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -31,7 +31,7 @@ public class TencentCosPlugin : IBotSharpPlugin services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } }