diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 446b8869..51e18819 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -25,6 +25,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs new file mode 100644 index 00000000..dab5bc81 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Files.Constants; + +public class FileConstants +{ + public static readonly IEnumerable AudioExtensions = new List + { + ".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma" + }; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs similarity index 62% rename from src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index dd91a2bb..b0e5d261 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Abstraction.Files; -public interface IBotSharpFileService +public interface IFileBasicService { #region Conversation /// @@ -11,13 +11,13 @@ public interface IBotSharpFileService /// /// /// - /// + /// /// /// /// /// Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable dialogs, IEnumerable? contentTypes, bool includeScreenShot = false, int? offset = null); /// @@ -28,7 +28,7 @@ public interface IBotSharpFileService /// /// /// - IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); bool SaveMessageFiles(string conversationId, string messageId, string source, List files); @@ -45,38 +45,20 @@ public interface IBotSharpFileService bool DeleteConversationFiles(IEnumerable conversationIds); #endregion - #region Image - Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); - #endregion - - #region Pdf - /// - /// Take screenshots of pdf pages and get response from llm - /// - /// - /// Pdf files - /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); - #endregion - #region User string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion #region Common - /// - /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" - /// - /// - /// - (string, byte[]) GetFileInfoFromData(string data); string GetDirectory(string conversationId); - string GetFileContentType(string filePath); byte[] GetFileBytes(string fileStorageUrl); - bool SavefileToPath(string filePath, Stream stream); + bool SaveFileStreamToPath(string filePath, Stream stream); + bool SaveFileBytesToPath(string filePath, byte[] bytes); + string GetParentDir(string dir, int level = 1); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs new file mode 100644 index 00000000..433a582f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -0,0 +1,29 @@ +namespace BotSharp.Abstraction.Files; + +public interface IFileInstructService +{ + #region Image + Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task GenerateImage(string? provider, string? model, string text); + Task VaryImage(string? provider, string? model, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + #endregion + + #region Pdf + /// + /// Take screenshots of pdf pages and get response from llm + /// + /// + /// Pdf files + /// + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + #endregion + + #region Select file + Task> SelectMessageFiles(string conversationId, + string? agentId = null, string? template = null, string? description = null, + bool includeBotFile = false, bool fromBreakpoint = false, + int? offset = null, IEnumerable? contentTypes = null); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs new file mode 100644 index 00000000..d13b4f1e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileSelectContext +{ + [JsonPropertyName("selected_ids")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Selecteds { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 7cd93269..05568e66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -39,6 +39,6 @@ public class MessageFileModel public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}"; + return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs new file mode 100644 index 00000000..df33906d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -0,0 +1,40 @@ +using Microsoft.AspNetCore.StaticFiles; + +namespace BotSharp.Abstraction.Files.Utilities; + +public static class FileUtility +{ + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + public static (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + + public static string GetFileContentType(string filePath) + { + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) + { + contentType = string.Empty; + } + + return contentType; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 89c45510..757d9937 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -1,7 +1,12 @@ +using BotSharp.Abstraction.Routing.Models; +using System.Collections.Concurrent; + namespace BotSharp.Core.Agents.Services; public partial class AgentService { + public static ConcurrentDictionary> AgentParameterTypes = new(); + [MemoryCache(10 * 60, perInstanceCache: true)] public async Task LoadAgent(string id) { @@ -49,6 +54,7 @@ public partial class AgentService agent.Instruction = inheritedAgent.Instruction; } } + AddOrUpdateParameters(agent); agent.TemplateDict = new Dictionary(); @@ -96,4 +102,43 @@ public partial class AgentService dict[t.Key] = t.Value; } } + + private void AddOrUpdateParameters(Agent agent) + { + var agentId = agent.Id ?? agent.Name; + if (AgentParameterTypes.ContainsKey(agentId)) return; + + AddOrUpdateRoutesParameters(agentId, agent.RoutingRules); + AddOrUpdateFunctionsParameters(agentId, agent.Functions); + } + + private void AddOrUpdateRoutesParameters(string agentId, List routingRules) + { + if(!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new(); + foreach (var rule in routingRules.Where(x => x.Required)) + { + if (string.IsNullOrEmpty(rule.FieldType)) continue; + parameterTypes.TryAdd(rule.Field, rule.FieldType); + } + AgentParameterTypes.TryAdd(agentId, parameterTypes); + } + + private void AddOrUpdateFunctionsParameters(string agentId, List functions) + { + if (!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new(); + var parameters = functions.Select(p => p.Parameters); + foreach (var param in parameters) + { + foreach (JsonProperty prop in param.Properties.RootElement.EnumerateObject()) + { + var name = prop.Name; + var node = prop.Value; + if (node.TryGetProperty("type", out var type)) + { + parameterTypes.TryAdd(name, type.GetString()); + } + } + } + AgentParameterTypes.TryAdd(agentId, parameterTypes); + } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 1b58a984..279d735c 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -45,6 +45,15 @@ 1701;1702 + + + + + + + + + @@ -69,6 +78,7 @@ + @@ -155,6 +165,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest @@ -172,7 +185,6 @@ - @@ -181,9 +193,4 @@ - - - - - diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 26c62b62..36b588ae 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -150,6 +150,7 @@ public partial class ConversationService await HookEmitter.Emit(_services, async hook => await hook.OnConversationEnding(response) ); + response.FunctionName = "conversation_end"; } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 451cdeed..3d6cc79b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -5,7 +5,7 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true); fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 87beba41..74b49d3d 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -37,7 +37,7 @@ public partial class ConversationService : IConversationService public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var isDeleted = db.DeleteConversations(ids); fileService.DeleteConversationFiles(ids); return await Task.FromResult(isDeleted); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 57f09c71..aa518ee7 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -360,9 +360,28 @@ public class ConversationStateService : IConversationStateService, IDisposable stateValue = stateValue?.ToLower(); } - SetState(property.Name, stateValue, source: StateSource.Application); + if (CheckArgType(property.Name, stateValue)) + { + SetState(property.Name, stateValue, source: StateSource.Application); + } } } } } + + private bool CheckArgType(string name, string value) + { + var agentTypes = AgentService.AgentParameterTypes.SelectMany(p => p.Value).ToList(); + var filed = agentTypes.FirstOrDefault(t => t.Key == name); + if (filed.Key != null) + { + return filed.Value switch + { + "boolean" => bool.TryParse(value, out _), + "number" => long.TryParse(value, out _), + _ => true, + }; + } + return true; + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index 90397b93..d429ca28 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -20,7 +20,8 @@ public class FilePlugin : IBotSharpPlugin if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) { - services.AddScoped(); + services.AddScoped(); } + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs new file mode 100644 index 00000000..a1208a31 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs @@ -0,0 +1,73 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileBasicService +{ + public string GetDirectory(string conversationId) + { + var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return dir; + } + + public byte[] GetFileBytes(string fileStorageUrl) + { + using var stream = File.OpenRead(fileStorageUrl); + var bytes = new byte[stream.Length]; + stream.Read(bytes, 0, (int)stream.Length); + return bytes; + } + + public bool SaveFileStreamToPath(string filePath, Stream stream) + { + if (string.IsNullOrEmpty(filePath)) return false; + + using (var fileStream = new FileStream(filePath, FileMode.Create)) + { + stream.CopyTo(fileStream); + } + return true; + } + + public bool SaveFileBytesToPath(string filePath, byte[] bytes) + { + using (var fs = new FileStream(filePath, FileMode.Create)) + { + fs.Write(bytes, 0, bytes.Length); + fs.Flush(); + fs.Close(); + } + return true; + } + + public string GetParentDir(string dir, int level = 1) + { + var segs = dir.Split(Path.DirectorySeparatorChar); + return string.Join(Path.DirectorySeparatorChar, segs.SkipLast(level)); + } + + public string BuildDirectory(params string[] segments) + { + var relativePath = Path.Combine(segments); + return Path.Combine(_baseDir, relativePath); + } + + public void CreateDirectory(string dir) + { + Directory.CreateDirectory(dir); + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); + } + + public void DeleteDirectory(string dir) + { + Directory.Delete(dir, true); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs similarity index 88% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index 1b98f0d9..403e2da5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -4,19 +4,19 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileBasicService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable dialogs, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(conversations, offset); + var messageIds = GetMessageIds(dialogs, offset); var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) @@ -29,8 +29,11 @@ public partial class BotSharpFileService var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + { + continue; + } var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); if (foundFiles.IsNullOrEmpty()) continue; @@ -43,7 +46,7 @@ public partial class BotSharpFileService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -62,8 +65,8 @@ public partial class BotSharpFileService foreach (var file in Directory.GetFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } @@ -141,7 +144,7 @@ public partial class BotSharpFileService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { @@ -180,7 +183,7 @@ public partial class BotSharpFileService { if (ExistDirectory(newDir)) { - Directory.Delete(newDir, true); + DeleteDirectory(newDir); } Directory.Move(prevDir, newDir); @@ -189,7 +192,7 @@ public partial class BotSharpFileService var botDir = Path.Combine(newDir, BOT_FILE_FOLDER); if (ExistDirectory(botDir)) { - Directory.Delete(botDir, true); + DeleteDirectory(botDir); } } } @@ -200,7 +203,7 @@ public partial class BotSharpFileService if (!ExistDirectory(dir)) continue; Thread.Sleep(100); - Directory.Delete(dir, true); + DeleteDirectory(dir); } return true; @@ -215,7 +218,7 @@ public partial class BotSharpFileService var convDir = GetConversationDirectory(conversationId); if (!ExistDirectory(convDir)) continue; - Directory.Delete(convDir, true); + DeleteDirectory(convDir); } return true; } @@ -244,27 +247,23 @@ public partial class BotSharpFileService return dir; } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset.HasValue && offset < 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); if (offset.HasValue) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); } else { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); } return messageIds; @@ -285,7 +284,7 @@ public partial class BotSharpFileService { foreach (var screenShot in Directory.GetFiles(screenShotDir)) { - contentType = GetFileContentType(screenShot); + contentType = FileUtility.GetFileContentType(screenShot); if (!_imageTypes.Contains(contentType)) continue; var fileName = Path.GetFileNameWithoutExtension(screenShot); @@ -307,7 +306,7 @@ public partial class BotSharpFileService var images = await ConvertPdfToImages(file, screenShotDir); foreach (var image in images) { - contentType = GetFileContentType(image); + contentType = FileUtility.GetFileContentType(image); var fileName = Path.GetFileNameWithoutExtension(image); var fileType = Path.GetExtension(image).Substring(1); var model = new MessageFileModel() diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs index fa99b13f..f26763c9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileBasicService { public string GetUserAvatar() { @@ -30,11 +30,11 @@ public partial class BotSharpFileService if (Directory.Exists(dir)) { - Directory.Delete(dir, true); + DeleteDirectory(dir); } dir = GetUserAvatarDir(user?.Id, createNewDir: true); - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes); return true; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs similarity index 70% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs index b7c9a946..1d2079b9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs @@ -1,14 +1,13 @@ -using Microsoft.AspNetCore.StaticFiles; using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService : IBotSharpFileService +public partial class FileBasicService : IFileBasicService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _imageTypes = new List { @@ -25,13 +24,10 @@ public partial class BotSharpFileService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - public BotSharpFileService( + public FileBasicService( BotSharpDatabaseSettings dbSettings, IUserIdentity user, - ILogger logger, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; @@ -40,11 +36,4 @@ public partial class BotSharpFileService : IBotSharpFileService _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); - } - #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs deleted file mode 100644 index be5f3180..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Microsoft.AspNetCore.StaticFiles; -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ - public string GetDirectory(string conversationId) - { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - return dir; - } - - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - - public byte[] GetFileBytes(string fileStorageUrl) - { - using var stream = File.OpenRead(fileStorageUrl); - var bytes = new byte[stream.Length]; - stream.Read(bytes, 0, (int)stream.Length); - return bytes; - } - - public bool SavefileToPath(string filePath, Stream stream) - { - using (var fileStream = new FileStream(filePath, FileMode.Create)) - { - stream.CopyTo(fileStream); - } - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs deleted file mode 100644 index daca7711..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs +++ /dev/null @@ -1,143 +0,0 @@ -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) - { - var content = string.Empty; - - if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) - { - return content; - } - - var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); - if (!ExistDirectory(sessionDir)) - { - Directory.CreateDirectory(sessionDir); - } - - try - { - var pdfFiles = await DownloadFiles(sessionDir, files); - var images = await ConvertPdfToImages(pdfFiles); - if (images.IsNullOrEmpty()) return content; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", - model: model, modelId: modelId ?? "gpt-4", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, prompt) - { - Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() - } - }); - - content = message.Content; - return content; - } - catch (Exception ex) - { - _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}"); - return content; - } - finally - { - Directory.Delete(sessionDir, true); - } - } - - #region Private methods - private string GetSessionDirectory(string id) - { - var dir = Path.Combine(_baseDir, SESSION_FOLDER, id); - return dir; - } - - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") - { - if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var locs = new List(); - foreach (var file in files) - { - try - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - if (!bytes.IsNullOrEmpty()) - { - var guid = Guid.NewGuid().ToString(); - var fileDir = Path.Combine(dir, guid); - if (!ExistDirectory(fileDir)) - { - Directory.CreateDirectory(fileDir); - } - - var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}"); - using (var fs = new FileStream(pdfDir, FileMode.Create)) - { - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - locs.Add(pdfDir); - Thread.Sleep(100); - } - } - } - catch (Exception ex) - { - _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return locs; - } - - private async Task> ConvertPdfToImages(IEnumerable files) - { - var images = new List(); - var converter = GetPdf2ImageConverter(); - if (converter == null || files.IsNullOrEmpty()) - { - return images; - } - - foreach (var file in files) - { - try - { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); - var urls = await converter.ConvertPdfToImages(file, folder); - images.AddRange(urls); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return images; - } - #endregion -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs similarity index 84% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 619360d2..c9d35cb7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -2,8 +2,24 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileInstructService { + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + { + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, text) + { + Files = images?.ToList() ?? new List() + } + }); + return message; + } + public async Task GenerateImage(string? provider, string? model, string text) { var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); @@ -31,7 +47,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -53,7 +69,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -82,7 +98,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - + imageStream.Close(); maskStream.Close(); return message; @@ -100,7 +116,7 @@ public partial class BotSharpFileService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } return bytes; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs similarity index 78% rename from src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 1efbad6d..d4413983 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,6 +1,9 @@ -namespace BotSharp.Plugin.TencentCos.Services; +using BotSharp.Abstraction.Files.Converters; +using System.IO; -public partial class TencentCosService +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService { public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { @@ -12,7 +15,9 @@ public partial class TencentCosService } var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); + + var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid); + DeleteIfExistDirectory(sessionDir); try { @@ -32,9 +37,7 @@ public partial class TencentCosService Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } }); - - content = message.Content; - return content; + return message.Content; } catch (Exception ex) { @@ -43,17 +46,11 @@ public partial class TencentCosService } finally { - Directory.Delete(sessionDir, true); + _fileBasic.DeleteDirectory(sessionDir); } } #region Private methods - private string GetSessionDirectory(string id) - { - var dir = $"{SESSION_FOLDER}/{id}"; - return dir; - } - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) @@ -75,18 +72,17 @@ public partial class TencentCosService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } if (!bytes.IsNullOrEmpty()) { var guid = Guid.NewGuid().ToString(); - var fileDir = $"{dir}/{guid}"; + var fileDir = _fileBasic.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir); - var pdfDir = $"{fileDir}/{guid}.{extension}"; - - - _cosClient.BucketClient.UploadBytes(pdfDir, bytes); + var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileBasic.SaveFileBytesToPath(pdfDir, bytes); locs.Add(pdfDir); } } @@ -102,7 +98,7 @@ public partial class TencentCosService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(); if (converter == null || files.IsNullOrEmpty()) { return images; @@ -112,9 +108,8 @@ public partial class TencentCosService { try { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); + var dir = _fileBasic.GetParentDir(file); + var folder = _fileBasic.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs new file mode 100644 index 00000000..dfed9f77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -0,0 +1,114 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task> SelectMessageFiles(string conversationId, + string? agentId = null, string? template = null, string? description = null, + bool includeBotFile = false, bool fromBreakpoint = false, + int? offset = null, IEnumerable? contentTypes = null) + { + if (string.IsNullOrEmpty(conversationId)) + { + return Enumerable.Empty(); + } + + var convService = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint); + var messageIds = GetMessageIds(dialogs, offset); + + var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); + if (includeBotFile) + { + var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + files = files.Concat(botFiles); + } + + if (files.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + return await SelectFiles(agentId, template, description, files, dialogs); + } + + private async Task> SelectFiles(string? agentId, string? template, string? description, + IEnumerable files, List dialogs) + { + if (files.IsNullOrEmpty()) return new List(); + + var llmProviderService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + try + { + var promptFiles = files.Select((x, idx) => + { + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + }).ToList(); + + agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant; + template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt"; + + var foundAgent = db.GetAgent(agentId); + var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); + prompt = render.Render(prompt, new Dictionary + { + { "file_list", promptFiles } + }); + + var agent = new Agent + { + Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = foundAgent?.Name ?? "Utility Assistant", + Instruction = prompt + }; + + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + + var message = dialogs.Last(); + if (!string.IsNullOrWhiteSpace(description)) + { + message = RoleDialogModel.From(message, AgentRole.User, description); + } + + var response = await completion.GetChatCompletions(agent, new List { message }); + var content = response?.Content ?? string.Empty; + var selecteds = JsonSerializer.Deserialize(content); + var fids = selecteds?.Selecteds ?? new List(); + return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when selecting files. {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + + private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + { + if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + + if (offset.HasValue && offset < 1) + { + offset = 1; + } + + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + + return messageIds; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs new file mode 100644 index 00000000..f5d7ede1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService : IFileInstructService +{ + private readonly IFileBasicService _fileBasic; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private const string SESSION_FOLDER = "sessions"; + + public FileInstructService( + IFileBasicService fileBasic, + ILogger logger, + IServiceProvider services) + { + _fileBasic = fileBasic; + _logger = logger; + _services = services; + } + + private void DeleteIfExistDirectory(string? dir) + { + if (_fileBasic.ExistDirectory(dir)) + { + _fileBasic.DeleteDirectory(dir); + } + else + { + _fileBasic.CreateDirectory(dir); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 4a686c6b..e8431cce 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -26,6 +26,10 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler new ParameterPropertyDef("user_goal_agent", "agent who can acheive user initial task, must align with user_goal_description.", required: true), + new ParameterPropertyDef("conversation_end", + "user is ending the conversation.", + type: "boolean", + required: true), new ParameterPropertyDef("is_new_task", "whether the user is requesting a new task that is different from the previous topic.", type: "boolean") diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 88293d97..fa3e94d2 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -29,6 +29,7 @@ global using BotSharp.Abstraction.Translation; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Core.Repository; @@ -37,4 +38,4 @@ global using BotSharp.Core.Agents.Services; global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; -global using Aspects.Cache; +global using Aspects.Cache; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 536ca340..9fc0c389 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -7,5 +7,11 @@ "updatedDateTime": "2024-01-15T14:39:32Z", "iconUrl": "/images/logo.png", "disabled": false, - "isPublic": true + "isPublic": true, + "llmConfig": { + "is_inherit": false, + "provider": "openai", + "model": "gpt-4o-mini", + "max_recursion_depth": 3 + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid similarity index 97% rename from src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid index f4295baa..57f9895c 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid @@ -2,7 +2,7 @@ Please take a look at the files in the [FILES] section from the conversation and ** Ensure the output is only in JSON format without any additional text. ** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id. +** You may need to look at the file_name as a reference to find the correct file id or ids. Here is the JSON format to use: { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 22649d72..d81f069a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Constants; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; @@ -80,7 +81,7 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); @@ -348,7 +349,7 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var dir = fileService.GetDirectory(conversationId); foreach (var file in files) { @@ -356,7 +357,7 @@ public class ConversationController : ControllerBase var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var filePath = Path.Combine(dir, fileName); - fileService.SavefileToPath(filePath, file.OpenReadStream()); + fileService.SaveFileStreamToPath(filePath, file.OpenReadStream()); } return Ok(new { message = "File uploaded successfully." }); @@ -371,7 +372,7 @@ public class ConversationController : ControllerBase var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); var conv = await convService.GetConversationRecordOrCreateNew(agentId); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); return isSaved ? messageId : string.Empty; @@ -380,15 +381,15 @@ public class ConversationController : ControllerBase [HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")] public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { - var fileService = _services.GetRequiredService(); - var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source, imageOnly: false); + var fileService = _services.GetRequiredService(); + var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source); return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName); if (string.IsNullOrEmpty(file)) { @@ -413,7 +414,9 @@ public class ConversationController : ControllerBase using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); - return File(bytes, "application/octet-stream", Path.GetFileName(file)); + var fileExtension = Path.GetExtension(file).ToLower(); + var enableRangeProcessing = FileConstants.AudioExtensions.Contains(fileExtension); + return File(bytes, "application/octet-stream", Path.GetFileName(file), enableRangeProcessing: enableRangeProcessing); } private async Task OnChunkReceived(HttpResponse response, ChatResponseModel message) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 2189af2f..48fdafe1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using NetTopologySuite.IO; namespace BotSharp.OpenAPI.Controllers; @@ -87,18 +86,8 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - model: input.Model ?? "gpt-4o", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, input.Text) - { - Files = input.Files - } - }); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); return message.Content; } catch (Exception ex) @@ -114,14 +103,14 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-generation")] public async Task ImageGeneration([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -140,7 +129,6 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-variation")] public async Task ImageVariation([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -152,7 +140,9 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.VaryImage(input.Provider, input.Model, image); + + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -169,7 +159,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -181,7 +171,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -198,7 +188,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -211,7 +201,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image, mask); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -236,8 +226,8 @@ public class InstructModeController : ControllerBase try { - var fileService = _services.GetRequiredService(); - var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); viewModel.Content = content; return viewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 966f51c9..74db3bd0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -137,14 +137,14 @@ public class UserController : ControllerBase [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] BotSharpFile file) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); return fileService.SaveUserAvatar(file); } [HttpGet("/user/avatar")] public IActionResult GetUserAvatar() { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var file = fileService.GetUserAvatar(); if (string.IsNullOrEmpty(file)) { @@ -158,7 +158,7 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var bytes = fileService.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 16e2841a..72034317 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat; @@ -196,7 +197,6 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -270,13 +270,13 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); using var stream = File.OpenRead(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f5926a53..3aa65e97 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -28,9 +28,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 45dc65be..8169844e 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -74,57 +74,11 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var conversationId = convService.ConversationId; - var dialogs = convService.GetDialogHistory(fromBreakpoint: false); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); - var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot); - return await SelectFiles(userFiles.Concat(botFiles), dialogs); - } - private async Task> SelectFiles(IEnumerable files, List dialogs) - { - if (files.IsNullOrEmpty()) return new List(); - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptFiles = files.Select((x, idx) => - { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; - }).ToList(); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "file_list", promptFiles } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var latest = dialogs.LastOrDefault(); - var response = await completion.GetChatCompletions(agent, new List { latest }); - var content = response?.Content ?? string.Empty; - var selecteds = JsonSerializer.Deserialize(content); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}"); - return new List(); - } + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true); + return selecteds; } private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 78d77ac4..9b097fed 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -47,9 +47,6 @@ PreserveNewest - - PreserveNewest - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 63938c5e..e9d1851b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -28,7 +28,7 @@ public class EditImageFn : IFunctionCallback Init(message); SetImageOptions(); - var image = await SelectConversationImage(descrpition); + var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; return true; @@ -48,64 +48,11 @@ public class EditImageFn : IFunctionCallback state.SetState("image_count", "1"); } - private async Task SelectConversationImage(string? description) + private async Task SelectImage(string? description) { - var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); - return await SelectImage(userImages, dialogs.LastOrDefault(), description); - } - - private async Task SelectImage(IEnumerable images, RoleDialogModel message, string? description) - { - if (images.IsNullOrEmpty()) return null; - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) => - { - return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}"; - }).ToList(); - - if (promptImages.IsNullOrEmpty()) return null; - - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "image_list", promptImages } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - - var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; - var dialog = RoleDialogModel.From(message, AgentRole.User, text); - - var response = await completion.GetChatCompletions(agent, new List { dialog }); - var content = response?.Content ?? string.Empty; - var selected = JsonSerializer.Deserialize(content); - var fid = selected?.Selected ?? -1; - return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}"); - return null; - } + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, description: description, contentTypes: new List { MediaTypeNames.Image.Png }); + return selecteds?.FirstOrDefault(); } private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) @@ -154,7 +101,7 @@ public class EditImageFn : IFunctionCallback } }; - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4d53f880..3869a419 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" }).ToList(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 66f69353..cdff6cf4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -51,7 +51,7 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); foreach (var dialog in dialogs) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 85c2afbc..d3c21737 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -50,7 +50,7 @@ public class ReadPdfFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); foreach (var dialog in dialogs) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid deleted file mode 100644 index 9e67faad..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid +++ /dev/null @@ -1,41 +0,0 @@ -Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. - -** Ensure the output is only in JSON format without any additional text. -** You may need to look at the image_name as a reference to find the correct image id. - -Here is the JSON format to use: -{ - "selected_id": the id selected from the [IMAGES] section -} - - -Suppose there are four images: - -id: 1, image_name: example_image_a.png -id: 2, image_name: example_image_b.png -id: 3, image_name: example_image_c.png -id: 4, image_name: example_image_d.png - -===== -Example 1: -USER: I want to add a dog in the first file. -OUTPUT: { "selected_id": 1 } - -Example 2: -USER: Add a coffee cup in the second image I uploaded. -OUTPUT: { "selected_id": 2 } - -Example 3: -USER: Please remove the left tree in the third and the first images. -OUTPUT: { "selected_id": 3 } - -Example 4: -USER: Circle the head of the dog in example_image_b.png. -OUTPUT: { "selected_id": 4 } -===== - - -[IMAGES] -{% for image in image_list -%} -{{ image }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 23084ead..f12e8b34 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -197,7 +198,6 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -271,13 +271,13 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); using var stream = File.OpenRead(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e9631786..9de15321 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,4 +1,4 @@ -using Microsoft.AspNetCore.StaticFiles; +using System.IO; namespace BotSharp.Plugin.TencentCos.Services; @@ -9,41 +9,11 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - public byte[] GetFileBytes(string fileStorageUrl) { try { var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); - return fileData; } catch (Exception ex) @@ -53,7 +23,7 @@ public partial class TencentCosService return Array.Empty(); } - public bool SavefileToPath(string filePath, Stream stream) + public bool SaveFileStreamToPath(string filePath, Stream stream) { if (string.IsNullOrEmpty(filePath)) return false; @@ -63,8 +33,49 @@ public partial class TencentCosService } catch (Exception ex) { - _logger.LogWarning($"Error when saving file to path: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when saving file stream to path: {ex.Message}\r\n{ex.InnerException}"); return false; } } + + public bool SaveFileBytesToPath(string filePath, byte[] bytes) + { + if (string.IsNullOrEmpty(filePath)) return false; + + try + { + return _cosClient.BucketClient.UploadBytes(filePath, bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving file bytes to path: {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public string GetParentDir(string dir, int level = 1) + { + var segs = dir.Split("/"); + return string.Join("/", segs.SkipLast(level)); + } + + public string BuildDirectory(params string[] segments) + { + return string.Join("/", segments); + } + + public void CreateDirectory(string dir) + { + + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); + } + + public void DeleteDirectory(string dir) + { + _cosClient.BucketClient.DeleteDir(dir); + } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index df39e0a9..d67d69f0 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Utilities; using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; @@ -7,16 +8,16 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable dialogs, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(conversations, offset); + var messageIds = GetMessageIds(dialogs, offset); var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; foreach (var messageId in messageIds) @@ -28,8 +29,11 @@ public partial class TencentCosService var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + { + continue; + } var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); if (foundFiles.IsNullOrEmpty()) continue; @@ -42,7 +46,7 @@ public partial class TencentCosService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -59,8 +63,8 @@ public partial class TencentCosService { foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } @@ -135,7 +139,7 @@ public partial class TencentCosService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = $"{dir}/{source}/{i + 1}"; @@ -225,13 +229,9 @@ public partial class TencentCosService { if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset <= 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); @@ -264,7 +264,7 @@ public partial class TencentCosService { foreach (var screenShot in fileList) { - contentType = GetFileContentType(screenShot); + contentType = FileUtility.GetFileContentType(screenShot); if (!_imageTypes.Contains(contentType)) continue; var fileName = Path.GetFileNameWithoutExtension(screenShot); @@ -286,7 +286,7 @@ public partial class TencentCosService var images = await ConvertPdfToImages(file, screenShotDir); foreach (var image in images) { - contentType = GetFileContentType(image); + contentType = FileUtility.GetFileContentType(image); var fileName = Path.GetFileNameWithoutExtension(image); var fileType = Path.GetExtension(image).Substring(1); var model = new MessageFileModel() diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs deleted file mode 100644 index e8628ce3..00000000 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace BotSharp.Plugin.TencentCos.Services; - -public partial class TencentCosService -{ - public async Task GenerateImage(string? provider, string? model, string text) - { - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); - var message = await completion.GetImageGeneration(new Agent() - { - Id = Guid.Empty.ToString(), - }, new RoleDialogModel(AgentRole.User, text)); - return message; - } - - public async Task VaryImage(string? provider, string? model, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageVariation(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) - { - if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || - (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) - { - throw new ArgumentException($"Cannot find image/mask url or data"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var imageBytes = await DownloadFile(image); - var maskBytes = await DownloadFile(mask); - - using var imageStream = new MemoryStream(); - imageStream.Write(imageBytes, 0, imageBytes.Length); - imageStream.Position = 0; - - using var maskStream = new MemoryStream(); - maskStream.Write(maskBytes, 0, maskBytes.Length); - maskStream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - - imageStream.Close(); - maskStream.Close(); - return message; - } - - #region Private methods - private async Task DownloadFile(BotSharpFile file) - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - return bytes; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 23ce68c0..55e26d81 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Files.Utilities; + namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -26,10 +28,8 @@ public partial class TencentCosService if (string.IsNullOrEmpty(dir)) return false; - var (_, bytes) = GetFileInfoFromData(file.FileData); - + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var extension = Path.GetExtension(file.FileName); - var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 80b8fd78..78c6bcc9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -5,8 +5,9 @@ using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; -public partial class TencentCosService : IBotSharpFileService +public partial class TencentCosService : IFileBasicService { + private readonly TencentCosClient _cosClient; private readonly TencentCosSettings _settings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; @@ -27,10 +28,6 @@ public partial class TencentCosService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - private readonly TencentCosClient _cosClient; public TencentCosService( TencentCosSettings settings, @@ -46,11 +43,4 @@ public partial class TencentCosService : IBotSharpFileService _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; _cosClient = cosClient; } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); - } - #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 93a99c14..25cdb277 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -31,7 +31,7 @@ public class TencentCosPlugin : IBotSharpPlugin services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } }