diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index 7f861c1f..b0e0209c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -3,9 +3,10 @@ namespace BotSharp.Abstraction.Files; public interface IBotSharpFileService { string GetDirectory(string conversationId); - IEnumerable GetChatImages(string conversationId, List conversations, int offset = 2); + IEnumerable GetChatImages(string conversationId, List conversations, int? offset = null); IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, bool imageOnly = false); string GetMessageFile(string conversationId, string messageId, string fileName); + bool HasConversationFiles(string conversationId); Task SaveMessageFiles(string conversationId, string messageId, List files); string GetUserAvatar(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index de226f58..7e556e67 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -14,4 +14,10 @@ public class BotSharpFile [JsonPropertyName("file_url")] public string FileUrl { get; set; } = string.Empty; + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } = string.Empty; + + [JsonPropertyName("file_storage_url")] + public string FileStorageUrl { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 90eb3298..f06e2ba2 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Google.Settings; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Messaging; @@ -6,7 +5,6 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Settings; using BotSharp.Abstraction.Templating; -using BotSharp.Core.Files; using BotSharp.Core.Instructs; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; @@ -44,7 +42,6 @@ public class ConversationPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); // Rich content messaging diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index 50c7f487..330eeae5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -1,14 +1,14 @@ using BotSharp.Abstraction.Browsing; -using Microsoft.AspNetCore.StaticFiles; using Microsoft.EntityFrameworkCore; using System.IO; +using System.Linq; using System.Threading; namespace BotSharp.Core.Files; public partial class BotSharpFileService { - public IEnumerable GetChatImages(string conversationId, List conversations, int offset = 1) + public IEnumerable GetChatImages(string conversationId, List conversations, int? offset = null) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) @@ -25,7 +25,16 @@ public partial class BotSharpFileService offset = MAX_OFFSET; } - var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList(); + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList(); return files; } @@ -83,6 +92,16 @@ public partial class BotSharpFileService return found; } + public bool HasConversationFiles(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); + if (!ExistDirectory(dir)) return false; + + return Directory.GetDirectories(dir).Count() > 0; + } + public async Task SaveMessageFiles(string conversationId, string messageId, List files) { if (files.IsNullOrEmpty()) return false; @@ -122,7 +141,7 @@ public partial class BotSharpFileService { var path = Path.Combine(preFixPath, fileName); await web.GoToPage(contextId, path); - path = Path.Combine(preFixPath, $"{Guid.NewGuid()}.png"); + path = Path.Combine(preFixPath, $"{Guid.NewGuid()}.{i + 1}.png"); await web.ScreenshotAsync(contextId, path); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs new file mode 100644 index 00000000..d62e331b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -0,0 +1,21 @@ +using BotSharp.Core.Files.Hooks; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Files; + +public class FilePlugin : IBotSharpPlugin +{ + public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; + + public string Name => "File"; + + public string Description => "Provides file processing funcationality."; + + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + + services.AddScoped(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs new file mode 100644 index 00000000..4bf7a0f0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs @@ -0,0 +1,89 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Files.Functions; + +public class LoadAttachmentFn : IFunctionCallback +{ + public string Name => "load_attachment"; + public string Indication => "Analyzing files"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a"; + + public LoadAttachmentFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var conv = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + + var wholeDialogs = conv.GetDialogHistory(); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); + var agent = await agentService.LoadAgent(AIAssistant); + var fileAgent = new Agent + { + Id = agent.Id, + Name = agent.Name, + Instruction = "Please describe the images.", + TemplateDict = new Dictionary() + }; + + var response = await GetChatCompletion(fileAgent, dialogs); + message.Content = response; + message.StopCompletion = true; + return true; + } + + private List AssembleFiles(string conversationId, List dialogs) + { + if (dialogs.IsNullOrEmpty()) + { + return new List(); + } + + var fileService = _services.GetRequiredService(); + var files = fileService.GetChatImages(conversationId, dialogs); + + foreach (var dialog in dialogs) + { + var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); + if (found.IsNullOrEmpty()) continue; + + dialog.Files = found.Select(x => new BotSharpFile + { + FileName = x.FileName, + ContentType = x.ContentType, + FileStorageUrl = x.FileStorageUrl + }).ToList(); + } + + return dialogs; + } + + private async Task GetChatCompletion(Agent agent, List dialogs) + { + try + { + var llmProviderService = _services.GetRequiredService(); + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var response = await completion.GetChatCompletions(agent, dialogs); + return response.Content; + } + catch (Exception ex) + { + var error = $"Error when analyzing files."; + _logger.LogWarning($"{error} {ex.Message}"); + return error; + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs new file mode 100644 index 00000000..9cbd0d3f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -0,0 +1,50 @@ +namespace BotSharp.Core.Files.Hooks; + +public class AttachmentProcessingHook : AgentHookBase +{ + private readonly IServiceProvider _services; + private readonly AgentSettings _agentSettings; + + public override string SelfId => string.Empty; + + public AttachmentProcessingHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + _services = services; + _agentSettings = settings; + } + + public override bool OnFunctionsLoaded(List functions) + { + var fileService = _services.GetRequiredService(); + var conv = _services.GetRequiredService(); + var hasConvFiles = fileService.HasConversationFiles(conv.ConversationId); + + if (hasConvFiles) + { + var json = JsonSerializer.Serialize(new + { + user_question = new + { + type = "string", + description = $"The question asked by user, which is related to analyzing images or other files." + } + }); + + functions.Add(new FunctionDef + { + Name = "load_attachment", + Description = $"If the user's request is related to analyzing files, you can call this function to analyze files.", + Parameters = + { + Properties = JsonSerializer.Deserialize(json), + Required = new List + { + "user_question" + } + } + }); + } + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 990a74ab..2fcbf4c7 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -229,12 +229,6 @@ public class ChatCompletionProvider : IChatCompletion var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; - var chatFiles = new List(); - if (allowMultiModal) - { - chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList(); - } - var chatCompletionsOptions = new ChatCompletionsOptions(); if (!string.IsNullOrEmpty(agent.Instruction)) @@ -304,16 +298,6 @@ public class ChatCompletionProvider : IChatCompletion new ChatMessageTextContentItem(text) }; - var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList(); - if (!files.IsNullOrEmpty()) - { - foreach (var file in files) - { - using var stream = File.OpenRead(file.FileStorageUrl); - chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low)); - } - } - if (!message.Files.IsNullOrEmpty()) { foreach (var file in message.Files) @@ -329,6 +313,11 @@ public class ChatCompletionProvider : IChatCompletion using var stream = new MemoryStream(bytes, 0, bytes.Length); chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low)); } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + using var stream = File.OpenRead(file.FileStorageUrl); + chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low)); + } } }