From 83a78d2b5af71219fc59a5ded3e541ae80c341d7 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 11:51:39 -0500 Subject: [PATCH] add instruct multi modal --- .../Files/IBotSharpFileService.cs | 7 +++ .../Files/Models/BotSharpFile.cs | 11 ++-- .../Files/BotSharpFileService.cs | 59 ++++++++++++------- .../Infrastructures/CompletionProvider.cs | 2 +- .../Controllers/InstructModeController.cs | 35 ++++++++++- .../Providers/ChatCompletionProvider.cs | 22 ++++++- 6 files changed, 104 insertions(+), 32 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index edc04b7b..272abf0d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -18,4 +18,11 @@ public interface IBotSharpFileService /// bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null); bool DeleteConversationFiles(IEnumerable conversationIds); + + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + (string, byte[]) GetFileInfoFromData(string data); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index f679d52e..9581b83f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -4,14 +4,11 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile { [JsonPropertyName("file_name")] - public string FileName { get; set; } + public string FileName { get; set; } = string.Empty; [JsonPropertyName("file_data")] - public string FileData { get; set; } + public string FileData { get; set; } = string.Empty; - [JsonPropertyName("content_type")] - public string ContentType { get; set; } - - [JsonPropertyName("file_size")] - public int FileSize { get; set; } + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index 581569a1..d7e961be 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -8,6 +8,7 @@ public class BotSharpFileService : IBotSharpFileService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _allowedTypes = new List { "image/png", "image/jpeg" }; @@ -18,9 +19,11 @@ public class BotSharpFileService : IBotSharpFileService public BotSharpFileService( BotSharpDatabaseSettings dbSettings, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; + _logger = logger; _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } @@ -117,19 +120,26 @@ public class BotSharpFileService : IBotSharpFileService var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true); if (string.IsNullOrEmpty(dir)) return; - for (int i = 0; i < files.Count; i++) + try { - var file = files[i]; - if (string.IsNullOrEmpty(file.FileData)) + for (int i = 0; i < files.Count; i++) { - continue; - } + var file = files[i]; + if (string.IsNullOrEmpty(file.FileData)) + { + continue; + } - var bytes = GetFileBytes(file.FileData); - var fileType = Path.GetExtension(file.FileName); - var fileName = $"{i + 1}{fileType}"; - Thread.Sleep(100); - File.WriteAllBytes(Path.Combine(dir, fileName), bytes); + var (_, bytes) = GetFileInfoFromData(file.FileData); + var fileType = Path.GetExtension(file.FileName); + var fileName = $"{i + 1}{fileType}"; + Thread.Sleep(100); + File.WriteAllBytes(Path.Combine(dir, fileName), bytes); + } + } + catch (Exception ex) + { + _logger.LogError($"Error when saving conversation files: {ex.Message}"); } } @@ -179,6 +189,23 @@ public class BotSharpFileService : IBotSharpFileService return true; } + public (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + #region Private methods private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false) { @@ -212,18 +239,6 @@ public class BotSharpFileService : IBotSharpFileService return dir; } - private byte[] GetFileBytes(string data) - { - if (string.IsNullOrEmpty(data)) - { - return new byte[0]; - } - - var startIdx = data.IndexOf(','); - var base64Str = data.Substring(startIdx + 1); - return Convert.FromBase64String(base64Str); - } - private string GetFileContentType(string filePath) { string contentType; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index c55ed9a8..bc0266ed 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -47,7 +47,7 @@ public class CompletionProvider logger.LogError($"Can't resolve completion provider by {provider}"); } - completer.SetModelName(model); + completer?.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 985526d9..9f00a3e0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -11,10 +11,12 @@ namespace BotSharp.OpenAPI.Controllers; public class InstructModeController : ControllerBase { private readonly IServiceProvider _services; + private readonly ILogger _logger; - public InstructModeController(IServiceProvider services) + public InstructModeController(IServiceProvider services, ILogger logger) { _services = services; + _logger = logger; } [HttpPost("/instruct/{agentId}")] @@ -72,4 +74,35 @@ public class InstructModeController : ControllerBase }); return message.Content; } + + [HttpPost("/instruct/multi-modal")] + public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + state.SetState("provider", input.Provider, source: StateSource.External) + .SetState("model", input.Model, source: StateSource.External) + .SetState("model_id", input.ModelId, source: StateSource.External); + + try + { + var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo"); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, input.Text) + { + Files = input.Files + } + }); + return message.Content; + } + catch (Exception ex) + { + _logger.LogError($"Error in analyzing files. {ex.Message}"); + return $"Error in analyzing files. {ex.Message}"; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index b2bbe71b..d884e386 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices.ComTypes; using System.Threading.Tasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; @@ -226,9 +227,10 @@ public class ChatCompletionProvider : IChatCompletion var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); + var allowMultiModal = settings != null && settings.MultiModal; var chatFiles = new List(); - if (settings != null && settings.MultiModal) + if (allowMultiModal) { chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList(); } @@ -308,6 +310,24 @@ public class ChatCompletionProvider : IChatCompletion } } + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + foreach (var file in message.Files) + { + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + using var stream = new MemoryStream(bytes, 0, bytes.Length); + chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low)); + } + } + } + //if (!string.IsNullOrEmpty(message.ImageUrl)) //{ // var uri = new Uri(message.ImageUrl);