From f9e63097a9ccb91449039eaa6ad7ca7b63db077b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 13 May 2024 16:53:41 -0500 Subject: [PATCH 1/7] add chat files --- .../Files/IBotSharpFileService.cs | 3 +- .../Files/Models/MessageFileModel.cs | 32 +++++ .../Files/Models/OutputFileModel.cs | 13 -- .../MLTasks/Settings/LlmModelSetting.cs | 5 + .../BotSharp.Core/BotSharp.Core.csproj | 1 + .../ConversationService.SendMessage.cs | 7 ++ .../Files/BotSharpFileService.cs | 114 ++++++++++-------- .../Routing/RoutingService.InvokeAgent.cs | 8 +- .../Controllers/FileController.cs | 5 +- src/Infrastructure/BotSharp.OpenAPI/Using.cs | 3 +- .../ViewModels/Files/MessageFileViewModel.cs | 34 ++++++ .../Providers/ChatCompletionProvider.cs | 47 ++++++-- 12 files changed, 193 insertions(+), 79 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index 9c27d7ff..edc04b7b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -3,7 +3,8 @@ namespace BotSharp.Abstraction.Files; public interface IBotSharpFileService { string GetDirectory(string conversationId); - IEnumerable GetConversationFiles(string conversationId, string messageId); + IEnumerable GetChatImages(string conversationId, List conversations, int offset = 2); + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, bool imageOnly = false); string? GetMessageFile(string conversationId, string messageId, string fileName); void SaveMessageFiles(string conversationId, string messageId, List files); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs new file mode 100644 index 00000000..3ec63fd8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class MessageFileModel +{ + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } + + [JsonPropertyName("file_storage_url")] + public string FileStorageUrl { get; set; } + + [JsonPropertyName("file_name")] + public string FileName { get; set; } + + [JsonPropertyName("file_type")] + public string FileType { get; set; } + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } + + public MessageFileModel() + { + + } + + public override string ToString() + { + return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}"; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs deleted file mode 100644 index 962b01e1..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class OutputFileModel -{ - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } - - [JsonPropertyName("file_name")] - public string FileName { get; set; } - - [JsonPropertyName("file_type")] - public string FileType { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 1faf3c52..b86578fe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -27,6 +27,11 @@ public class LlmModelSetting public string Endpoint { get; set; } public LlmModelType Type { get; set; } = LlmModelType.Chat; + /// + /// If true, allow sending images/vidoes to this model + /// + public bool MultiModal { get; set; } + /// /// Prompt cost per 1K token /// diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 8d05c40b..0ddc52e8 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -159,6 +159,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index d6940fe1..4ca64922 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -151,6 +151,13 @@ public partial class ConversationService Message = new TextMessage(response.SecondaryContent ?? response.Content) }; + response.RichContent = new RichContent + { + Recipient = new Recipient { Id = state.GetConversationId() }, + Editor = "file", + Message = new TextMessage(response.SecondaryContent ?? response.Content) + }; + // Patch return function name if (response.PostbackFunctionName != null) { diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index ad687274..581569a1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.StaticFiles; using System.IO; using System.Threading; @@ -8,9 +9,12 @@ public class BotSharpFileService : IBotSharpFileService private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly string _baseDir; + private readonly IEnumerable _allowedTypes = new List { "image/png", "image/jpeg" }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; + private const int MIN_OFFSET = 1; + private const int MAX_OFFSET = 5; public BotSharpFileService( BotSharpDatabaseSettings dbSettings, @@ -31,29 +35,67 @@ public class BotSharpFileService : IBotSharpFileService return dir; } - public IEnumerable GetConversationFiles(string conversationId, string messageId) + public IEnumerable GetChatImages(string conversationId, List conversations, int offset = 2) { - var outputFiles = new List(); - var dir = GetConversationFileDirectory(conversationId, messageId); - if (string.IsNullOrEmpty(dir)) + var files = new List(); + if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) { - return outputFiles; + return files; } - foreach (var file in Directory.GetFiles(dir)) + if (offset <= 0) { - var fileName = Path.GetFileNameWithoutExtension(file); - var extension = Path.GetExtension(file); - var fileType = extension.Substring(1); - var model = new OutputFileModel() - { - FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}", - FileName = fileName, - FileType = fileType - }; - outputFiles.Add(model); + offset = MIN_OFFSET; } - return outputFiles; + else if (offset > MAX_OFFSET) + { + offset = MAX_OFFSET; + } + + var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList(); + files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList(); + return files; + } + + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, bool imageOnly = false) + { + var files = new List(); + if (messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var dir = GetConversationFileDirectory(conversationId, messageId); + if (string.IsNullOrEmpty(dir)) + { + continue; + } + + foreach (var file in Directory.GetFiles(dir)) + { + var contentType = GetFileContentType(file); + if (imageOnly && !_allowedTypes.Contains(contentType)) + { + continue; + } + + var fileName = Path.GetFileNameWithoutExtension(file); + var extension = Path.GetExtension(file); + var fileType = extension.Substring(1); + + var model = new MessageFileModel() + { + MessageId = messageId, + FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}", + FileStorageUrl = file, + FileName = fileName, + FileType = fileType, + ContentType = contentType + }; + files.Add(model); + } + } + + return files; } public string? GetMessageFile(string conversationId, string messageId, string fileName) @@ -182,42 +224,16 @@ public class BotSharpFileService : IBotSharpFileService return Convert.FromBase64String(base64Str); } - private string GetFileType(string data) + private string GetFileContentType(string filePath) { - if (string.IsNullOrEmpty(data)) + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) { - return string.Empty; + contentType = string.Empty; } - var startIdx = data.IndexOf(':'); - var endIdx = data.IndexOf(';'); - var fileType = data.Substring(startIdx + 1, endIdx - startIdx - 1); - return fileType; - } - - private string ParseFileFormat(string type) - { - var parsed = string.Empty; - switch (type) - { - case "image/png": - parsed = ".png"; - break; - case "image/jpeg": - case "image/jpg": - parsed = ".jpeg"; - break; - case "application/pdf": - parsed = ".pdf"; - break; - case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": - parsed = ".xlsx"; - break; - case "text/plain": - parsed = ".txt"; - break; - } - return parsed; + return contentType; } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 6bdac0ef..6046c5bb 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -17,18 +17,18 @@ public partial class RoutingService return false; } - var provide = agent.LlmConfig.Provider; + var provider = agent.LlmConfig.Provider; var model = agent.LlmConfig.Model; - if (provide == null || model == null) + if (provider == null || model == null) { var agentSettings = _services.GetRequiredService(); - provide = agentSettings.LlmConfig.Provider; + provider = agentSettings.LlmConfig.Provider; model = agentSettings.LlmConfig.Model; } var chatCompletion = CompletionProvider.GetChatCompletion(_services, - provider: provide, + provider: provider, model: model); var message = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs index a24357a2..cfae602c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs @@ -38,10 +38,11 @@ public class FileController : ControllerBase } [HttpGet("/conversation/{conversationId}/files/{messageId}")] - public IEnumerable GetConversationFiles([FromRoute] string conversationId, [FromRoute] string messageId) + public IEnumerable GetMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId) { var fileService = _services.GetRequiredService(); - return fileService.GetConversationFiles(conversationId, messageId); + var files = fileService.GetMessageFiles(conversationId, new List { messageId }); + return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/Using.cs b/src/Infrastructure/BotSharp.OpenAPI/Using.cs index f3ba775f..8771b81c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Using.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Using.cs @@ -28,4 +28,5 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files; global using BotSharp.OpenAPI.ViewModels.Conversations; global using BotSharp.OpenAPI.ViewModels.Users; -global using BotSharp.OpenAPI.ViewModels.Agents; \ No newline at end of file +global using BotSharp.OpenAPI.ViewModels.Agents; +global using BotSharp.OpenAPI.ViewModels.Files; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs new file mode 100644 index 00000000..a9eb33bd --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -0,0 +1,34 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Files; + +public class MessageFileViewModel +{ + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } + + [JsonPropertyName("file_name")] + public string FileName { get; set; } + + [JsonPropertyName("file_type")] + public string FileType { get; set; } + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } + + public MessageFileViewModel() + { + + } + + public static MessageFileViewModel Transform(MessageFileModel model) + { + return new MessageFileViewModel + { + FileUrl = model.FileUrl, + FileName = model.FileName, + FileType = model.FileType, + ContentType = model.ContentType + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 07a3ff8f..b2bbe71b 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -4,13 +4,17 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.AzureOpenAI.Settings; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -218,6 +222,16 @@ public class ChatCompletionProvider : IChatCompletion protected (string, ChatCompletionsOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + var chatFiles = new List(); + if (settings != null && settings.MultiModal) + { + chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList(); + } var chatCompletionsOptions = new ChatCompletionsOptions(); @@ -279,19 +293,34 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == ChatRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - var userMessage = new ChatRequestUserMessage(text) + var chatItems = new List() + { + new ChatMessageTextContentItem(text) + }; + + var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList(); + if (!files.IsNullOrEmpty()) + { + foreach (var file in files) + { + using var stream = File.OpenRead(file.FileStorageUrl); + chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low)); + } + } + + //if (!string.IsNullOrEmpty(message.ImageUrl)) + //{ + // var uri = new Uri(message.ImageUrl); + // userMessage.MultimodalContentItems.Add( + // new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + //} + + var userMessage = new ChatRequestUserMessage(chatItems) { // To display Planner name in log Name = message.FunctionName, }; - if (!string.IsNullOrEmpty(message.ImageUrl)) - { - var uri = new Uri(message.ImageUrl); - userMessage.MultimodalContentItems.Add( - new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); - } - chatCompletionsOptions.Messages.Add(userMessage); } else if (message.Role == ChatRole.Assistant) @@ -301,7 +330,7 @@ public class ChatCompletionProvider : IChatCompletion } // https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683 - var state = _services.GetRequiredService(); + //var state = _services.GetRequiredService(); var temperature = float.Parse(state.GetState("temperature", "0.0")); var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0")); chatCompletionsOptions.Temperature = temperature; From f3bbb0259f281269db4fd58deaae205449e3dd50 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 10:14:08 -0500 Subject: [PATCH 2/7] remove test code --- .../Services/ConversationService.SendMessage.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 4ca64922..d6940fe1 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -151,13 +151,6 @@ public partial class ConversationService Message = new TextMessage(response.SecondaryContent ?? response.Content) }; - response.RichContent = new RichContent - { - Recipient = new Recipient { Id = state.GetConversationId() }, - Editor = "file", - Message = new TextMessage(response.SecondaryContent ?? response.Content) - }; - // Patch return function name if (response.PostbackFunctionName != null) { From 83a78d2b5af71219fc59a5ded3e541ae80c341d7 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 11:51:39 -0500 Subject: [PATCH 3/7] add instruct multi modal --- .../Files/IBotSharpFileService.cs | 7 +++ .../Files/Models/BotSharpFile.cs | 11 ++-- .../Files/BotSharpFileService.cs | 59 ++++++++++++------- .../Infrastructures/CompletionProvider.cs | 2 +- .../Controllers/InstructModeController.cs | 35 ++++++++++- .../Providers/ChatCompletionProvider.cs | 22 ++++++- 6 files changed, 104 insertions(+), 32 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index edc04b7b..272abf0d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -18,4 +18,11 @@ public interface IBotSharpFileService /// bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null); bool DeleteConversationFiles(IEnumerable conversationIds); + + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + (string, byte[]) GetFileInfoFromData(string data); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index f679d52e..9581b83f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -4,14 +4,11 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile { [JsonPropertyName("file_name")] - public string FileName { get; set; } + public string FileName { get; set; } = string.Empty; [JsonPropertyName("file_data")] - public string FileData { get; set; } + public string FileData { get; set; } = string.Empty; - [JsonPropertyName("content_type")] - public string ContentType { get; set; } - - [JsonPropertyName("file_size")] - public int FileSize { get; set; } + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index 581569a1..d7e961be 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -8,6 +8,7 @@ public class BotSharpFileService : IBotSharpFileService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _allowedTypes = new List { "image/png", "image/jpeg" }; @@ -18,9 +19,11 @@ public class BotSharpFileService : IBotSharpFileService public BotSharpFileService( BotSharpDatabaseSettings dbSettings, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; + _logger = logger; _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } @@ -117,19 +120,26 @@ public class BotSharpFileService : IBotSharpFileService var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true); if (string.IsNullOrEmpty(dir)) return; - for (int i = 0; i < files.Count; i++) + try { - var file = files[i]; - if (string.IsNullOrEmpty(file.FileData)) + for (int i = 0; i < files.Count; i++) { - continue; - } + var file = files[i]; + if (string.IsNullOrEmpty(file.FileData)) + { + continue; + } - var bytes = GetFileBytes(file.FileData); - var fileType = Path.GetExtension(file.FileName); - var fileName = $"{i + 1}{fileType}"; - Thread.Sleep(100); - File.WriteAllBytes(Path.Combine(dir, fileName), bytes); + var (_, bytes) = GetFileInfoFromData(file.FileData); + var fileType = Path.GetExtension(file.FileName); + var fileName = $"{i + 1}{fileType}"; + Thread.Sleep(100); + File.WriteAllBytes(Path.Combine(dir, fileName), bytes); + } + } + catch (Exception ex) + { + _logger.LogError($"Error when saving conversation files: {ex.Message}"); } } @@ -179,6 +189,23 @@ public class BotSharpFileService : IBotSharpFileService return true; } + public (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + #region Private methods private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false) { @@ -212,18 +239,6 @@ public class BotSharpFileService : IBotSharpFileService return dir; } - private byte[] GetFileBytes(string data) - { - if (string.IsNullOrEmpty(data)) - { - return new byte[0]; - } - - var startIdx = data.IndexOf(','); - var base64Str = data.Substring(startIdx + 1); - return Convert.FromBase64String(base64Str); - } - private string GetFileContentType(string filePath) { string contentType; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index c55ed9a8..bc0266ed 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -47,7 +47,7 @@ public class CompletionProvider logger.LogError($"Can't resolve completion provider by {provider}"); } - completer.SetModelName(model); + completer?.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 985526d9..9f00a3e0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -11,10 +11,12 @@ namespace BotSharp.OpenAPI.Controllers; public class InstructModeController : ControllerBase { private readonly IServiceProvider _services; + private readonly ILogger _logger; - public InstructModeController(IServiceProvider services) + public InstructModeController(IServiceProvider services, ILogger logger) { _services = services; + _logger = logger; } [HttpPost("/instruct/{agentId}")] @@ -72,4 +74,35 @@ public class InstructModeController : ControllerBase }); return message.Content; } + + [HttpPost("/instruct/multi-modal")] + public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + state.SetState("provider", input.Provider, source: StateSource.External) + .SetState("model", input.Model, source: StateSource.External) + .SetState("model_id", input.ModelId, source: StateSource.External); + + try + { + var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo"); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, input.Text) + { + Files = input.Files + } + }); + return message.Content; + } + catch (Exception ex) + { + _logger.LogError($"Error in analyzing files. {ex.Message}"); + return $"Error in analyzing files. {ex.Message}"; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index b2bbe71b..d884e386 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices.ComTypes; using System.Threading.Tasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; @@ -226,9 +227,10 @@ public class ChatCompletionProvider : IChatCompletion var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); + var allowMultiModal = settings != null && settings.MultiModal; var chatFiles = new List(); - if (settings != null && settings.MultiModal) + if (allowMultiModal) { chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList(); } @@ -308,6 +310,24 @@ public class ChatCompletionProvider : IChatCompletion } } + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + foreach (var file in message.Files) + { + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + using var stream = new MemoryStream(bytes, 0, bytes.Length); + chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low)); + } + } + } + //if (!string.IsNullOrEmpty(message.ImageUrl)) //{ // var uri = new Uri(message.ImageUrl); From 7e908d33868a8a972ed18663929312bf747584c9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 11:55:11 -0500 Subject: [PATCH 4/7] add comment --- .../BotSharp.Abstraction/Files/Models/BotSharpFile.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 9581b83f..de226f58 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -6,6 +6,9 @@ public class BotSharpFile [JsonPropertyName("file_name")] public string FileName { get; set; } = string.Empty; + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// [JsonPropertyName("file_data")] public string FileData { get; set; } = string.Empty; From 3d3359cb951c40a7b04584e764fe3a837cfd3e74 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 13:34:10 -0500 Subject: [PATCH 5/7] filter by model id and multi-modal --- .../MLTasks/ILlmProviderService.cs | 2 +- .../Infrastructures/CompletionProvider.cs | 13 +++++++++---- .../Infrastructures/LlmProviderService.cs | 4 ++-- .../Controllers/InstructModeController.cs | 5 +---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 75fd60e6..120304d2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id); + LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index bc0266ed..ace1e664 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -35,10 +35,13 @@ public class CompletionProvider public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null, string? model = null, + string? modelId = null, + bool multiModal = false, AgentLlmConfig? agentConfig = null) { var completions = services.GetServices(); - (provider, model) = GetProviderAndModel(services, provider: provider, model: model, agentConfig: agentConfig); + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, + multiModal: multiModal, agentConfig: agentConfig); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) @@ -55,6 +58,8 @@ public class CompletionProvider private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, + string? modelId = null, + bool multiModal = false, AgentLlmConfig? agentConfig = null) { var agentSetting = services.GetRequiredService(); @@ -73,11 +78,11 @@ public class CompletionProvider { model = state.GetState("model", model ?? "gpt-35-turbo-4k"); } - else if (state.ContainsState("model_id")) + else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) { - var modelId = state.GetState("model_id"); + var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; var llmProviderService = services.GetRequiredService(); - model = llmProviderService.GetProviderModel(provider, modelId)?.Name; + model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal)?.Name; } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index eb92ac51..7d92a687 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,10 +44,10 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id) + public LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false) { var models = GetProviderModels(provider) - .Where(x => x.Id == id) + .Where(x => x.Id == id && x.MultiModal == multiModal) .ToList(); var random = new Random(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 9f00a3e0..96fe3728 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -80,13 +80,10 @@ public class InstructModeController : ControllerBase { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - state.SetState("provider", input.Provider, source: StateSource.External) - .SetState("model", input.Model, source: StateSource.External) - .SetState("model_id", input.ModelId, source: StateSource.External); try { - var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo"); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4-turbo", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), From 971018ba503df6acd85bd405506a81490c9fb5bf Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 14:30:27 -0500 Subject: [PATCH 6/7] remove error message --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 96fe3728..157cc33a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -83,7 +83,7 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4-turbo", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-multi-modal", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), @@ -99,7 +99,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { _logger.LogError($"Error in analyzing files. {ex.Message}"); - return $"Error in analyzing files. {ex.Message}"; + return $"Error in analyzing files."; } } } From d33e0f69185d131b0f4683fb33548b8d6c7f3f57 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 15:10:21 -0500 Subject: [PATCH 7/7] change model id --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 157cc33a..a884fd7e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -83,7 +83,7 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-multi-modal", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(),