From f9e63097a9ccb91449039eaa6ad7ca7b63db077b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 13 May 2024 16:53:41 -0500 Subject: [PATCH 01/24] add chat files --- .../Files/IBotSharpFileService.cs | 3 +- .../Files/Models/MessageFileModel.cs | 32 +++++ .../Files/Models/OutputFileModel.cs | 13 -- .../MLTasks/Settings/LlmModelSetting.cs | 5 + .../BotSharp.Core/BotSharp.Core.csproj | 1 + .../ConversationService.SendMessage.cs | 7 ++ .../Files/BotSharpFileService.cs | 114 ++++++++++-------- .../Routing/RoutingService.InvokeAgent.cs | 8 +- .../Controllers/FileController.cs | 5 +- src/Infrastructure/BotSharp.OpenAPI/Using.cs | 3 +- .../ViewModels/Files/MessageFileViewModel.cs | 34 ++++++ .../Providers/ChatCompletionProvider.cs | 47 ++++++-- 12 files changed, 193 insertions(+), 79 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index 9c27d7ff..edc04b7b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -3,7 +3,8 @@ namespace BotSharp.Abstraction.Files; public interface IBotSharpFileService { string GetDirectory(string conversationId); - IEnumerable GetConversationFiles(string conversationId, string messageId); + IEnumerable GetChatImages(string conversationId, List conversations, int offset = 2); + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, bool imageOnly = false); string? GetMessageFile(string conversationId, string messageId, string fileName); void SaveMessageFiles(string conversationId, string messageId, List files); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs new file mode 100644 index 00000000..3ec63fd8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class MessageFileModel +{ + [JsonPropertyName("message_id")] + public string MessageId { get; set; } + + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } + + [JsonPropertyName("file_storage_url")] + public string FileStorageUrl { get; set; } + + [JsonPropertyName("file_name")] + public string FileName { get; set; } + + [JsonPropertyName("file_type")] + public string FileType { get; set; } + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } + + public MessageFileModel() + { + + } + + public override string ToString() + { + return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}"; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs deleted file mode 100644 index 962b01e1..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class OutputFileModel -{ - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } - - [JsonPropertyName("file_name")] - public string FileName { get; set; } - - [JsonPropertyName("file_type")] - public string FileType { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 1faf3c52..b86578fe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -27,6 +27,11 @@ public class LlmModelSetting public string Endpoint { get; set; } public LlmModelType Type { get; set; } = LlmModelType.Chat; + /// + /// If true, allow sending images/vidoes to this model + /// + public bool MultiModal { get; set; } + /// /// Prompt cost per 1K token /// diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 8d05c40b..0ddc52e8 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -159,6 +159,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index d6940fe1..4ca64922 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -151,6 +151,13 @@ public partial class ConversationService Message = new TextMessage(response.SecondaryContent ?? response.Content) }; + response.RichContent = new RichContent + { + Recipient = new Recipient { Id = state.GetConversationId() }, + Editor = "file", + Message = new TextMessage(response.SecondaryContent ?? response.Content) + }; + // Patch return function name if (response.PostbackFunctionName != null) { diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index ad687274..581569a1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.StaticFiles; using System.IO; using System.Threading; @@ -8,9 +9,12 @@ public class BotSharpFileService : IBotSharpFileService private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly string _baseDir; + private readonly IEnumerable _allowedTypes = new List { "image/png", "image/jpeg" }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; + private const int MIN_OFFSET = 1; + private const int MAX_OFFSET = 5; public BotSharpFileService( BotSharpDatabaseSettings dbSettings, @@ -31,29 +35,67 @@ public class BotSharpFileService : IBotSharpFileService return dir; } - public IEnumerable GetConversationFiles(string conversationId, string messageId) + public IEnumerable GetChatImages(string conversationId, List conversations, int offset = 2) { - var outputFiles = new List(); - var dir = GetConversationFileDirectory(conversationId, messageId); - if (string.IsNullOrEmpty(dir)) + var files = new List(); + if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) { - return outputFiles; + return files; } - foreach (var file in Directory.GetFiles(dir)) + if (offset <= 0) { - var fileName = Path.GetFileNameWithoutExtension(file); - var extension = Path.GetExtension(file); - var fileType = extension.Substring(1); - var model = new OutputFileModel() - { - FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}", - FileName = fileName, - FileType = fileType - }; - outputFiles.Add(model); + offset = MIN_OFFSET; } - return outputFiles; + else if (offset > MAX_OFFSET) + { + offset = MAX_OFFSET; + } + + var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList(); + files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList(); + return files; + } + + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, bool imageOnly = false) + { + var files = new List(); + if (messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var dir = GetConversationFileDirectory(conversationId, messageId); + if (string.IsNullOrEmpty(dir)) + { + continue; + } + + foreach (var file in Directory.GetFiles(dir)) + { + var contentType = GetFileContentType(file); + if (imageOnly && !_allowedTypes.Contains(contentType)) + { + continue; + } + + var fileName = Path.GetFileNameWithoutExtension(file); + var extension = Path.GetExtension(file); + var fileType = extension.Substring(1); + + var model = new MessageFileModel() + { + MessageId = messageId, + FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}", + FileStorageUrl = file, + FileName = fileName, + FileType = fileType, + ContentType = contentType + }; + files.Add(model); + } + } + + return files; } public string? GetMessageFile(string conversationId, string messageId, string fileName) @@ -182,42 +224,16 @@ public class BotSharpFileService : IBotSharpFileService return Convert.FromBase64String(base64Str); } - private string GetFileType(string data) + private string GetFileContentType(string filePath) { - if (string.IsNullOrEmpty(data)) + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) { - return string.Empty; + contentType = string.Empty; } - var startIdx = data.IndexOf(':'); - var endIdx = data.IndexOf(';'); - var fileType = data.Substring(startIdx + 1, endIdx - startIdx - 1); - return fileType; - } - - private string ParseFileFormat(string type) - { - var parsed = string.Empty; - switch (type) - { - case "image/png": - parsed = ".png"; - break; - case "image/jpeg": - case "image/jpg": - parsed = ".jpeg"; - break; - case "application/pdf": - parsed = ".pdf"; - break; - case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": - parsed = ".xlsx"; - break; - case "text/plain": - parsed = ".txt"; - break; - } - return parsed; + return contentType; } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 6bdac0ef..6046c5bb 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -17,18 +17,18 @@ public partial class RoutingService return false; } - var provide = agent.LlmConfig.Provider; + var provider = agent.LlmConfig.Provider; var model = agent.LlmConfig.Model; - if (provide == null || model == null) + if (provider == null || model == null) { var agentSettings = _services.GetRequiredService(); - provide = agentSettings.LlmConfig.Provider; + provider = agentSettings.LlmConfig.Provider; model = agentSettings.LlmConfig.Model; } var chatCompletion = CompletionProvider.GetChatCompletion(_services, - provider: provide, + provider: provider, model: model); var message = dialogs.Last(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs index a24357a2..cfae602c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs @@ -38,10 +38,11 @@ public class FileController : ControllerBase } [HttpGet("/conversation/{conversationId}/files/{messageId}")] - public IEnumerable GetConversationFiles([FromRoute] string conversationId, [FromRoute] string messageId) + public IEnumerable GetMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId) { var fileService = _services.GetRequiredService(); - return fileService.GetConversationFiles(conversationId, messageId); + var files = fileService.GetMessageFiles(conversationId, new List { messageId }); + return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/Using.cs b/src/Infrastructure/BotSharp.OpenAPI/Using.cs index f3ba775f..8771b81c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Using.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Using.cs @@ -28,4 +28,5 @@ global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files; global using BotSharp.OpenAPI.ViewModels.Conversations; global using BotSharp.OpenAPI.ViewModels.Users; -global using BotSharp.OpenAPI.ViewModels.Agents; \ No newline at end of file +global using BotSharp.OpenAPI.ViewModels.Agents; +global using BotSharp.OpenAPI.ViewModels.Files; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs new file mode 100644 index 00000000..a9eb33bd --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -0,0 +1,34 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Files; + +public class MessageFileViewModel +{ + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } + + [JsonPropertyName("file_name")] + public string FileName { get; set; } + + [JsonPropertyName("file_type")] + public string FileType { get; set; } + + [JsonPropertyName("content_type")] + public string ContentType { get; set; } + + public MessageFileViewModel() + { + + } + + public static MessageFileViewModel Transform(MessageFileModel model) + { + return new MessageFileViewModel + { + FileUrl = model.FileUrl, + FileName = model.FileName, + FileType = model.FileType, + ContentType = model.ContentType + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 07a3ff8f..b2bbe71b 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -4,13 +4,17 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.AzureOpenAI.Settings; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -218,6 +222,16 @@ public class ChatCompletionProvider : IChatCompletion protected (string, ChatCompletionsOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + var chatFiles = new List(); + if (settings != null && settings.MultiModal) + { + chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList(); + } var chatCompletionsOptions = new ChatCompletionsOptions(); @@ -279,19 +293,34 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == ChatRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - var userMessage = new ChatRequestUserMessage(text) + var chatItems = new List() + { + new ChatMessageTextContentItem(text) + }; + + var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList(); + if (!files.IsNullOrEmpty()) + { + foreach (var file in files) + { + using var stream = File.OpenRead(file.FileStorageUrl); + chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low)); + } + } + + //if (!string.IsNullOrEmpty(message.ImageUrl)) + //{ + // var uri = new Uri(message.ImageUrl); + // userMessage.MultimodalContentItems.Add( + // new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + //} + + var userMessage = new ChatRequestUserMessage(chatItems) { // To display Planner name in log Name = message.FunctionName, }; - if (!string.IsNullOrEmpty(message.ImageUrl)) - { - var uri = new Uri(message.ImageUrl); - userMessage.MultimodalContentItems.Add( - new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); - } - chatCompletionsOptions.Messages.Add(userMessage); } else if (message.Role == ChatRole.Assistant) @@ -301,7 +330,7 @@ public class ChatCompletionProvider : IChatCompletion } // https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683 - var state = _services.GetRequiredService(); + //var state = _services.GetRequiredService(); var temperature = float.Parse(state.GetState("temperature", "0.0")); var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0")); chatCompletionsOptions.Temperature = temperature; From f3bbb0259f281269db4fd58deaae205449e3dd50 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 10:14:08 -0500 Subject: [PATCH 02/24] remove test code --- .../Services/ConversationService.SendMessage.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 4ca64922..d6940fe1 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -151,13 +151,6 @@ public partial class ConversationService Message = new TextMessage(response.SecondaryContent ?? response.Content) }; - response.RichContent = new RichContent - { - Recipient = new Recipient { Id = state.GetConversationId() }, - Editor = "file", - Message = new TextMessage(response.SecondaryContent ?? response.Content) - }; - // Patch return function name if (response.PostbackFunctionName != null) { From 83a78d2b5af71219fc59a5ded3e541ae80c341d7 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 11:51:39 -0500 Subject: [PATCH 03/24] add instruct multi modal --- .../Files/IBotSharpFileService.cs | 7 +++ .../Files/Models/BotSharpFile.cs | 11 ++-- .../Files/BotSharpFileService.cs | 59 ++++++++++++------- .../Infrastructures/CompletionProvider.cs | 2 +- .../Controllers/InstructModeController.cs | 35 ++++++++++- .../Providers/ChatCompletionProvider.cs | 22 ++++++- 6 files changed, 104 insertions(+), 32 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index edc04b7b..272abf0d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -18,4 +18,11 @@ public interface IBotSharpFileService /// bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null); bool DeleteConversationFiles(IEnumerable conversationIds); + + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + (string, byte[]) GetFileInfoFromData(string data); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index f679d52e..9581b83f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -4,14 +4,11 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile { [JsonPropertyName("file_name")] - public string FileName { get; set; } + public string FileName { get; set; } = string.Empty; [JsonPropertyName("file_data")] - public string FileData { get; set; } + public string FileData { get; set; } = string.Empty; - [JsonPropertyName("content_type")] - public string ContentType { get; set; } - - [JsonPropertyName("file_size")] - public int FileSize { get; set; } + [JsonPropertyName("file_url")] + public string FileUrl { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs index 581569a1..d7e961be 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs @@ -8,6 +8,7 @@ public class BotSharpFileService : IBotSharpFileService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _allowedTypes = new List { "image/png", "image/jpeg" }; @@ -18,9 +19,11 @@ public class BotSharpFileService : IBotSharpFileService public BotSharpFileService( BotSharpDatabaseSettings dbSettings, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; + _logger = logger; _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } @@ -117,19 +120,26 @@ public class BotSharpFileService : IBotSharpFileService var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true); if (string.IsNullOrEmpty(dir)) return; - for (int i = 0; i < files.Count; i++) + try { - var file = files[i]; - if (string.IsNullOrEmpty(file.FileData)) + for (int i = 0; i < files.Count; i++) { - continue; - } + var file = files[i]; + if (string.IsNullOrEmpty(file.FileData)) + { + continue; + } - var bytes = GetFileBytes(file.FileData); - var fileType = Path.GetExtension(file.FileName); - var fileName = $"{i + 1}{fileType}"; - Thread.Sleep(100); - File.WriteAllBytes(Path.Combine(dir, fileName), bytes); + var (_, bytes) = GetFileInfoFromData(file.FileData); + var fileType = Path.GetExtension(file.FileName); + var fileName = $"{i + 1}{fileType}"; + Thread.Sleep(100); + File.WriteAllBytes(Path.Combine(dir, fileName), bytes); + } + } + catch (Exception ex) + { + _logger.LogError($"Error when saving conversation files: {ex.Message}"); } } @@ -179,6 +189,23 @@ public class BotSharpFileService : IBotSharpFileService return true; } + public (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + #region Private methods private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false) { @@ -212,18 +239,6 @@ public class BotSharpFileService : IBotSharpFileService return dir; } - private byte[] GetFileBytes(string data) - { - if (string.IsNullOrEmpty(data)) - { - return new byte[0]; - } - - var startIdx = data.IndexOf(','); - var base64Str = data.Substring(startIdx + 1); - return Convert.FromBase64String(base64Str); - } - private string GetFileContentType(string filePath) { string contentType; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index c55ed9a8..bc0266ed 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -47,7 +47,7 @@ public class CompletionProvider logger.LogError($"Can't resolve completion provider by {provider}"); } - completer.SetModelName(model); + completer?.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 985526d9..9f00a3e0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -11,10 +11,12 @@ namespace BotSharp.OpenAPI.Controllers; public class InstructModeController : ControllerBase { private readonly IServiceProvider _services; + private readonly ILogger _logger; - public InstructModeController(IServiceProvider services) + public InstructModeController(IServiceProvider services, ILogger logger) { _services = services; + _logger = logger; } [HttpPost("/instruct/{agentId}")] @@ -72,4 +74,35 @@ public class InstructModeController : ControllerBase }); return message.Content; } + + [HttpPost("/instruct/multi-modal")] + public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + state.SetState("provider", input.Provider, source: StateSource.External) + .SetState("model", input.Model, source: StateSource.External) + .SetState("model_id", input.ModelId, source: StateSource.External); + + try + { + var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo"); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, input.Text) + { + Files = input.Files + } + }); + return message.Content; + } + catch (Exception ex) + { + _logger.LogError($"Error in analyzing files. {ex.Message}"); + return $"Error in analyzing files. {ex.Message}"; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index b2bbe71b..d884e386 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices.ComTypes; using System.Threading.Tasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; @@ -226,9 +227,10 @@ public class ChatCompletionProvider : IChatCompletion var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); + var allowMultiModal = settings != null && settings.MultiModal; var chatFiles = new List(); - if (settings != null && settings.MultiModal) + if (allowMultiModal) { chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList(); } @@ -308,6 +310,24 @@ public class ChatCompletionProvider : IChatCompletion } } + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + foreach (var file in message.Files) + { + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + using var stream = new MemoryStream(bytes, 0, bytes.Length); + chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low)); + } + } + } + //if (!string.IsNullOrEmpty(message.ImageUrl)) //{ // var uri = new Uri(message.ImageUrl); From 7e908d33868a8a972ed18663929312bf747584c9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 11:55:11 -0500 Subject: [PATCH 04/24] add comment --- .../BotSharp.Abstraction/Files/Models/BotSharpFile.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 9581b83f..de226f58 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -6,6 +6,9 @@ public class BotSharpFile [JsonPropertyName("file_name")] public string FileName { get; set; } = string.Empty; + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// [JsonPropertyName("file_data")] public string FileData { get; set; } = string.Empty; From 3d3359cb951c40a7b04584e764fe3a837cfd3e74 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 13:34:10 -0500 Subject: [PATCH 05/24] filter by model id and multi-modal --- .../MLTasks/ILlmProviderService.cs | 2 +- .../Infrastructures/CompletionProvider.cs | 13 +++++++++---- .../Infrastructures/LlmProviderService.cs | 4 ++-- .../Controllers/InstructModeController.cs | 5 +---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 75fd60e6..120304d2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id); + LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index bc0266ed..ace1e664 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -35,10 +35,13 @@ public class CompletionProvider public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null, string? model = null, + string? modelId = null, + bool multiModal = false, AgentLlmConfig? agentConfig = null) { var completions = services.GetServices(); - (provider, model) = GetProviderAndModel(services, provider: provider, model: model, agentConfig: agentConfig); + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, + multiModal: multiModal, agentConfig: agentConfig); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) @@ -55,6 +58,8 @@ public class CompletionProvider private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, + string? modelId = null, + bool multiModal = false, AgentLlmConfig? agentConfig = null) { var agentSetting = services.GetRequiredService(); @@ -73,11 +78,11 @@ public class CompletionProvider { model = state.GetState("model", model ?? "gpt-35-turbo-4k"); } - else if (state.ContainsState("model_id")) + else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) { - var modelId = state.GetState("model_id"); + var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; var llmProviderService = services.GetRequiredService(); - model = llmProviderService.GetProviderModel(provider, modelId)?.Name; + model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal)?.Name; } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index eb92ac51..7d92a687 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,10 +44,10 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id) + public LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false) { var models = GetProviderModels(provider) - .Where(x => x.Id == id) + .Where(x => x.Id == id && x.MultiModal == multiModal) .ToList(); var random = new Random(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 9f00a3e0..96fe3728 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -80,13 +80,10 @@ public class InstructModeController : ControllerBase { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - state.SetState("provider", input.Provider, source: StateSource.External) - .SetState("model", input.Model, source: StateSource.External) - .SetState("model_id", input.ModelId, source: StateSource.External); try { - var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo"); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4-turbo", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), From 971018ba503df6acd85bd405506a81490c9fb5bf Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 14:30:27 -0500 Subject: [PATCH 06/24] remove error message --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 96fe3728..157cc33a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -83,7 +83,7 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4-turbo", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-multi-modal", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), @@ -99,7 +99,7 @@ public class InstructModeController : ControllerBase catch (Exception ex) { _logger.LogError($"Error in analyzing files. {ex.Message}"); - return $"Error in analyzing files. {ex.Message}"; + return $"Error in analyzing files."; } } } From d33e0f69185d131b0f4683fb33548b8d6c7f3f57 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 14 May 2024 15:10:21 -0500 Subject: [PATCH 07/24] change model id --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 157cc33a..a884fd7e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -83,7 +83,7 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-multi-modal", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), From 1fa31ad68046fe5fdfb570824d30ea37f6e858ba Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 15 May 2024 06:43:05 -0500 Subject: [PATCH 08/24] release v1.4 --- Directory.Build.props | 4 ++-- .../BotSharp.Plugin.AnthropicAI.csproj | 10 +++++++--- src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs | 6 ++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 99a8ba4c..068b6e13 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,8 +2,8 @@ net8.0 10.0 - 1.3.1 - false + 1.4.0 + true false \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj index 357e827f..40f54f60 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj @@ -1,9 +1,13 @@ - + - net8.0 - enable + netstandard2.1 enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs index ceb477d6..d00446fc 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs @@ -1,3 +1,9 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Threading.Tasks; +global using System.Linq; +global using System.Text.Json; global using Anthropic.SDK; global using Anthropic.SDK.Constants; global using Anthropic.SDK.Messaging; From 6fba6c8e76384af35170b6ee2000a67ad8e8334f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 15 May 2024 11:23:02 -0500 Subject: [PATCH 09/24] add conv user endpoint --- .../Controllers/ConversationController.cs | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 810bc08d..d84e62fa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Routing; using Newtonsoft.Json.Serialization; using Newtonsoft.Json; -using BotSharp.Abstraction.Files.Models; -using BotSharp.Abstraction.Files; namespace BotSharp.OpenAPI.Controllers; @@ -138,6 +136,34 @@ public class ConversationController : ControllerBase return result; } + [HttpGet("/conversation/{conversationId}/user")] + public async Task GetConversationUser([FromRoute] string conversationId) + { + var service = _services.GetRequiredService(); + var conversations = await service.GetConversations(new ConversationFilter + { + Id = conversationId + }); + + var userService = _services.GetRequiredService(); + var conversation = conversations?.Items?.FirstOrDefault(); + var userId = conversation == null ? _user.Id : conversation.UserId; + var user = await userService.GetUser(userId); + if (user == null) + { + return new UserViewModel + { + Id = _user.Id, + FirstName = _user.FirstName, + LastName = _user.LastName, + Email = _user.Email, + Source = "Unknown" + }; + } + + return UserViewModel.FromUser(user); + } + [HttpDelete("/conversation/{conversationId}")] public async Task DeleteConversation([FromRoute] string conversationId) { From 2a9b5ec0c89b51045ef956a3d4639feba40b4f0e Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 15 May 2024 11:23:29 -0500 Subject: [PATCH 10/24] minor change --- .../BotSharp.OpenAPI/Controllers/ConversationController.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index d84e62fa..34095acf 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -154,6 +154,7 @@ public class ConversationController : ControllerBase return new UserViewModel { Id = _user.Id, + UserName = _user.UserName, FirstName = _user.FirstName, LastName = _user.LastName, Email = _user.Email, From 842f1e788edf5237c3a1eff1fb9438e1e0a4b7cb Mon Sep 17 00:00:00 2001 From: "C. Oceania" Date: Wed, 15 May 2024 14:09:38 -0500 Subject: [PATCH 11/24] Update translation_prompt.liquid --- .../templates/translation_prompt.liquid | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid index 9d9672b5..be4f1077 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid @@ -1,7 +1,9 @@ {% for text in text_list %} {{ text }} {% endfor %} - ===== -Translate the above sentences in the list into {{ language }}. -Output the translated text in JSON {"input_lang":"", "output_lang":"{{ language }}", "texts":[]}, input_lang is based on the original sentences. \ No newline at end of file +Translate the above sentences into {{ language }}. +Output the translated text in JSON {"input_lang":"original text language", "output_lang":"{{ language }}", "texts":[""]}. +Do not include the serial number before each sentence. +Do not include double quotes outside the sentence. +The number of output sentences must be {{ text_list | size }}. From 3bf73bec4aeb05c2d4ecb5e5927cbff2de032f39 Mon Sep 17 00:00:00 2001 From: "C. Oceania" Date: Wed, 15 May 2024 14:32:59 -0500 Subject: [PATCH 12/24] Update RoutingService.cs --- src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 5e0ca8ac..1a9ce427 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -88,7 +88,7 @@ public partial class RoutingService : IRoutingService { var translator = _services.GetRequiredService(); - var language = states.GetState(StateConst.LANGUAGE, LanguageType.UNKNOWN); + var language = states.GetState(StateConst.LANGUAGE, LanguageType.ENGLISH); if (language != LanguageType.ENGLISH) { message.SecondaryContent = message.Content; From 03837c1e8348c6c5124ebe0cc0a6981be82cbe11 Mon Sep 17 00:00:00 2001 From: "C. Oceania" Date: Wed, 15 May 2024 14:34:57 -0500 Subject: [PATCH 13/24] Update ChatCompletionProvider.cs --- .../Providers/ChatCompletionProvider.cs | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index d884e386..990a74ab 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -295,51 +295,64 @@ public class ChatCompletionProvider : IChatCompletion else if (message.Role == ChatRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - var chatItems = new List() - { - new ChatMessageTextContentItem(text) - }; - var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList(); - if (!files.IsNullOrEmpty()) + ChatRequestUserMessage userMessage = null; + if (allowMultiModal) { - foreach (var file in files) + var chatItems = new List() { - using var stream = File.OpenRead(file.FileStorageUrl); - chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low)); - } - } - - if (allowMultiModal && !message.Files.IsNullOrEmpty()) - { - foreach (var file in message.Files) + new ChatMessageTextContentItem(text) + }; + + var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList(); + if (!files.IsNullOrEmpty()) { - if (!string.IsNullOrEmpty(file.FileUrl)) + foreach (var file in files) { - var uri = new Uri(file.FileUrl); - chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); - using var stream = new MemoryStream(bytes, 0, bytes.Length); - chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low)); + using var stream = File.OpenRead(file.FileStorageUrl); + chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low)); } } + + if (!message.Files.IsNullOrEmpty()) + { + foreach (var file in message.Files) + { + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + using var stream = new MemoryStream(bytes, 0, bytes.Length); + chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low)); + } + } + } + + //if (!string.IsNullOrEmpty(message.ImageUrl)) + //{ + // var uri = new Uri(message.ImageUrl); + // userMessage.MultimodalContentItems.Add( + // new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); + //} + + userMessage = new ChatRequestUserMessage(chatItems) + { + // To display Planner name in log + Name = message.FunctionName, + }; } - - //if (!string.IsNullOrEmpty(message.ImageUrl)) - //{ - // var uri = new Uri(message.ImageUrl); - // userMessage.MultimodalContentItems.Add( - // new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low)); - //} - - var userMessage = new ChatRequestUserMessage(chatItems) + else { - // To display Planner name in log - Name = message.FunctionName, - }; + userMessage = new ChatRequestUserMessage(text) + { + // To display Planner name in log + Name = message.FunctionName, + }; + } chatCompletionsOptions.Messages.Add(userMessage); } @@ -396,9 +409,12 @@ public class ChatCompletionProvider : IChatCompletion else if (x.Role == ChatRole.User) { var m = x as ChatRequestUserMessage; + var content = m.Content ?? string.Join(", ", m.MultimodalContentItems + .Where(m => m is ChatMessageTextContentItem) + .Select(m => (m as ChatMessageTextContentItem)?.Text)); return !string.IsNullOrEmpty(m.Name) && m.Name != "route_to_agent" ? - $"{m.Name}: {m.Content}" : - $"{m.Role}: {m.Content}"; + $"{m.Name}: {content}" : + $"{m.Role}: {content}"; } else if (x.Role == ChatRole.Assistant) { From d4493e0db64ef687e77e1235721fbbe91210c700 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 15 May 2024 21:45:31 -0500 Subject: [PATCH 14/24] prevent send event if it is not conversation --- .../Hooks/StreamingLogHook.cs | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index ab662546..0d578f99 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.Loggers.Enums; @@ -49,6 +50,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnMessageReceived(RoleDialogModel message) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var log = $"{GetMessageContent(message)}"; var input = new ContentLogInputModel(conversationId, message) @@ -63,6 +66,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnPostbackMessageReceived(RoleDialogModel message, PostbackMessageModel replyMsg) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var log = $"{GetMessageContent(message)}"; var replyContent = JsonSerializer.Serialize(replyMsg, _options.JsonSerializerOptions); log += $"\r\n```json\r\n{replyContent}\r\n```"; @@ -81,6 +86,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR if (!_convSettings.ShowVerboseLog) return; var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; var log = $"{agent.Name} is using template {name}"; var message = new RoleDialogModel(AgentRole.System, log) @@ -104,12 +110,11 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnFunctionExecuting(RoleDialogModel message) { - if (message.FunctionName == "route_to_agent") - { - return; - } - var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + + if (message.FunctionName == "route_to_agent") return; + var agent = await _agentService.LoadAgent(message.CurrentAgentId); message.FunctionArgs = message.FunctionArgs ?? "{}"; var args = JsonSerializer.Serialize(JsonDocument.Parse(message.FunctionArgs), _options.JsonSerializerOptions); @@ -127,12 +132,11 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnFunctionExecuted(RoleDialogModel message) { - if (message.FunctionName == "route_to_agent") - { - return; - } - var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + + if (message.FunctionName == "route_to_agent") return; + var agent = await _agentService.LoadAgent(message.CurrentAgentId); message.FunctionArgs = message.FunctionArgs ?? "{}"; // var args = JsonSerializer.Serialize(JsonDocument.Parse(message.FunctionArgs), _options.JsonSerializerOptions); @@ -159,6 +163,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR if (!_convSettings.ShowVerboseLog) return; var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var agent = await _agentService.LoadAgent(message.CurrentAgentId); var log = tokenStats.Prompt; @@ -180,8 +186,10 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR /// public override async Task OnResponseGenerated(RoleDialogModel message) { - var conv = _services.GetRequiredService(); + var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var conv = _services.GetRequiredService(); await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStateLogGenerated", BuildStateLog(conv.ConversationId, _state.GetStates(), message)); if (message.Role == AgentRole.Assistant) @@ -208,6 +216,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnTaskCompleted(RoleDialogModel message) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var log = $"{GetMessageContent(message)}"; var agent = await _agentService.LoadAgent(message.CurrentAgentId); @@ -223,6 +233,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnConversationEnding(RoleDialogModel message) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var log = $"Conversation ended"; var agent = await _agentService.LoadAgent(message.CurrentAgentId); @@ -237,6 +249,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnBreakpointUpdated(string conversationId, bool resetStates) { + if (string.IsNullOrEmpty(conversationId)) return; + var log = $"Conversation breakpoint is updated"; if (resetStates) { @@ -263,6 +277,9 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnStateChanged(StateChangeModel stateChange) { + var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + if (stateChange == null) return; await _chatHub.Clients.User(_user.Id).SendAsync("OnStateChangeGenerated", BuildStateChangeLog(stateChange)); @@ -273,6 +290,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public async Task OnAgentEnqueued(string agentId, string preAgentId, string? reason = null) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var agent = await _agentService.LoadAgent(agentId); // Agent queue log @@ -298,6 +317,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public async Task OnAgentDequeued(string agentId, string currentAgentId, string? reason = null) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var agent = await _agentService.LoadAgent(agentId); var currentAgent = await _agentService.LoadAgent(currentAgentId); @@ -324,6 +345,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public async Task OnAgentReplaced(string fromAgentId, string toAgentId, string? reason = null) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var fromAgent = await _agentService.LoadAgent(fromAgentId); var toAgent = await _agentService.LoadAgent(toAgentId); @@ -350,6 +373,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public async Task OnAgentQueueEmptied(string agentId, string? reason = null) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; // Agent queue log var log = $"Agent queue is empty"; @@ -374,6 +398,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public async Task OnRoutingInstructionReceived(FunctionCallFromLlm instruct, RoleDialogModel message) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var agent = await _agentService.LoadAgent(message.CurrentAgentId); var log = JsonSerializer.Serialize(instruct, _options.JsonSerializerOptions); log = $"```json\r\n{log}\r\n```"; @@ -391,6 +417,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public async Task OnRoutingInstructionRevised(FunctionCallFromLlm instruct, RoleDialogModel message) { var conversationId = _state.GetConversationId(); + if (string.IsNullOrEmpty(conversationId)) return; + var agent = await _agentService.LoadAgent(message.CurrentAgentId); var log = $"Revised user goal agent to {instruct.OriginalAgent}"; From 57122f7833cd73ceee1ce24289fe41a5d6497c63 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 16 May 2024 16:32:58 -0500 Subject: [PATCH 15/24] Fix llm selection bug. --- .../MLTasks/ILlmProviderService.cs | 2 +- .../Infrastructures/CompletionProvider.cs | 6 +++--- .../Infrastructures/LlmProviderService.cs | 10 +++++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 120304d2..20762fe0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index ace1e664..4655b1de 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -36,7 +36,7 @@ public class CompletionProvider string? provider = null, string? model = null, string? modelId = null, - bool multiModal = false, + bool? multiModal = null, AgentLlmConfig? agentConfig = null) { var completions = services.GetServices(); @@ -59,7 +59,7 @@ public class CompletionProvider string? provider = null, string? model = null, string? modelId = null, - bool multiModal = false, + bool? multiModal = null, AgentLlmConfig? agentConfig = null) { var agentSetting = services.GetRequiredService(); @@ -82,7 +82,7 @@ public class CompletionProvider { var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; var llmProviderService = services.GetRequiredService(); - model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal)?.Name; + model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name; } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 7d92a687..8320bdb7 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,11 +44,15 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null) { var models = GetProviderModels(provider) - .Where(x => x.Id == id && x.MultiModal == multiModal) - .ToList(); + .Where(x => x.Id == id); + + if (multiModal.HasValue) + { + models = models.Where(x => x.MultiModal == multiModal); + } var random = new Random(); var index = random.Next(0, models.Count()); From 98592438684900ce8cad133dbcf47b016ddb15c4 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 17 May 2024 11:35:20 -0500 Subject: [PATCH 16/24] add default model --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index a884fd7e..45120a26 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -83,7 +83,8 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", modelId: "gpt-4", multiModal: true); + var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", + modelId: input.ModelId ?? "gpt-4", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), From b35f0e653656640fda3b375dba9aa2d271194bdc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 20 May 2024 11:35:45 -0500 Subject: [PATCH 17/24] refine log in --- .../BotSharp.Abstraction/Users/IUserService.cs | 2 +- .../BotSharp.Core/Users/Services/UserService.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 35d74aa4..debf68f4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -6,6 +6,6 @@ public interface IUserService { Task GetUser(string id); Task CreateUser(User user); - Task GetToken(string authorization); + Task GetToken(string authorization); Task GetMyProfile(); } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 6e856b9f..9508dd2b 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Users.Models; using Microsoft.Extensions.Configuration; using Microsoft.IdentityModel.Tokens; @@ -60,7 +59,7 @@ public class UserService : IUserService return record; } - public async Task GetToken(string authorization) + public async Task GetToken(string authorization) { var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization)); var (id, password) = base64.SplitAsTuple(":"); @@ -72,13 +71,14 @@ public class UserService : IUserService record = db.GetUserByUserName(id); } + User? user = null; var hooks = _services.GetServices(); if (record == null || record.Source != "internal") { // check 3rd party user foreach (var hook in hooks) { - var user = await hook.Authenticate(id, password); + user = await hook.Authenticate(id, password); if (user == null) { continue; @@ -109,7 +109,7 @@ public class UserService : IUserService } } - if (record == null) + if ((!hooks.IsNullOrEmpty() && user == null) || record == null) { return default; } From efdf5a30da9a9f368a4168fb1ba624223adac952 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 20 May 2024 12:54:38 -0500 Subject: [PATCH 18/24] Improve translation. --- .../Translation/Models/TranslationInput.cs | 10 ++++++++++ .../Templating/ResponseTemplateService.cs | 2 -- .../BotSharp.Core/Templating/TemplateRender.cs | 2 ++ .../Translation/TranslationService.cs | 17 +++++++++++++---- .../templates/translation_prompt.liquid | 8 ++------ 5 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationInput.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationInput.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationInput.cs new file mode 100644 index 00000000..6897ca42 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationInput.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Translation.Models; + +public class TranslationInput +{ + [JsonPropertyName("id")] + public int Id { get; set; } = -1; + + [JsonPropertyName("text")] + public string Text { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs b/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs index 5119bc91..463013bf 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Templating; using System.Reflection; diff --git a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs index 3bdbf7e1..33b27177 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Templating; +using BotSharp.Abstraction.Translation.Models; using Fluid; namespace BotSharp.Core.Templating; @@ -30,6 +31,7 @@ public class TemplateRender : ITemplateRender _options.MemberAccessStrategy.Register(); _options.MemberAccessStrategy.Register(); _options.MemberAccessStrategy.Register(); + _options.MemberAccessStrategy.Register(); } public string Render(string template, Dictionary dict) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index d24fd7d4..ff28b4e6 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -57,8 +57,11 @@ public class TranslationService : ITranslationService var keys = unique.ToArray(); var texts = unique.ToArray() - .Select((text, i) => $"{i + 1}. \"{text}\"") - .ToList(); + .Select((text, i) => new TranslationInput + { + Id = i + 1, + Text = text + }).ToList(); var translatedStringList = await InnerTranslate(texts, language, template); try @@ -297,15 +300,21 @@ public class TranslationService : ITranslationService /// /// /// - private async Task InnerTranslate(List texts, string language, string template) + private async Task InnerTranslate(List texts, string language, string template) { + var jsonString = JsonSerializer.Serialize(texts, new JsonSerializerOptions + { + WriteIndented = true, + }) ; var translator = new Agent { Id = Guid.Empty.ToString(), Name = "Translator", + Instruction = "You are a translation expert.", TemplateDict = new Dictionary { - { "text_list", texts }, + { "text_list", jsonString }, + { "text_list_size", texts.Count }, { StateConst.LANGUAGE, language } } }; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid index be4f1077..7403130c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid @@ -1,9 +1,5 @@ -{% for text in text_list %} -{{ text }} -{% endfor %} +{{ text_list }} + ===== Translate the above sentences into {{ language }}. Output the translated text in JSON {"input_lang":"original text language", "output_lang":"{{ language }}", "texts":[""]}. -Do not include the serial number before each sentence. -Do not include double quotes outside the sentence. -The number of output sentences must be {{ text_list | size }}. From 2147d69ecf433638cb8c3388ab3ad295a792a7df Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 20 May 2024 21:13:12 -0500 Subject: [PATCH 19/24] translation improvement. --- .../BotSharp.Core/Translation/TranslationService.cs | 5 +---- .../templates/translation_prompt.liquid | 4 ++-- .../Controllers/ConversationController.cs | 9 +-------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index ff28b4e6..5c60e108 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -302,10 +302,7 @@ public class TranslationService : ITranslationService /// private async Task InnerTranslate(List texts, string language, string template) { - var jsonString = JsonSerializer.Serialize(texts, new JsonSerializerOptions - { - WriteIndented = true, - }) ; + var jsonString = JsonSerializer.Serialize(texts); var translator = new Agent { Id = Guid.Empty.ToString(), diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid index 7403130c..3d33375b 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid @@ -1,5 +1,5 @@ {{ text_list }} ===== -Translate the above sentences into {{ language }}. -Output the translated text in JSON {"input_lang":"original text language", "output_lang":"{{ language }}", "texts":[""]}. +Translate all the above sentences into {{ language }}. +Output the translated text in JSON {"input_lang":"original text language", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""}]}. diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 34095acf..a1d2ca3e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Routing; -using Newtonsoft.Json.Serialization; -using Newtonsoft.Json; namespace BotSharp.OpenAPI.Controllers; @@ -303,12 +301,7 @@ public class ConversationController : ControllerBase private async Task OnChunkReceived(HttpResponse response, RoleDialogModel message) { - var json = JsonConvert.SerializeObject(message, new JsonSerializerSettings - { - Formatting = Formatting.None, - ContractResolver = new CamelCasePropertyNamesContractResolver(), - NullValueHandling = NullValueHandling.Ignore, - }); + var json = JsonSerializer.Serialize(message); var buffer = Encoding.UTF8.GetBytes($"data:{json}\n"); await response.Body.WriteAsync(buffer, 0, buffer.Length); From 6a0a400500951cfef9cae03bf88533ede37ba008 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Tue, 21 May 2024 12:02:06 -0500 Subject: [PATCH 20/24] Fix SSE response format. --- .../Translation/Models/TranslationOutput.cs | 2 +- .../Translation/TranslationService.cs | 2 +- .../Controllers/ConversationController.cs | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationOutput.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationOutput.cs index 52ad54ec..b15bfef4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationOutput.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationOutput.cs @@ -9,5 +9,5 @@ public class TranslationOutput public string OutputLanguage { get; set; } = LanguageType.ENGLISH; [JsonPropertyName("texts")] - public string[] Texts { get; set; } = Array.Empty(); + public TranslationInput[] Texts { get; set; } = Array.Empty(); } diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index 5c60e108..e5149751 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -79,7 +79,7 @@ public class TranslationService : ITranslationService for (var i = 0; i < texts.Count; i++) { - map[keys[i]] = translatedTexts[i]; + map[keys[i]] = translatedTexts[i].Text; } clonedData = Assign(clonedData, map); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index a1d2ca3e..80374a6d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -257,7 +257,11 @@ public class ConversationController : ControllerBase conv.SetConversationId(conversationId, input.States); SetStates(conv, input); - var response = new ChatResponseModel(); + var response = new ChatResponseModel + { + ConversationId = conversationId, + MessageId = inputMsg.MessageId, + }; Response.StatusCode = 200; Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.ContentType, "text/event-stream"); @@ -266,6 +270,7 @@ public class ConversationController : ControllerBase await conv.SendMessage(agentId, inputMsg, replyMessage: input.Postback, + // responsed generated async msg => { response.Text = !string.IsNullOrEmpty(msg.SecondaryContent) ? msg.SecondaryContent : msg.Content; @@ -274,18 +279,21 @@ public class ConversationController : ControllerBase response.Instruction = msg.Instruction; response.Data = msg.Data; - await OnChunkReceived(Response, msg); + await OnChunkReceived(Response, response); }, + // executing async msg => { - var message = new RoleDialogModel(AgentRole.Function, msg.Content) + var indicator = new ChatResponseModel { - FunctionArgs = msg.FunctionArgs, - FunctionName = msg.FunctionName, - Indication = msg.Indication + ConversationId = conversationId, + MessageId = msg.MessageId, + Text = msg.Indication, + Function = "indicating", }; - await OnChunkReceived(Response, message); + await OnChunkReceived(Response, indicator); }, + // executed async msg => { @@ -299,7 +307,7 @@ public class ConversationController : ControllerBase // await OnEventCompleted(Response); } - private async Task OnChunkReceived(HttpResponse response, RoleDialogModel message) + private async Task OnChunkReceived(HttpResponse response, ChatResponseModel message) { var json = JsonSerializer.Serialize(message); From 343f3b91abcea27648b6dc62b424733aa36e5d78 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 21 May 2024 22:07:26 -0500 Subject: [PATCH 21/24] Retry language translation. --- .../BotSharp.Core/Translation/TranslationService.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index e5149751..44bb22a8 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -62,10 +62,18 @@ public class TranslationService : ITranslationService Id = i + 1, Text = text }).ToList(); - var translatedStringList = await InnerTranslate(texts, language, template); try { + var translatedStringList = await InnerTranslate(texts, language, template); + + int retry = 0; + while (translatedStringList.Texts.Length != texts.Count && retry < 3) + { + translatedStringList = await InnerTranslate(texts, language, template); + retry++; + } + // Override language if it's Unknown, it's used to output the corresponding language. var states = _services.GetRequiredService(); if (!states.ContainsState(StateConst.LANGUAGE)) From adc6486345fe53bd20d8d40b565fe32d9140f1a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E7=A3=8A?= Date: Wed, 22 May 2024 14:53:36 +0800 Subject: [PATCH 22/24] Update TranslationService.cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hotfix json escaped question error: [ERR] [] '<' is not a hex digit following '\u' within a JSON string. The string should be correctly escaped. Path: $.texts[0].text | LineNumber: 0 | BytePositionInLine: 411. json: {"input_lang":"English", "output_count":3, "output_lang":"Spanish", "texts":[{"id":1,"text":"Aquí hay un resumen de los horarios que ha indicado que está disponible. Las fechas no están confirmadas. El técnico revisará y se pondrá en contacto para programar.\u003Cp\u003E\u003Cb\u003EJueves, 23 de mayo de 2024\u003C/b\u003E\u003C/br\u003ETodo el día (08:00 AM - 08:00 PM)\u003C/p\u003E\u003Cp\u003E\u003Viernes, 24 de mayo de 2024\u003C/b\u003E\u003C/br\u003EMañana (08:00 AM - 01:00 PM)\u003C/p\u003E\u003Cp\u003E\u003Lunes, 27 de mayo de 2024\u003C/b\u003E\u003C/br\u003ETarde (01:00 PM - 08:00 PM)\u003C/p\u003E"},{"id":2,"text":"Se ve bien"},{"id":3,"text":"Empezar de nuevo"}]} --- .../BotSharp.Core/Translation/TranslationService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index 44bb22a8..7067eb7e 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -310,7 +310,8 @@ public class TranslationService : ITranslationService /// private async Task InnerTranslate(List texts, string language, string template) { - var jsonString = JsonSerializer.Serialize(texts); + var options = new JsonSerializerOptions() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; + var jsonString = JsonSerializer.Serialize(texts, options); var translator = new Agent { Id = Guid.Empty.ToString(), From 6f8c840dcd273ded0c03d237efef2c0a4369602d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E7=A3=8A?= Date: Wed, 22 May 2024 15:08:13 +0800 Subject: [PATCH 23/24] Update translation_prompt.liquid optimize translation prompt 1.Enrich the translation examples. 2.Add necessary validation conditions for "output_count". --- .../templates/translation_prompt.liquid | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid index 3d33375b..6b3a8677 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid @@ -2,4 +2,5 @@ ===== Translate all the above sentences into {{ language }}. -Output the translated text in JSON {"input_lang":"original text language", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""}]}. +Output the translated text in JSON {"input_lang":"original text language", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""},{"id": 2, "text":""}]}. +The "output_count" must equal the length of the "texts" array in the output. From 80a60b23eb98a570e73d3296f95b1f8fa2477840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E7=A3=8A?= Date: Wed, 22 May 2024 20:12:15 +0800 Subject: [PATCH 24/24] Update TranslationService.cs add using --- .../BotSharp.Core/Translation/TranslationService.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index 7067eb7e..97e997c7 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -6,6 +6,7 @@ using BotSharp.Abstraction.Templating; using BotSharp.Abstraction.Translation.Models; using System.Collections; using System.Reflection; +using System.Text.Encodings.Web; namespace BotSharp.Core.Translation;