From bc76a49f52a10d5099b033b4c55ea6623b2cda1a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 24 Jul 2024 17:54:49 -0500 Subject: [PATCH] refine json format refine file select prompt --- .../MLTasks/Settings/LlmModelSetting.cs | 5 ++ .../Utilities/JsonExtensions.cs | 41 +++++++++++++++++ .../Providers/Text/TextCompletionProvider.cs | 4 +- .../Hooks/StreamingLogHook.cs | 36 +-------------- .../Functions/HandleEmailRequestFn.cs | 12 +++-- .../LlmContexts/LlmContextOut.cs | 10 ++++ .../templates/handle_email_request.fn.liquid | 4 +- .../templates/select_attachment_prompt.liquid | 46 ++++++++++++++----- .../Functions/EditImageFn.cs | 25 ++++++---- .../LlmContexts/LlmContextOut.cs | 10 ++++ .../templates/select_edit_image_prompt.liquid | 22 ++++++--- 11 files changed, 143 insertions(+), 72 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs create mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs create mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 2247c212..16e19a6f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -17,6 +17,11 @@ public class LlmModelSetting /// public string Version { get; set; } = "1106-Preview"; + /// + /// Api version + /// + public string? ApiVersion { get; set; } + /// /// Deployment same functional model in a group. /// It can be used to deploy same model in different regions. diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs new file mode 100644 index 00000000..2cc56012 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs @@ -0,0 +1,41 @@ +using Newtonsoft.Json.Linq; +using Newtonsoft.Json; + +namespace BotSharp.Abstraction.Utilities; + +public static class JsonExtensions +{ + public static string FormatJson(this string? json, Formatting format = Formatting.Indented) + { + if (string.IsNullOrWhiteSpace(json)) + { + return "{}"; + } + + try + { + var parsedJson = JObject.Parse(json); + foreach (var item in parsedJson) + { + try + { + var key = item.Key; + var value = parsedJson[key].ToString(); + var parsedValue = JObject.Parse(value); + parsedJson[key] = parsedValue; + } + catch { continue; } + } + + var jsonSettings = new JsonSerializerSettings + { + Formatting = format + }; + return JsonConvert.SerializeObject(parsedJson, jsonSettings); + } + catch + { + return json; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index ff817b25..4953e114 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -132,11 +132,11 @@ public class TextCompletionProvider : ITextCompletion private string BuildApiUrl(LlmModelSetting modelSetting) { - var url = string.Empty; + var apiVersion = !string.IsNullOrWhiteSpace(modelSetting.ApiVersion) ? modelSetting.ApiVersion : "2023-09-15-preview"; var endpoint = modelSetting.Endpoint.EndsWith("/") ? modelSetting.Endpoint.Substring(0, modelSetting.Endpoint.Length - 1) : modelSetting.Endpoint; - url = $"{endpoint}/openai/deployments/{_model}/completions?api-version=2023-09-15-preview"; + var url = $"{endpoint}/openai/deployments/{_model}/completions?api-version={apiVersion}"; return url; } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 0b40ede1..99bb1937 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -118,7 +118,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR var agent = await _agentService.LoadAgent(message.CurrentAgentId); message.FunctionArgs = message.FunctionArgs ?? "{}"; - var args = FormatJson(message.FunctionArgs); + var args = message.FunctionArgs.FormatJson(); var log = $"{message.FunctionName} executing\r\n```json\r\n{args}\r\n```"; var input = new ContentLogInputModel(conversationId, message) @@ -559,39 +559,5 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR return localOptions; } - - private string FormatJson(string? json) - { - if (string.IsNullOrWhiteSpace(json)) - { - return "{}"; - } - - try - { - var parsedJson = JObject.Parse(json); - foreach (var item in parsedJson) - { - try - { - var key = item.Key; - var value = parsedJson[key].ToString(); - var parsedValue = JObject.Parse(value); - parsedJson[key] = parsedValue; - } - catch { continue; } - } - - var jsonSettings = new JsonSerializerSettings - { - Formatting = Newtonsoft.Json.Formatting.Indented - }; - return JsonConvert.SerializeObject(parsedJson, jsonSettings); - } - catch - { - return json; - } - } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs index 525eae8b..84be6e16 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -78,8 +78,9 @@ public class HandleEmailRequestFn : IFunctionCallback var conversationId = convService.ConversationId; var dialogs = convService.GetDialogHistory(fromBreakpoint: false); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); - return await SelectFiles(files, dialogs); + var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); + var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot); + return await SelectFiles(userFiles.Concat(botFiles), dialogs); } private async Task> SelectFiles(IEnumerable files, List dialogs) @@ -94,7 +95,7 @@ public class HandleEmailRequestFn : IFunctionCallback { var promptFiles = files.Select((x, idx) => { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}"; + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, author: {x.FileSource}, content_type: {x.ContentType}"; }).ToList(); var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt"); prompt = render.Render(prompt, new Dictionary @@ -114,7 +115,8 @@ public class HandleEmailRequestFn : IFunctionCallback var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); var response = await completion.GetChatCompletions(agent, dialogs); var content = response?.Content ?? string.Empty; - var fids = JsonSerializer.Deserialize>(content) ?? new List(); + var selecteds = JsonSerializer.Deserialize(content); + var fids = selecteds?.Selecteds ?? new List(); return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); } catch (Exception ex) @@ -146,6 +148,6 @@ public class HandleEmailRequestFn : IFunctionCallback await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password); var response = await smtpClient.SendAsync(mailMessage); - return response; + return response ?? "Email sent"; } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 00000000..5371a181 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.EmailHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_ids")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable Selecteds { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid index 0ad3e377..9091f6bf 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid @@ -1,2 +1,2 @@ -Please call handle_email_request if user wants to send out an email. -** Please take a look at the conversation, and decide whether user wants to send email with attachments or not. \ No newline at end of file +Please call handle_email_request if user wants to send email. +** Please take a look at the conversation and decide whether user wants to send email with files/attachments/images or not. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid index 0b1b87c3..f96e286a 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid @@ -1,18 +1,42 @@ Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. -Your response must be a list of file ids. -** Please only output the list. Do not prepend or append anything. -For example: -Suppose there are three files: +** Ensure the output is only in JSON format without any additional text. +** If no files are selected, you must output an empty list []. +** You may need to look at the file_name as a reference to find the correct file id. -id: 1, file_name: example_file.png, content_type: image/png -id: 2, file_name: example_file.jpeg, content_type: image/jpeg -id: 3, file_name: example_file.pdf, content_type: application/pdf +Here is the JSON format to use: +{ + "selected_ids": a list of id selected from the [FILES] section +} -If user wants the first file and the third file, the ouput should be [1, 3]. -If user wants the all the images, the output should be [1, 2]. -If user wants the pdf file, the output should be [3]. -If user does not want any files, the ouput should be []; +Suppose there are 4 files: + +id: 1, file_name: example_file.jpg, author: user, content_type: image/jpeg +id: 2, file_name: example_file.pdf, author: user, content_type: application/pdf +id: 3, file_name: example_file.png, author: bot, content_type: image/png +id: 4, file_name: example_file.png, author: bot, content_type: image/png + +===== +Example 1: +USER: I want to send the first file and the third file. +OUTPUT: { "selected_ids": [1, 3] } + +Example 2: +USER: Send all the images. +OUTPUT: { "selected_ids": [1, 2, 4] } + +Example 3: +USER: Send all the images I uploaded. +OUTPUT: { "selected_ids": [1] } + +Example 4: +USER: Send the image and the pdf file. +OUTPUT: { "selected_ids": [1, 2] } + +Example 5: +USER: Send the images generated by bot +OUTPUT: { "selected_ids": [3, 4] } +===== [FILES] {% for file in file_list -%} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 37adf4fb..63938c5e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -28,7 +28,7 @@ public class EditImageFn : IFunctionCallback Init(message); SetImageOptions(); - var image = await SelectConversationImage(); + var image = await SelectConversationImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; return true; @@ -48,17 +48,17 @@ public class EditImageFn : IFunctionCallback state.SetState("image_count", "1"); } - private async Task SelectConversationImage() + private async Task SelectConversationImage(string? description) { var convService = _services.GetRequiredService(); var fileService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: false); + var dialogs = convService.GetDialogHistory(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); - return await SelectImage(images, dialogs); + var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); + return await SelectImage(userImages, dialogs.LastOrDefault(), description); } - private async Task SelectImage(IEnumerable images, List dialogs) + private async Task SelectImage(IEnumerable images, RoleDialogModel message, string? description) { if (images.IsNullOrEmpty()) return null; @@ -91,10 +91,15 @@ public class EditImageFn : IFunctionCallback var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var response = await completion.GetChatCompletions(agent, dialogs); + + var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; + var dialog = RoleDialogModel.From(message, AgentRole.User, text); + + var response = await completion.GetChatCompletions(agent, new List { dialog }); var content = response?.Content ?? string.Empty; - var fid = JsonSerializer.Deserialize(content); - return images.Where((x, idx) => idx == fid - 1).FirstOrDefault(); + var selected = JsonSerializer.Deserialize(content); + var fid = selected?.Selected ?? -1; + return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null; } catch (Exception ex) { @@ -126,7 +131,7 @@ public class EditImageFn : IFunctionCallback stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed."; + return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 00000000..ee7d40dc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.FileHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Selected { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid index 2af9bf43..4d050563 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid @@ -1,30 +1,38 @@ Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. Your response must be an interger number. -** Please ONLY output the interger number. Do not prepend or append anything else. -** If you think user requests multiple images. Please ONLY select the first image and output its id. + +** Ensure the output is only in JSON format without any additional text. +** You may need to look at the image_name as a reference to find the correct image id. + +Here is the JSON format to use: +{ + "selected_id": the id selected from the [IMAGES] section +} + Suppose there are three images: id: 1, image_name: example_image_a.png id: 2, image_name: example_image_b.png id: 3, image_name: example_image_c.png +id: 4, image_name: example_image_d.png ===== Example 1: USER: I want to add a dog in the first file. -OUTPUT: 1 +OUTPUT: { "selected_id": 1 } Example 2: USER: Add a coffee cup in the second image I uploaded. -OUTPUT: 2 +OUTPUT: { "selected_id": 2 } Example 3: USER: Please remove the left tree in the third and the first images. -OUTPUT: 3 +OUTPUT: { "selected_id": 3 } Example 4: -USER: Add a boat in the images. -OUTPUT: 1 +USER: Circle the head of the dog in example_image_b.png. +OUTPUT: { "selected_id": 4 } =====