refine json format

refine file select prompt

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -17,6 +17,11 @@ public class LlmModelSetting
     /// </summary>
     public string Version { get; set; } = "1106-Preview";
 
+    /// <summary>
+    /// Api version
+    /// </summary>
+    public string? ApiVersion { get; set; }
+
     /// <summary>
     /// Deployment same functional model in a group.
     /// It can be used to deploy same model in different regions.

src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs

@@ -0,0 +1,41 @@
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json;
+
+namespace BotSharp.Abstraction.Utilities;
+
+public static class JsonExtensions
+{
+    public static string FormatJson(this string? json, Formatting format = Formatting.Indented)
+    {
+        if (string.IsNullOrWhiteSpace(json))
+        {
+            return "{}";
+        }
+
+        try
+        {
+            var parsedJson = JObject.Parse(json);
+            foreach (var item in parsedJson)
+            {
+                try
+                {
+                    var key = item.Key;
+                    var value = parsedJson[key].ToString();
+                    var parsedValue = JObject.Parse(value);
+                    parsedJson[key] = parsedValue;
+                }
+                catch { continue; }
+            }
+
+            var jsonSettings = new JsonSerializerSettings
+            {
+                Formatting = format
+            };
+            return JsonConvert.SerializeObject(parsedJson, jsonSettings);
+        }
+        catch
+        {
+            return json;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs

@@ -132,11 +132,11 @@ public class TextCompletionProvider : ITextCompletion
 
     private string BuildApiUrl(LlmModelSetting modelSetting)
     {
-        var url = string.Empty;
+        var apiVersion = !string.IsNullOrWhiteSpace(modelSetting.ApiVersion) ? modelSetting.ApiVersion : "2023-09-15-preview";
         var endpoint = modelSetting.Endpoint.EndsWith("/") ?
             modelSetting.Endpoint.Substring(0, modelSetting.Endpoint.Length - 1) : modelSetting.Endpoint;
 
-        url = $"{endpoint}/openai/deployments/{_model}/completions?api-version=2023-09-15-preview";
+        var url = $"{endpoint}/openai/deployments/{_model}/completions?api-version={apiVersion}";
         return url;
     }
 

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs

@@ -118,7 +118,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
 
         var agent = await _agentService.LoadAgent(message.CurrentAgentId);
         message.FunctionArgs = message.FunctionArgs ?? "{}";
-        var args = FormatJson(message.FunctionArgs);
+        var args = message.FunctionArgs.FormatJson();
         var log = $"{message.FunctionName} <u>executing</u>\r\n```json\r\n{args}\r\n```";
 
         var input = new ContentLogInputModel(conversationId, message)
@@ -559,39 +559,5 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
 
         return localOptions;
     }
-
-    private string FormatJson(string? json)
-    {
-        if (string.IsNullOrWhiteSpace(json))
-        {
-            return "{}";
-        }
-
-        try
-        {
-            var parsedJson = JObject.Parse(json);
-            foreach (var item in parsedJson)
-            {
-                try
-                {
-                    var key = item.Key;
-                    var value = parsedJson[key].ToString();
-                    var parsedValue = JObject.Parse(value);
-                    parsedJson[key] = parsedValue;
-                }
-                catch { continue; }
-            }
-
-            var jsonSettings = new JsonSerializerSettings
-            {
-                Formatting = Newtonsoft.Json.Formatting.Indented
-            };
-            return JsonConvert.SerializeObject(parsedJson, jsonSettings);
-        }
-        catch
-        {
-            return json;
-        }
-    }
     #endregion
 }

src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs

@@ -78,8 +78,9 @@ public class HandleEmailRequestFn : IFunctionCallback
         var conversationId = convService.ConversationId;
         var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
         var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
-        var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
-        return await SelectFiles(files, dialogs);
+        var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
+        var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot);
+        return await SelectFiles(userFiles.Concat(botFiles), dialogs);
     }
 
     private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
@@ -94,7 +95,7 @@ public class HandleEmailRequestFn : IFunctionCallback
         {
             var promptFiles = files.Select((x, idx) =>
             {
-                return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}";
+                return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, author: {x.FileSource}, content_type: {x.ContentType}";
             }).ToList();
             var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt");
             prompt = render.Render(prompt, new Dictionary<string, object>
@@ -114,7 +115,8 @@ public class HandleEmailRequestFn : IFunctionCallback
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
             var response = await completion.GetChatCompletions(agent, dialogs);
             var content = response?.Content ?? string.Empty;
-            var fids = JsonSerializer.Deserialize<List<int>>(content) ?? new List<int>();
+            var selecteds = JsonSerializer.Deserialize<LlmContextOut>(content);
+            var fids = selecteds?.Selecteds ?? new List<int>();
             return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
         }
         catch (Exception ex)
@@ -146,6 +148,6 @@ public class HandleEmailRequestFn : IFunctionCallback
         await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls);
         await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password);
         var response = await smtpClient.SendAsync(mailMessage);
-        return response;
+        return response ?? "Email sent";
     }
 }

src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.EmailHandler.LlmContexts;
+
+public class LlmContextOut
+{
+    [JsonPropertyName("selected_ids")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<int> Selecteds { get; set; }
+}

src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid

@@ -1,2 +1,2 @@
-Please call handle_email_request if user wants to send out an email.
-** Please take a look at the conversation, and decide whether user wants to send email with attachments or not.
+Please call handle_email_request if user wants to send email.
+** Please take a look at the conversation and decide whether user wants to send email with files/attachments/images or not.

src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid

@@ -1,18 +1,42 @@
 Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
-Your response must be a list of file ids.
-** Please only output the list. Do not prepend or append anything.
 
-For example:
-Suppose there are three files:
+** Ensure the output is only in JSON format without any additional text.
+** If no files are selected, you must output an empty list [].
+** You may need to look at the file_name as a reference to find the correct file id.
 
-id: 1, file_name: example_file.png, content_type: image/png
-id: 2, file_name: example_file.jpeg, content_type: image/jpeg
-id: 3, file_name: example_file.pdf, content_type: application/pdf
+Here is the JSON format to use:
+{
+    "selected_ids": a list of id selected from the [FILES] section
+}
 
-If user wants the first file and the third file, the ouput should be [1, 3].
-If user wants the all the images, the output should be [1, 2].
-If user wants the pdf file, the output should be [3].
-If user does not want any files, the ouput should be [];
+Suppose there are 4 files:
+
+id: 1, file_name: example_file.jpg, author: user, content_type: image/jpeg
+id: 2, file_name: example_file.pdf, author: user, content_type: application/pdf
+id: 3, file_name: example_file.png, author: bot, content_type: image/png
+id: 4, file_name: example_file.png, author: bot, content_type: image/png
+
+=====
+Example 1:
+USER: I want to send the first file and the third file.
+OUTPUT: { "selected_ids": [1, 3] }
+
+Example 2:
+USER: Send all the images.
+OUTPUT: { "selected_ids": [1, 2, 4] }
+
+Example 3:
+USER: Send all the images I uploaded.
+OUTPUT: { "selected_ids": [1] }
+
+Example 4:
+USER: Send the image and the pdf file.
+OUTPUT: { "selected_ids": [1, 2] }
+
+Example 5:
+USER: Send the images generated by bot
+OUTPUT: { "selected_ids": [3, 4] }
+=====
 
 [FILES]
 {% for file in file_list -%}

src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs

@@ -28,7 +28,7 @@ public class EditImageFn : IFunctionCallback
         Init(message);
         SetImageOptions();
 
-        var image = await SelectConversationImage();
+        var image = await SelectConversationImage(descrpition);
         var response = await GetImageEditGeneration(message, descrpition, image);
         message.Content = response;
         return true;
@@ -48,17 +48,17 @@ public class EditImageFn : IFunctionCallback
         state.SetState("image_count", "1");
     }
 
-    private async Task<MessageFileModel?> SelectConversationImage()
+    private async Task<MessageFileModel?> SelectConversationImage(string? description)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         var fileService = _services.GetRequiredService<IBotSharpFileService>();
-        var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
+        var dialogs = convService.GetDialogHistory();
         var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
-        var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
-        return await SelectImage(images, dialogs);
+        var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
+        return await SelectImage(userImages, dialogs.LastOrDefault(), description);
     }
 
-    private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, List<RoleDialogModel> dialogs)
+    private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, RoleDialogModel message, string? description)
     {
         if (images.IsNullOrEmpty()) return null;
 
@@ -91,10 +91,15 @@ public class EditImageFn : IFunctionCallback
             var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
             var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
-            var response = await completion.GetChatCompletions(agent, dialogs);
+
+            var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
+            var dialog = RoleDialogModel.From(message, AgentRole.User, text);
+
+            var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { dialog });
             var content = response?.Content ?? string.Empty;
-            var fid = JsonSerializer.Deserialize<int?>(content);
-            return images.Where((x, idx) => idx == fid - 1).FirstOrDefault();
+            var selected = JsonSerializer.Deserialize<LlmContextOut>(content);
+            var fid = selected?.Selected ?? -1;
+            return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null;
         }
         catch (Exception ex)
         {
@@ -126,7 +131,7 @@ public class EditImageFn : IFunctionCallback
             stream.Close();
             SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
 
-            return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed.";
+            return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted.";
         }
         catch (Exception ex)
         {

src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.FileHandler.LlmContexts;
+
+public class LlmContextOut
+{
+    [JsonPropertyName("selected_id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? Selected { get; set; }
+}

src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid

@@ -1,30 +1,38 @@
 Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user.
 Your response must be an interger number.
-** Please ONLY output the interger number. Do not prepend or append anything else.
-** If you think user requests multiple images. Please ONLY select the first image and output its id.
+
+** Ensure the output is only in JSON format without any additional text.
+** You may need to look at the image_name as a reference to find the correct image id.
+
+Here is the JSON format to use:
+{
+    "selected_id": the id selected from the [IMAGES] section
+}
+
 
 Suppose there are three images:
 
 id: 1, image_name: example_image_a.png
 id: 2, image_name: example_image_b.png
 id: 3, image_name: example_image_c.png
+id: 4, image_name: example_image_d.png
 
 =====
 Example 1:
 USER: I want to add a dog in the first file.
-OUTPUT: 1
+OUTPUT: { "selected_id": 1 }
 
 Example 2:
 USER: Add a coffee cup in the second image I uploaded.
-OUTPUT: 2
+OUTPUT: { "selected_id": 2 }
 
 Example 3:
 USER: Please remove the left tree in the third and the first images.
-OUTPUT: 3
+OUTPUT: { "selected_id": 3 }
 
 Example 4:
-USER: Add a boat in the images.
-OUTPUT: 1
+USER: Circle the head of the dog in example_image_b.png.
+OUTPUT: { "selected_id": 4 }
 =====