SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

clean file select prompt

Browse source
This commit is contained in:
Jicheng Lu 2024-08-07 17:40:39 -05:00
parent 0d8532c3c7
commit 2a10294d70
10 changed files with 43 additions and 143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs
View file

@ -17,7 +17,7 @@ public interface IFileBasicService
/// <param name="offset"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string>? contentTypes,
bool includeScreenShot = false, int? offset = null);
/// <summary>

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -39,6 +39,6 @@ public class MessageFileModel
public override string ToString()
{
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}";
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}";
}
}

9
src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs
View file

@ -7,7 +7,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileBasicService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
{
var files = new List<MessageFileModel>();
@ -30,7 +30,10 @@ public partial class FileBasicService
if (file == null) continue;
var contentType = FileUtility.GetFileContentType(file);
if (contentTypes?.Contains(contentType) != true) continue;
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
@ -63,7 +66,7 @@ public partial class FileBasicService
foreach (var file in Directory.GetFiles(subDir))
{
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType))
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}

29
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid
View file

@ -9,6 +9,35 @@ Here is the JSON format to use:
"selected_ids": a list of id selected from the [FILES] section
}
Suppose there are four files:
id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user
id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user
id: 3, file_name: example_file.png, content_type: image/png, author: bot
id: 4, file_name: example_file.png, content_type: image/png, author: bot
=====
Example 1:
USER: I want to send the first file and the third file.
OUTPUT: { "selected_ids": [1, 3] }
Example 2:
USER: Send all the images.
OUTPUT: { "selected_ids": [1, 2, 4] }
Example 3:
USER: Send all the images I uploaded.
OUTPUT: { "selected_ids": [1] }
Example 4:
USER: Send the image and the pdf file.
OUTPUT: { "selected_ids": [1, 2] }
Example 5:
USER: Send the images generated by bot
OUTPUT: { "selected_ids": [3, 4] }
=====
[FILES]
{% for file in file_list -%}
{{ file }}{{ "\r\n" }}

3
src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj
View file

@ -28,9 +28,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_sender.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_attachment_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_reader.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

44
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -81,50 +81,6 @@ public class HandleEmailSenderFn : IFunctionCallback
return selecteds;
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}";
}).ToList();
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt");
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var latest = dialogs.LastOrDefault();
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { latest });
var content = response?.Content ?? string.Empty;
var selecteds = JsonSerializer.Deserialize<LlmContextOut>(content);
var fids = selecteds?.Selecteds ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}");
return new List<MessageFileModel>();
}
}
private void BuildEmailAttachments(BodyBuilder builder, IEnumerable<MessageFileModel> files)
{
if (files.IsNullOrEmpty()) return;

44
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid
View file

@ -1,44 +0,0 @@
Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
** Ensure the output is only in JSON format without any additional text.
** If no files are selected, you must output an empty list [].
** You may need to look at the file_name as a reference to find the correct file id.
Here is the JSON format to use:
{
"selected_ids": a list of id selected from the [FILES] section
}
Suppose there are four files:
id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user
id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user
id: 3, file_name: example_file.png, content_type: image/png, author: bot
id: 4, file_name: example_file.png, content_type: image/png, author: bot
=====
Example 1:
USER: I want to send the first file and the third file.
OUTPUT: { "selected_ids": [1, 3] }
Example 2:
USER: Send all the images.
OUTPUT: { "selected_ids": [1, 2, 4] }
Example 3:
USER: Send all the images I uploaded.
OUTPUT: { "selected_ids": [1] }
Example 4:
USER: Send the image and the pdf file.
OUTPUT: { "selected_ids": [1, 2] }
Example 5:
USER: Send the images generated by bot
OUTPUT: { "selected_ids": [3, 4] }
=====
[FILES]
{% for file in file_list -%}
{{ file }}{{ "\r\n" }}
{%- endfor %}

3
src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj
View file

@ -47,9 +47,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\edit_image.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>

41
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid
View file

@ -1,41 +0,0 @@
Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user.
** Ensure the output is only in JSON format without any additional text.
** You may need to look at the image_name as a reference to find the correct image id.
Here is the JSON format to use:
{
"selected_id": the id selected from the [IMAGES] section
}
Suppose there are four images:
id: 1, image_name: example_image_a.png
id: 2, image_name: example_image_b.png
id: 3, image_name: example_image_c.png
id: 4, image_name: example_image_d.png
=====
Example 1:
USER: I want to add a dog in the first file.
OUTPUT: { "selected_id": 1 }
Example 2:
USER: Add a coffee cup in the second image I uploaded.
OUTPUT: { "selected_id": 2 }
Example 3:
USER: Please remove the left tree in the third and the first images.
OUTPUT: { "selected_id": 3 }
Example 4:
USER: Circle the head of the dog in example_image_b.png.
OUTPUT: { "selected_id": 4 }
=====
[IMAGES]
{% for image in image_list -%}
{{ image }}{{ "\r\n" }}
{%- endfor %}

9
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
{
var files = new List<MessageFileModel>();
@ -30,7 +30,10 @@ public partial class TencentCosService
if (file == null) continue;
var contentType = FileUtility.GetFileContentType(file);
if (contentTypes?.Contains(contentType) != true) continue;
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
@ -61,7 +64,7 @@ public partial class TencentCosService
foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir))
{
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType))
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}