SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add pdf and image analysis

Browse source
This commit is contained in:
Jicheng Lu 2024-06-06 23:02:10 -05:00
parent c1687e8c45
commit c5c7dc97d7
6 changed files with 160 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
IEnumerable<MessageFileModel> GetChatImages(string conversationId, string source, List<RoleDialogModel> conversations, int? offset = null);
Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source, IEnumerable<string> fileTypes, List<RoleDialogModel> conversations, int? offset = null);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
bool HasConversationUserFiles(string conversationId);

10
src/Infrastructure/BotSharp.Abstraction/Files/Models/LlmFileContext.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Files.Models;
public class LlmFileContext
{
[JsonPropertyName("user_request")]
public string UserRequest { get; set; }
[JsonPropertyName("file_types")]
public string FileTypes { get; set; }
}

100
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs
View file

@ -8,12 +8,13 @@ namespace BotSharp.Core.Files;
public partial class BotSharpFileService
{
public IEnumerable<MessageFileModel> GetChatImages(string conversationId, string source, List<RoleDialogModel> conversations, int? offset = null)
public async Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source, IEnumerable<string> fileTypes,
List<RoleDialogModel> conversations, int? offset = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
{
return files;
return new List<MessageFileModel>();
}
if (offset <= 0)
@ -35,7 +36,98 @@ public partial class BotSharpFileService
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
files = GetMessageFiles(conversationId, messageIds, source, imageOnly: true).ToList();
files = await GetMessageFiles(conversationId, messageIds, source, fileTypes);
return files;
}
private async Task<List<MessageFileModel>> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string> fileTypes)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty() || fileTypes.IsNullOrEmpty()) return files;
var isNeedScreenShot = fileTypes.Any(x => _allowScreenShotTypes.Contains(x));
var onlyScreenShot = fileTypes.All(x => _allowScreenShotTypes.Contains(x));
try
{
var contextId = string.Empty;
var web = _services.GetRequiredService<IWebBrowser>();
var preFixPath = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
if (isNeedScreenShot)
{
contextId = Guid.NewGuid().ToString();
await web.LaunchBrowser(contextId, string.Empty);
}
foreach (var messageId in messageIds)
{
var dir = Path.Combine(preFixPath, messageId, source);
if (!ExistDirectory(dir)) continue;
foreach (var subDir in Directory.GetDirectories(dir))
{
var file = Directory.GetFiles(subDir).FirstOrDefault();
if (file == null) continue;
var index = subDir.Split(Path.DirectorySeparatorChar).Last();
var contentType = GetFileContentType(file);
if ((!isNeedScreenShot || (isNeedScreenShot && !onlyScreenShot)) && _allowedImageTypes.Contains(contentType))
{
var model = new MessageFileModel()
{
MessageId = messageId,
FileStorageUrl = file,
ContentType = contentType
};
files.Add(model);
}
else if ((isNeedScreenShot && !onlyScreenShot || onlyScreenShot) && !_allowedImageTypes.Contains(contentType))
{
var screenShotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenShotDir) && Directory.GetFiles(screenShotDir).Any())
{
file = Directory.GetFiles(screenShotDir).First();
contentType = GetFileContentType(file);
var model = new MessageFileModel()
{
MessageId = messageId,
FileStorageUrl = file,
ContentType = contentType
};
files.Add(model);
}
else
{
await web.GoToPage(contextId, file);
var path = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER, $"{Guid.NewGuid()}.png");
await web.ScreenshotAsync(contextId, path);
contentType = GetFileContentType(path);
var model = new MessageFileModel()
{
MessageId = messageId,
FileStorageUrl = path,
ContentType = contentType
};
files.Add(model);
}
}
}
}
if (isNeedScreenShot)
{
await web.CloseBrowser(contextId);
}
}
catch (Exception ex)
{
_logger.LogWarning($"Error when reading conversation ({conversationId}) files: {ex.Message}");
}
return files;
}
@ -102,7 +194,7 @@ public partial class BotSharpFileService
{
if (string.IsNullOrEmpty(conversationId)) return false;
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, USER_FILE_FOLDER);
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
if (!ExistDirectory(dir)) return false;
return Directory.GetDirectories(dir).Count() > 0;

3
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
View file

@ -13,11 +13,12 @@ public partial class BotSharpFileService : IBotSharpFileService
private readonly ILogger<BotSharpFileService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _allowedImageTypes = new List<string> { "image/png", "image/jpeg" };
private readonly IEnumerable<string> _allowScreenShotTypes = new List<string> { ".pdf" };
private readonly IEnumerable<string> _allowScreenShotTypes = new List<string> { "pdf" };
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
private const string USER_FILE_FOLDER = "user";
private const string SCREENSHOT_FILE_FOLDER = "screenshot";
private const string BOT_FILE_FOLDER = "bot";
private const string USERS_FOLDER = "users";
private const string USER_AVATAR_FOLDER = "avatar";

45
src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs
View file

@ -11,6 +11,7 @@ public class LoadAttachmentFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<LoadAttachmentFn> _logger;
private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
private readonly IEnumerable<string> _imageTypes = new List<string> { "image", "images", "png", "jpg", "jpeg" };
public LoadAttachmentFn(
IServiceProvider services,
@ -22,17 +23,19 @@ public class LoadAttachmentFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LlmFileContext>(message.FunctionArgs);
var conv = _services.GetRequiredService<IConversationService>();
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List<string>();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes);
var agent = await agentService.LoadAgent(AIAssistant);
var fileAgent = new Agent
{
Id = agent.Id,
Name = agent.Name,
Instruction = "Please describe the images.",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.",
TemplateDict = new Dictionary<string, object>()
};
@ -42,15 +45,16 @@ public class LoadAttachmentFn : IFunctionCallback
return true;
}
private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
private async Task<List<RoleDialogModel>> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs, List<string> fileTypes)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var parsedTypes = ParseFileTypes(fileTypes);
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var files = fileService.GetChatImages(conversationId, FileSourceType.User, dialogs);
var files = await fileService.GetChatImages(conversationId, FileSourceType.User, parsedTypes, dialogs);
foreach (var dialog in dialogs)
{
@ -59,7 +63,6 @@ public class LoadAttachmentFn : IFunctionCallback
dialog.Files = found.Select(x => new BotSharpFile
{
FileName = x.FileName,
ContentType = x.ContentType,
FileStorageUrl = x.FileStorageUrl
}).ToList();
@ -68,6 +71,38 @@ public class LoadAttachmentFn : IFunctionCallback
return dialogs;
}
private IEnumerable<string> ParseFileTypes(IEnumerable<string> fileTypes)
{
var imageType = "image";
var pdfType = "pdf";
var parsed = new List<string>();
if (fileTypes.IsNullOrEmpty())
{
return new List<string> { imageType };
}
foreach (var fileType in fileTypes)
{
var type = fileType?.Trim();
if (string.IsNullOrWhiteSpace(type) || _imageTypes.Any(x => type.IsEqualTo(x)))
{
parsed.Add(imageType);
}
else if (type.IsEqualTo("pdf"))
{
parsed.Add(pdfType);
}
}
if (parsed.IsNullOrEmpty())
{
parsed.Add(imageType);
}
return parsed.Distinct();
}
private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs)
{
try

16
src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs
View file

@ -24,27 +24,33 @@ public class AttachmentProcessingHook : AgentHookBase
{
var json = JsonSerializer.Serialize(new
{
user_question = new
user_request = new
{
type = "string",
description = $"The question asked by user, which is related to analyzing images or other files."
description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time."
},
file_types = new
{
type = "string",
description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'"
}
});
functions.Add(new FunctionDef
{
Name = "load_attachment",
Description = $"If the user's request is related to analyzing files, you can call this function to analyze files.",
Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.",
Parameters =
{
Properties = JsonSerializer.Deserialize<JsonDocument>(json),
Required = new List<string>
{
"user_question"
"user_request",
"file_types"
}
}
});
}
return true;
return base.OnFunctionsLoaded(functions); ;
}
}