SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine getting message files

Browse source
This commit is contained in:
Jicheng Lu 2024-07-16 17:09:54 -05:00
parent 7c5807ccb5
commit db15157ea5
7 changed files with 157 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -3,7 +3,9 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source, IEnumerable<string> fileTypes, List<RoleDialogModel> conversations, int? offset = null);
Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
bool includeScreenShot = false, int? offset = null);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);

215
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs
View file

@ -1,124 +1,47 @@
using AspectInjector.Broker;
using Azure.Core;
using BotSharp.Abstraction.Files.Converters;
using Microsoft.EntityFrameworkCore;
using System.IO;
using System.Net.Mime;
using System.Threading;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source, IEnumerable<string> fileTypes,
List<RoleDialogModel> conversations, int? offset = null)
public async Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
bool includeScreenShot = false, int? offset = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
{
return new List<MessageFileModel>();
return files;
}
if (offset <= 0)
var messageIds = GetMessageIds(conversations, offset);
var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
foreach (var messageId in messageIds)
{
offset = MIN_OFFSET;
}
else if (offset > MAX_OFFSET)
{
offset = MAX_OFFSET;
}
var dir = Path.Combine(pathPrefix, messageId, source);
if (!ExistDirectory(dir)) continue;
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
files = await GetMessageFiles(conversationId, messageIds, source, fileTypes);
return files;
}
private async Task<List<MessageFileModel>> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string> fileTypes)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty() || fileTypes.IsNullOrEmpty()) return files;
var isNeedScreenShot = fileTypes.Any(x => _allowScreenShotTypes.Contains(x));
var onlyScreenShot = fileTypes.All(x => _allowScreenShotTypes.Contains(x));
try
{
var preFixPath = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
foreach (var messageId in messageIds)
foreach (var subDir in Directory.GetDirectories(dir))
{
var dir = Path.Combine(preFixPath, messageId, source);
if (!ExistDirectory(dir)) continue;
var file = Directory.GetFiles(subDir).FirstOrDefault();
if (file == null) continue;
foreach (var subDir in Directory.GetDirectories(dir))
{
var file = Directory.GetFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = GetFileContentType(file);
if (contentTypes?.Contains(contentType) != true) continue;
var index = subDir.Split(Path.DirectorySeparatorChar).Last();
var contentType = GetFileContentType(file);
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
if ((!isNeedScreenShot || (isNeedScreenShot && !onlyScreenShot)) && _allowedImageTypes.Contains(contentType))
{
var model = new MessageFileModel()
{
MessageId = messageId,
FileStorageUrl = file,
ContentType = contentType
};
files.Add(model);
}
else if ((isNeedScreenShot && !onlyScreenShot || onlyScreenShot) && !_allowedImageTypes.Contains(contentType))
{
var screenShotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenShotDir) && Directory.GetFiles(screenShotDir).Any())
{
foreach (var screenShot in Directory.GetFiles(screenShotDir))
{
contentType = GetFileContentType(screenShot);
if (!_allowedImageTypes.Contains(contentType)) continue;
var model = new MessageFileModel()
{
MessageId = messageId,
FileStorageUrl = screenShot,
ContentType = contentType
};
files.Add(model);
}
}
else
{
var screenShotPath = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER);
var images = await ConvertPdfToImages(file, screenShotPath);
foreach (var image in images)
{
contentType = GetFileContentType(image);
var model = new MessageFileModel()
{
MessageId = messageId,
FileStorageUrl = image,
ContentType = contentType
};
files.Add(model);
}
}
}
}
files.AddRange(foundFiles);
}
}
catch (Exception ex)
{
_logger.LogWarning($"Error when reading conversation ({conversationId}) files: {ex.Message}\r\n{ex.InnerException}\r\n{ex.StackTrace}");
}
return files;
}
@ -144,7 +67,7 @@ public partial class BotSharpFileService
foreach (var file in Directory.GetFiles(subDir))
{
var contentType = GetFileContentType(file);
if (imageOnly && !_allowedImageTypes.Contains(contentType))
if (imageOnly && !_imageTypes.Contains(contentType))
{
continue;
}
@ -327,6 +250,102 @@ public partial class BotSharpFileService
return dir;
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> conversations, int? offset = null)
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset <= 0)
{
offset = MIN_OFFSET;
}
else if (offset > MAX_OFFSET)
{
offset = MAX_OFFSET;
}
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
return messageIds;
}
private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, string fileDir, string contentType,
string messageId, string source, bool includeScreenShot)
{
var files = new List<MessageFileModel>();
try
{
if (!_imageTypes.Contains(contentType) && includeScreenShot)
{
var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty())
{
foreach (var screenShot in Directory.GetFiles(screenShotDir))
{
contentType = GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = Path.GetFileName(screenShot),
FileStorageUrl = screenShot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = GetFileContentType(image);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = Path.GetFileName(image),
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
}
else
{
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = Path.GetFileName(file),
FileStorageUrl = file,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();

8
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs
View file

@ -1,5 +1,6 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
using System.Net.Mime;
namespace BotSharp.Core.Files.Services;
@ -10,8 +11,11 @@ public partial class BotSharpFileService : IBotSharpFileService
private readonly IUserIdentity _user;
private readonly ILogger<BotSharpFileService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _allowedImageTypes = new List<string> { "image/png", "image/jpeg" };
private readonly IEnumerable<string> _allowScreenShotTypes = new List<string> { "pdf" };
private readonly IEnumerable<string> _imageTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
};
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";

12
src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadImageFn.cs
View file

@ -5,6 +5,7 @@ using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.MLTasks;
using Microsoft.Extensions.Logging;
using BotSharp.Core.Infrastructures;
using System.Net.Mime;
namespace BotSharp.Plugin.FileHandler.Functions;
@ -16,8 +17,11 @@ public class ReadImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<ReadImageFn> _logger;
private const string DEFAULT_IMAGE = "image";
private readonly IEnumerable<string> _targetImageTypes = new List<string> { "image", "images", "png", "jpg", "jpeg" };
private readonly IEnumerable<string> _imageContentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg,
};
public ReadImageFn(
IServiceProvider services,
@ -57,7 +61,7 @@ public class ReadImageFn : IFunctionCallback
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var images = await fileService.GetChatImages(conversationId, FileSourceType.User, new List<string> { "image" }, dialogs);
var images = await fileService.GetChatImages(conversationId, FileSourceType.User, dialogs, _imageContentTypes);
foreach (var dialog in dialogs)
{
@ -88,7 +92,7 @@ public class ReadImageFn : IFunctionCallback
catch (Exception ex)
{
var error = $"Error when analyzing images.";
_logger.LogWarning($"{error} {ex.Message}");
_logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
return error;
}
}

10
src/Plugins/BotSharp.Plugin.FileReader/Functions/ReadPdfFn.cs
View file

@ -5,6 +5,7 @@ using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Core.Infrastructures;
using Microsoft.Extensions.Logging;
using System.Net.Mime;
namespace BotSharp.Plugin.FileHandler.Functions;
@ -16,7 +17,10 @@ public class ReadPdfFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<ReadPdfFn> _logger;
private const string DEFAULT_PDF = "pdf";
private readonly IEnumerable<string> _pdfContentTypes = new List<string>
{
MediaTypeNames.Application.Pdf
};
public ReadPdfFn(
IServiceProvider services,
@ -56,7 +60,7 @@ public class ReadPdfFn : IFunctionCallback
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var files = await fileService.GetChatImages(conversationId, FileSourceType.User, new List<string> { "pdf" }, dialogs);
var files = await fileService.GetChatImages(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true);
foreach (var dialog in dialogs)
{
@ -87,7 +91,7 @@ public class ReadPdfFn : IFunctionCallback
catch (Exception ex)
{
var error = $"Error when analyzing pdf file(s).";
_logger.LogWarning($"{error} {ex.Message}");
_logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
return error;
}
}

28
src/Plugins/BotSharp.Plugin.FileReader/Hooks/FileHandlerHook.cs
View file

@ -30,20 +30,6 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
base.OnAgentLoaded(agent);
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
private void AddUtility(Agent agent, string utility, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
@ -66,4 +52,18 @@ public class FileHandlerHook : AgentHookBase, IAgentHook
}
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, loadAttachmentFn);
}
}

3
src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs
View file

@ -1,4 +1,5 @@
using System.Net.Http;
using System.Net.Mime;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
@ -150,7 +151,7 @@ public class HandleHttpRequestFn : IFunctionCallback
_logger.LogWarning($"Error when build http content: {content}\n(Error: {ex.Message})");
}
return new StringContent(str, Encoding.UTF8, "application/json");
return new StringContent(str, Encoding.UTF8, MediaTypeNames.Application.Json);
}
private string BuildQuery(string url, string? content)