using BotSharp.Abstraction.Functions; using BotSharp.Abstraction.MLTasks; namespace BotSharp.Core.Files.Functions; public class ReadFileFn : IFunctionCallback { public string Name => "read_file"; public string Indication => "Reading files"; private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly IEnumerable _imageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; private readonly IEnumerable _pdfTypes = new List { "pdf" }; private static string UTILITY_ASSISTANT = Guid.Empty.ToString(); public ReadFileFn( IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); var conv = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List(); var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); var agent = await agentService.LoadAgent(UTILITY_ASSISTANT); var fileAgent = new Agent { Id = agent?.Id ?? Guid.Empty.ToString(), Name = agent?.Name ?? "Unkown", Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.", TemplateDict = new Dictionary() }; var response = await GetChatCompletion(fileAgent, dialogs); message.Content = response; message.StopCompletion = true; return true; } private async Task> AssembleFiles(string conversationId, List dialogs, List fileTypes) { if (dialogs.IsNullOrEmpty()) { return new List(); } var parsedTypes = ParseFileTypes(fileTypes); var fileService = _services.GetRequiredService(); var files = await fileService.GetChatImages(conversationId, FileSourceType.User, parsedTypes, dialogs); foreach (var dialog in dialogs) { var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, FileStorageUrl = x.FileStorageUrl }).ToList(); } return dialogs; } private IEnumerable ParseFileTypes(IEnumerable fileTypes) { var imageType = "image"; var pdfType = "pdf"; var parsed = new List(); if (fileTypes.IsNullOrEmpty()) { return new List { imageType }; } foreach (var fileType in fileTypes) { var type = fileType?.Trim(); if (string.IsNullOrWhiteSpace(type) || _imageTypes.Any(x => type.IsEqualTo(x))) { parsed.Add(imageType); } else if (_pdfTypes.Any(x => type.IsEqualTo(x))) { parsed.Add(pdfType); } } if (parsed.IsNullOrEmpty()) { parsed.Add(imageType); } return parsed.Distinct(); } private async Task GetChatCompletion(Agent agent, List dialogs) { try { var llmProviderService = _services.GetRequiredService(); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; } catch (Exception ex) { var error = $"Error when analyzing files."; _logger.LogWarning($"{error} {ex.Message}"); return error; } } }