From 3a4fae9be03c93c9de6239dc30b5edc12c9a8aad Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 1 Jul 2024 14:31:48 -0500 Subject: [PATCH] add pdf completion --- .../Files/IBotSharpFileService.cs | 8 + .../BotSharp.Core/BotSharp.Core.csproj | 1 + .../BotSharp.Core/Files/FilePlugin.cs | 1 + .../BotSharpFileService.Conversation.cs | 4 +- .../Files/Services/BotSharpFileService.Pdf.cs | 146 ++++++++++++++++++ .../BotSharpFileService.User.cs | 2 +- .../{ => Services}/BotSharpFileService.cs | 5 +- .../Controllers/InstructModeController.cs | 23 +++ .../Instructs/PdfCompletionViewModel.cs | 13 ++ 9 files changed, 196 insertions(+), 7 deletions(-) rename src/Infrastructure/BotSharp.Core/Files/{ => Services}/BotSharpFileService.Conversation.cs (99%) create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs rename src/Infrastructure/BotSharp.Core/Files/{ => Services}/BotSharpFileService.User.cs (97%) rename src/Infrastructure/BotSharp.Core/Files/{ => Services}/BotSharpFileService.cs (96%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs index ea703bd0..14d22ce2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs @@ -23,6 +23,14 @@ public interface IBotSharpFileService bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null); bool DeleteConversationFiles(IEnumerable conversationIds); + /// + /// Take screenshots of pdf pages and get response from llm + /// + /// + /// Pdf files + /// + Task AnalyzePdf(string? provider, string? model, string? modelId, string prompt, List files); + /// /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" /// diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index a78dd36b..558663ef 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -180,6 +180,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index 3110c8d5..d6c08b72 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Files.Hooks; +using BotSharp.Core.Files.Services; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Files; diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs similarity index 99% rename from src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs index ec99764b..894e603f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs @@ -1,11 +1,9 @@ using BotSharp.Abstraction.Files.Converters; using Microsoft.EntityFrameworkCore; -using System; using System.IO; -using System.Linq; using System.Threading; -namespace BotSharp.Core.Files; +namespace BotSharp.Core.Files.Services; public partial class BotSharpFileService { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs new file mode 100644 index 00000000..4b51a6e6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs @@ -0,0 +1,146 @@ +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; + +namespace BotSharp.Core.Files.Services; + +public partial class BotSharpFileService +{ + public async Task AnalyzePdf(string? provider, string? model, string? modelId, string prompt, List files) + { + var content = string.Empty; + + if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) + { + return content; + } + + var guid = Guid.NewGuid().ToString(); + var sessionDir = GetSessionDirectory(guid); + if (!ExistDirectory(sessionDir)) + { + Directory.CreateDirectory(sessionDir); + } + + try + { + var pdfFiles = await SaveFiles(sessionDir, files); + var images = await ConvertPdfToImages(pdfFiles); + if (images.IsNullOrEmpty()) return content; + + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", + model: model, modelId: modelId ?? "gpt-4", multiModal: true); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, prompt) + { + Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() + } + }); + + content = message.Content; + return content; + } + catch (Exception ex) + { + _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}"); + return content; + } + finally + { + Directory.Delete(sessionDir, true); + } + } + + #region Private methods + private string GetSessionDirectory(string id) + { + var dir = Path.Combine(_baseDir, SESSION_FOLDER, id); + return dir; + } + + private async Task> SaveFiles(string dir, List files, string extension = "pdf") + { + if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + var locs = new List(); + foreach (var file in files) + { + try + { + var bytes = new byte[0]; + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + bytes = await client.GetByteArrayAsync(file.FileUrl); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + (_, bytes) = GetFileInfoFromData(file.FileData); + } + + if (!bytes.IsNullOrEmpty()) + { + var guid = Guid.NewGuid().ToString(); + var fileDir = Path.Combine(dir, guid); + if (!ExistDirectory(fileDir)) + { + Directory.CreateDirectory(fileDir); + } + + var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}"); + using (var fs = new FileStream(pdfDir, FileMode.Create)) + { + fs.Write(bytes, 0, bytes.Length); + fs.Close(); + locs.Add(pdfDir); + Thread.Sleep(100); + } + } + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving pdf file: {ex.Message}"); + continue; + } + } + return locs; + } + + private async Task> ConvertPdfToImages(IEnumerable files) + { + var images = new List(); + var converter = GetPdf2ImageConverter(); + if (converter == null || files.IsNullOrEmpty()) + { + return images; + } + + foreach (var file in files) + { + try + { + var segs = file.Split(Path.DirectorySeparatorChar); + var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); + var folder = Path.Combine(dir, "screenshots"); + var urls = await converter.ConvertPdfToImages(file, folder); + images.AddRange(urls); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}"); + continue; + } + } + return images; + } + #endregion +} diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs similarity index 97% rename from src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs index b6a87993..717b4874 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs @@ -1,6 +1,6 @@ using System.IO; -namespace BotSharp.Core.Files; +namespace BotSharp.Core.Files.Services; public partial class BotSharpFileService { diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs similarity index 96% rename from src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs index 94c9c8e1..602e4316 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs @@ -1,9 +1,7 @@ using Microsoft.AspNetCore.StaticFiles; -using System; using System.IO; -using System.Threading; -namespace BotSharp.Core.Files; +namespace BotSharp.Core.Files.Services; public partial class BotSharpFileService : IBotSharpFileService { @@ -22,6 +20,7 @@ public partial class BotSharpFileService : IBotSharpFileService private const string BOT_FILE_FOLDER = "bot"; private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; + private const string SESSION_FOLDER = "sessions"; private const int MIN_OFFSET = 1; private const int MAX_OFFSET = 5; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b3018bd6..532ae7e0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -137,4 +137,27 @@ public class InstructModeController : ControllerBase return imageViewModel; } } + + [HttpPost("/instruct/pdf-completion")] + public async Task PdfCompletion([FromBody] IncomingMessageModel input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new PdfCompletionViewModel(); + + try + { + var fileService = _services.GetRequiredService(); + var content = await fileService.AnalyzePdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in pdf completion. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs new file mode 100644 index 00000000..13ed3eb9 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class PdfCompletionViewModel +{ + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +}