add pdf completion

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -23,6 +23,14 @@ public interface IBotSharpFileService
     bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
     bool DeleteConversationFiles(IEnumerable<string> conversationIds);
 
+    /// <summary>
+    /// Take screenshots of pdf pages and get response from llm
+    /// </summary>
+    /// <param name="prompt"></param>
+    /// <param name="files">Pdf files</param>
+    /// <returns></returns>
+    Task<string> AnalyzePdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
+
     /// <summary>
     /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
     /// </summary>

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -180,6 +180,7 @@
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="8.4.2" />
     <PackageReference Include="Fluid.Core" Version="2.8.0" />
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
     <PackageReference Include="Nanoid" Version="3.0.0" />
     <PackageReference Include="RedLock.net" Version="2.3.2" />
     <PackageReference Include="System.Drawing.Common" Version="8.0.6" />

src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs

@@ -1,4 +1,5 @@
 using BotSharp.Core.Files.Hooks;
+using BotSharp.Core.Files.Services;
 using Microsoft.Extensions.Configuration;
 
 namespace BotSharp.Core.Files;

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs

@@ -1,11 +1,9 @@
 using BotSharp.Abstraction.Files.Converters;
 using Microsoft.EntityFrameworkCore;
-using System;
 using System.IO;
-using System.Linq;
 using System.Threading;
 
-namespace BotSharp.Core.Files;
+namespace BotSharp.Core.Files.Services;
 
 public partial class BotSharpFileService
 {

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs

@@ -0,0 +1,146 @@
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading;
+
+namespace BotSharp.Core.Files.Services;
+
+public partial class BotSharpFileService
+{
+    public async Task<string> AnalyzePdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
+    {
+        var content = string.Empty;
+
+        if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty())
+        {
+            return content;
+        }
+
+        var guid = Guid.NewGuid().ToString();
+        var sessionDir = GetSessionDirectory(guid);
+        if (!ExistDirectory(sessionDir))
+        {
+            Directory.CreateDirectory(sessionDir);
+        }
+
+        try
+        {
+            var pdfFiles = await SaveFiles(sessionDir, files);
+            var images = await ConvertPdfToImages(pdfFiles);
+            if (images.IsNullOrEmpty()) return content;
+
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
+                model: model, modelId: modelId ?? "gpt-4", multiModal: true);
+            var message = await completion.GetChatCompletions(new Agent()
+            {
+                Id = Guid.Empty.ToString(),
+            }, new List<RoleDialogModel>
+            {
+                new RoleDialogModel(AgentRole.User, prompt)
+                {
+                    Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
+                }
+            });
+
+            content = message.Content;
+            return content;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}");
+            return content;
+        }
+        finally
+        {
+            Directory.Delete(sessionDir, true);
+        }
+    }
+
+    #region Private methods
+    private string GetSessionDirectory(string id)
+    {
+        var dir = Path.Combine(_baseDir, SESSION_FOLDER, id);
+        return dir;
+    }
+
+    private async Task<IEnumerable<string>> SaveFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
+    {
+        if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
+        {
+            return Enumerable.Empty<string>();
+        }
+
+        var locs = new List<string>();
+        foreach (var file in files)
+        {
+            try
+            {
+                var bytes = new byte[0];
+                if (!string.IsNullOrEmpty(file.FileUrl))
+                {
+                    var http = _services.GetRequiredService<IHttpClientFactory>();
+                    using var client = http.CreateClient();
+                    bytes = await client.GetByteArrayAsync(file.FileUrl);
+                }
+                else if (!string.IsNullOrEmpty(file.FileData))
+                {
+                    (_, bytes) = GetFileInfoFromData(file.FileData);
+                }
+
+                if (!bytes.IsNullOrEmpty())
+                {
+                    var guid = Guid.NewGuid().ToString();
+                    var fileDir = Path.Combine(dir, guid);
+                    if (!ExistDirectory(fileDir))
+                    {
+                        Directory.CreateDirectory(fileDir);
+                    }
+
+                    var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}");
+                    using (var fs = new FileStream(pdfDir, FileMode.Create))
+                    {
+                        fs.Write(bytes, 0, bytes.Length);
+                        fs.Close();
+                        locs.Add(pdfDir);
+                        Thread.Sleep(100);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning($"Error when saving pdf file: {ex.Message}");
+                continue;
+            }
+        }
+        return locs;
+    }
+
+    private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files)
+    {
+        var images = new List<string>();
+        var converter = GetPdf2ImageConverter();
+        if (converter == null || files.IsNullOrEmpty())
+        {
+            return images;
+        }
+
+        foreach (var file in files)
+        {
+            try
+            {
+                var segs = file.Split(Path.DirectorySeparatorChar);
+                var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1));
+                var folder = Path.Combine(dir, "screenshots");
+                var urls = await converter.ConvertPdfToImages(file, folder);
+                images.AddRange(urls);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}");
+                continue;
+            }
+        }
+        return images;
+    }
+    #endregion
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs

@@ -1,6 +1,6 @@
 using System.IO;
 
-namespace BotSharp.Core.Files;
+namespace BotSharp.Core.Files.Services;
 
 public partial class BotSharpFileService
 {

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs

@@ -1,9 +1,7 @@
 using Microsoft.AspNetCore.StaticFiles;
-using System;
 using System.IO;
-using System.Threading;
 
-namespace BotSharp.Core.Files;
+namespace BotSharp.Core.Files.Services;
 
 public partial class BotSharpFileService : IBotSharpFileService
 {
@@ -22,6 +20,7 @@ public partial class BotSharpFileService : IBotSharpFileService
     private const string BOT_FILE_FOLDER = "bot";
     private const string USERS_FOLDER = "users";
     private const string USER_AVATAR_FOLDER = "avatar";
+    private const string SESSION_FOLDER = "sessions";
 
     private const int MIN_OFFSET = 1;
     private const int MAX_OFFSET = 5;

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -137,4 +137,27 @@ public class InstructModeController : ControllerBase
             return imageViewModel;
         }
     }
+
+    [HttpPost("/instruct/pdf-completion")]
+    public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] IncomingMessageModel input)
+    {
+        var state = _services.GetRequiredService<IConversationStateService>();
+        input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
+        var viewModel = new PdfCompletionViewModel();
+
+        try
+        {
+            var fileService = _services.GetRequiredService<IBotSharpFileService>();
+            var content = await fileService.AnalyzePdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
+            viewModel.Content = content;
+            return viewModel;
+        }
+        catch (Exception ex)
+        {
+            var error = $"Error in pdf completion. {ex.Message}";
+            _logger.LogError(error);
+            viewModel.Message = error;
+            return viewModel;
+        }
+    }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs

@@ -0,0 +1,13 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Instructs;
+
+public class PdfCompletionViewModel
+{
+    [JsonPropertyName("content")]
+    public string Content { get; set; } = string.Empty;
+
+    [JsonPropertyName("message")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Message { get; set; }
+}