Merge pull request #109 from evan-cao-wb/add-PdfToTextConverter

MR for PdfToTextConverter

src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Abstraction.Knowledges
+{
+    public interface IPaddleOcrConverter
+    {
+        // void LoadModel();
+        Task<string> ConvertImageToText(string loadPath);
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.AspNetCore.Http;
+
+namespace BotSharp.Abstraction.Knowledges
+{
+    public interface IPdf2TextConverter
+    {
+        Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum);
+    }
+}

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -76,6 +76,7 @@
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="6.2.1" />
     <PackageReference Include="Fluid.Core" Version="2.4.0" />
     <PackageReference Include="TensorFlow.Keras" Version="0.11.2" />
+    <PackageReference Include="PdfPig" Version="0.1.9-alpha-20230806-4a480" />
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Functions;
 using BotSharp.Core.Functions;
 using BotSharp.Core.Hooks;
 using BotSharp.Core.Templating;
+using BotSharp.Core.Plugins.Knowledges.Services;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.Configuration;
 
@@ -95,5 +96,7 @@ public static class BotSharpServiceCollectionExtensions
         loader.Load();
 
         services.AddSingleton(loader);
+
+        services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs

@@ -5,4 +5,5 @@ public class KnowledgeBaseSettings
     public string VectorDb { get; set; }
     public string TextEmbedding { get; set; }
     public string TextCompletion { get; set; }
+    public string Pdf2TextConverter { get; set; }
 }

src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Microsoft.AspNetCore.Http;
+using UglyToad.PdfPig;
+using UglyToad.PdfPig.Content;
+
+namespace BotSharp.Core.Plugins.Knowledges.Services;
+
+public class PigPdf2TextConverter : IPdf2TextConverter
+{
+    public async Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum)
+    {
+        return await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum);
+    }
+
+    private async Task<string> OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum)
+    {
+        if (formFile.Length <= 0)
+        {
+            return await Task.FromResult(string.Empty);
+        }
+
+        var filePath = Path.GetTempFileName();
+
+        using (var stream = System.IO.File.Create(filePath))
+        {
+            await formFile.CopyToAsync(stream);
+        }
+
+        var document = PdfDocument.Open(filePath);
+        var content = "";
+        foreach (Page page in document.GetPages())
+        {
+            if (startPageNum.HasValue && page.Number < startPageNum.Value)
+            {
+                continue;
+            }
+
+            if (endPageNum.HasValue && page.Number > endPageNum.Value)
+            {
+                continue;
+            }
+            content += page.Text;
+        }
+        return content;
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
@@ -9,11 +9,11 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
-    <PackageReference Include="PdfPig" Version="0.1.8" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+    <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
 </Project>

src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs

@@ -3,6 +3,8 @@ using BotSharp.Abstraction.Knowledges.Models;
 using Microsoft.AspNetCore.Http;
 using UglyToad.PdfPig.Content;
 using UglyToad.PdfPig;
+using BotSharp.Core.Plugins.Knowledges;
+
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -11,11 +13,13 @@ namespace BotSharp.OpenAPI.Controllers;
 public class KnowledgeController : ControllerBase, IApiAdapter
 {
     private readonly IKnowledgeService _knowledgeService;
-    public KnowledgeController(IKnowledgeService knowledgeService)
+    private readonly IServiceProvider _services;
+
+    public KnowledgeController(IKnowledgeService knowledgeService, IServiceProvider services)
     {
         _knowledgeService = knowledgeService;
+        _services = services;
     }
-
     [HttpGet("/knowledge/{agentId}")]
     public async Task<List<RetrievedResult>> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question)
     {
@@ -27,44 +31,22 @@ public class KnowledgeController : ControllerBase, IApiAdapter
     }
 
     [HttpPost("/knowledge/{agentId}")]
-    public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, List<IFormFile> files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum)
+    public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, List<IFormFile> files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel)
     {
+        var setttings = _services.GetRequiredService<KnowledgeBaseSettings>();
+        var textConverter = _services.GetServices<IPdf2TextConverter>().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter));
         long size = files.Sum(f => f.Length);
 
         foreach (var formFile in files)
         {
-            if (formFile.Length <= 0)
-            {
-                continue;
-            }
-
-            var filePath = Path.GetTempFileName();
-
-            using (var stream = System.IO.File.Create(filePath))
-            {
-                await formFile.CopyToAsync(stream);
-            }
-
-            var document = PdfDocument.Open(filePath);
             var content = "";
-            foreach (Page page in document.GetPages())
-            {
-                if (startPageNum.HasValue && page.Number < startPageNum.Value)
-                {
-                    continue;
-                }
 
-                if (endPageNum.HasValue && page.Number > endPageNum.Value)
-                {
-                    continue;
-                }
-
-                content += page.Text;
-            }
+            content = await textConverter.ConvertPdfToText(formFile, startPageNum, endPageNum);
 
             // Process uploaded files
             // Don't rely on or trust the FileName property without validation.
 
+            // Add FeedWithMetaData
             await _knowledgeService.Feed(new KnowledgeFeedModel
             {
                 AgentId = agentId,

src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj

@@ -8,10 +8,15 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Docnet.Core" Version="2.5.0-alpha.1" />
+    <PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.2.0" />
+    <PackageReference Include="Magick.NET.Core" Version="13.2.0" />
     <PackageReference Include="OpenCvSharp4.runtime.win" Version="4.7.0.20230115" />
     <PackageReference Include="Sdcb.PaddleInference" Version="2.4.1.3" />
+    <PackageReference Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
     <PackageReference Include="Sdcb.PaddleOCR" Version="2.6.0.5" />
     <PackageReference Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.6.0.5" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.0-preview.7.23375.5" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs

@@ -1,4 +1,7 @@
+using BotSharp.Abstraction.Knowledges;
 using BotSharp.Abstraction.Plugins;
+using BotSharp.Plugin.PaddleSharp.Providers;
+using BotSharp.Plugin.PaddleSharp.Settings;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using System;
@@ -9,6 +12,9 @@ public class PaddleSharpPlugin : IBotSharpPlugin
 {
     public void RegisterDI(IServiceCollection services, IConfiguration config)
     {
-        
+        var settings = new PaddleSharpSettings();
+        config.Bind("PaddleSharp", settings);
+        services.AddSingleton(x => settings);
+        services.AddSingleton<IPdf2TextConverter, Pdf2TextConverter>();
     }
 }

src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs

@@ -0,0 +1,69 @@
+/*
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Sdcb.PaddleOCR;
+using Sdcb.PaddleOCR.Models;
+using Sdcb.PaddleInference;
+using Sdcb.PaddleOCR.Models.LocalV3;
+using OpenCvSharp;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Knowledges;
+using BotSharp.Plugin.PaddleSharp.Settings;
+
+namespace BotSharp.Plugin.PaddleSharp.Providers;
+
+public class PaddleOcrConverter : IPaddleOcrConverter
+{
+    private FullOcrModel _paddleFullOcrmodel;
+    private QueuedPaddleOcrAll _allModel;
+    private readonly PaddleSharpSettings _paddleSharpSettings;
+
+    public PaddleOcrConverter(FullOcrModel paddleFullOcrmodel, QueuedPaddleOcrAll allModel, PaddleSharpSettings paddleSharpSettings)
+    {
+        _paddleFullOcrmodel = paddleFullOcrmodel;
+        _allModel = allModel;
+        _paddleSharpSettings = paddleSharpSettings;
+    }
+
+    private void LoadModel()
+    {
+        _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device)
+        {
+            AllowRotateDetection = _paddleSharpSettings.allowRotateDetection,
+            Enable180Classification = _paddleSharpSettings.enable180Classification,
+        }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity);
+    }
+
+    private void DisposeModel()
+    {
+        _allModel.Dispose();
+    }
+
+    public async Task<string> ConvertImageToText(string loadPath)
+    {
+        _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device)
+        {
+            AllowRotateDetection = _paddleSharpSettings.allowRotateDetection,
+            Enable180Classification = _paddleSharpSettings.enable180Classification,
+        }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity);
+
+        var contents = "";
+        using (Mat src = Cv2.ImRead(loadPath))
+        {
+            PaddleOcrResult result = await _allModel.Run(src);
+
+            foreach (PaddleOcrResultRegion region in result.Regions)
+            {
+                if (region.Score > _paddleSharpSettings.acceptScore)
+                {
+                    contents += region.Text + " ";
+                }
+            }
+        }
+
+        _allModel.Dispose();
+        return contents;
+    }
+}
+*/

src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs

@@ -0,0 +1,164 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using ImageMagick;
+using OpenCvSharp;
+using Microsoft.AspNetCore.Http;
+using Sdcb.PaddleInference;
+using Sdcb.PaddleOCR.Models;
+using Sdcb.PaddleOCR.Models.LocalV3;
+using Sdcb.PaddleOCR;
+using System.Threading.Tasks;
+using BotSharp.Abstraction.Knowledges;
+using System.Linq;
+using Docnet;
+using Docnet.Core.Models;
+using Docnet.Core;
+using Docnet.Core.Converters;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Runtime.InteropServices;
+using BotSharp.Plugin.PaddleSharp.Settings;
+
+namespace BotSharp.Plugin.PaddleSharp.Providers;
+
+public class Pdf2TextConverter : IPdf2TextConverter
+{    
+    private Dictionary<int, string> _mappings = new Dictionary<int, string>();
+    private FullOcrModel _model = LocalFullModels.EnglishV3;
+    private PaddleSharpSettings _paddleSharpSettings;
+    public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings)
+    {
+        _paddleSharpSettings = paddleSharpSettings;
+    }
+
+    public async Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum)
+    {
+        await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum);
+        return await LocalImageToTextsAsync();
+    }
+
+    private async Task<string> LocalImageToTextsAsync()
+    {
+        string loadPath;
+        string contents = "";
+        if (!Directory.Exists(_paddleSharpSettings.tempFolderPath))
+        {
+            throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\".");
+        }
+
+        QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn())
+        {
+            AllowRotateDetection = true,
+            Enable180Classification = false,
+        }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity);
+        
+        
+        foreach (var item in _mappings.OrderBy(x => x.Key))
+        {
+            loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value);
+            
+            using (Mat src = Cv2.ImRead(loadPath))
+            {
+                PaddleOcrResult result = await all.Run(src);
+
+                foreach (PaddleOcrResultRegion region in result.Regions)
+                {
+                    if (region.Score > _paddleSharpSettings.acceptScore)
+                    {
+                        contents += region.Text + " ";
+                    }
+                }
+            }
+        }
+        return contents;
+    }
+
+    private static void AddBytes(Bitmap bmp, byte[] rawBytes)
+    {
+        var rect = new Rectangle(0, 0, bmp.Width, bmp.Height);
+
+        var bmpData = bmp.LockBits(rect, ImageLockMode.WriteOnly, bmp.PixelFormat);
+        var pNative = bmpData.Scan0;
+
+        Marshal.Copy(rawBytes, 0, pNative, rawBytes.Length);
+        bmp.UnlockBits(bmpData);
+    }
+
+    public void DocnetConverter(string filePath, int width = 1080, int height = 1920)
+    {
+        var pageSettings = new PageDimensions(width, height);
+
+        // using (var docReader = DocLib.Instance.GetDocReader("C:\\Users\\104199\\Postman\\files\\WM2077CW.pdf", new PageDimensions(1080, 1920)))
+        using (var docReader = DocLib.Instance.GetDocReader(filePath, pageSettings))
+        {
+            using (var pageReader = docReader.GetPageReader(17))
+            {
+                var rawBytes = pageReader.GetImage();
+                var pageWidth = pageReader.GetPageWidth();
+                var pageHeight = pageReader.GetPageHeight();
+                var characters = pageReader.GetCharacters();
+
+                using (var bmp = new Bitmap(pageWidth, pageHeight, PixelFormat.Format32bppArgb))
+                {
+                    AddBytes(bmp, rawBytes);
+
+                    using (var imageStream = new MemoryStream())
+                    {
+                        //saving and exporting
+                        bmp.Save(imageStream, ImageFormat.Png);
+                        System.IO.File.WriteAllBytes(filePath, imageStream.ToArray());
+                    };
+                }
+            }
+        };
+    }
+
+    private async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum)
+    {
+        string rootFileName;
+
+        var filePath = Path.GetTempFileName();
+
+        using (var stream = System.IO.File.Create(filePath))
+        {
+            await formFile.CopyToAsync(stream);
+        }
+
+        using var images = new MagickImageCollection();
+        // _magicReadSettings.Density = new Density((double)300);
+        /*
+        using var images = new MagickImageCollection();
+        MagickNET.SetGhostscriptDirectory("C:\\Users\\104199\\Downloads\\ghostpcl-10.01.2-win64\\ghostpcl-10.01.2-win64");
+
+        images.Read("C:\\Users\\104199\\Postman\\files\\page12.pdf", new MagickReadSettings
+        {
+            Density = new Density(300, 300)
+        });
+        */
+        images.Read(filePath, new MagickReadSettings
+        {
+            Density = new Density(300, 300)
+        });
+
+        if (images.Count == 0)
+        {
+            throw new Exception("PDF loading failed. Please check if the PDF format is correct!");
+        }
+
+        startPageNum = startPageNum.HasValue ? startPageNum : 1;
+        endPageNum = endPageNum.HasValue ? endPageNum : images.Count;
+
+        for (int page = (int)startPageNum; page <= (int)endPageNum; page++)
+        {
+            string tempFileName = Path.GetRandomFileName();
+            tempFileName = Path.ChangeExtension(tempFileName, "png");
+            rootFileName = Path.Combine(_paddleSharpSettings.tempFolderPath, tempFileName);
+
+            // image.Format = MagickFormat.Jpg; Set to "Jpg" format
+            images[page].Write(rootFileName);
+            _mappings[page] = rootFileName;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Sdcb.PaddleOCR;
+using ImageMagick;
+using Sdcb.PaddleOCR.Models;
+using System.IO;
+using Sdcb.PaddleInference;
+
+namespace BotSharp.Plugin.PaddleSharp.Settings
+{
+    public class PaddleSharpSettings
+    {
+        public MagickReadSettings magickReadSettings { get; set; }
+        public PaddleOcrAll paddleOcrAll { get; set; }
+        public string tempFolderPath { get; set; } = Path.GetTempPath();
+        public PaddleOcrAll paddleSettings { get; set; }
+        public MagickReadSettings magicReadSettings
+        {
+            get
+            {
+                return magicReadSettings;
+            }
+            set
+            {
+                magicReadSettings.Density = new Density(300, 300);
+            }
+        }
+        public int consumerCount { get; set; } = 1;
+        public int boundedCapacity { get; set; } = 64;
+        public double acceptScore { get; set; }
+        public Action<PaddleConfig> device { get; set; } = PaddleDevice.Mkldnn();
+        public bool allowRotateDetection { get; set; }
+        public bool enable180Classification { get; set; }
+        public bool paddleModel { get; set; } = true;
+    }
+}

src/WebStarter/appsettings.json

@@ -81,14 +81,15 @@
     "WeixinAppSecret": "#{WeixinAppSecret}#"
   },
 
-  "KnowledgeBase": {
-    "VectorDb": "MemVectorDatabase",
-    // "VectorDb": "QdrantDb",
-    "TextEmbedding": "fastTextEmbeddingProvider",
-    // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider",
-    "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider"
-    // "TextCompletion": "LLamaSharp.TextCompletionProvider"
-  },
+    "KnowledgeBase": {
+        "VectorDb": "MemVectorDatabase",
+        // "VectorDb": "QdrantDb",
+        "TextEmbedding": "fastTextEmbeddingProvider",
+        // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider",
+        "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider",
+        // "TextCompletion": "LLamaSharp.TextCompletionProvider",
+        "Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter"
+    },
 
   "PluginLoader": {
     "Assemblies": [