From 89cbc4adfca9decf6ba5a8a14b2736b951d1c679 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 21 Aug 2023 15:53:49 -0500 Subject: [PATCH 1/3] MR for PdfToTextConverter --- .../Knowledges/IPdf2TextConverter.cs | 17 ++ .../BotSharp.Plugin.PaddleSharp.csproj | 3 + .../Providers/Pdf2TextConverter.cs | 249 ++++++++++++++++++ .../Settings/PaddleSharpSettings.cs | 14 + src/WebStarter/appsettings.json | 17 +- 5 files changed, 292 insertions(+), 8 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs create mode 100644 src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs create mode 100644 src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs new file mode 100644 index 00000000..af6da85d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.AspNetCore.Http; + +namespace BotSharp.Abstraction.Knowledges +{ + public interface IPdf2TextConverter + { + Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel); + Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); + Task LocalImageToTextsAsync(); + Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); + void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); + void DeleteTempFolder(string filePath = ""); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 0069f97f..5d111ae1 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -8,6 +8,9 @@ + + + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs new file mode 100644 index 00000000..8c6dbfed --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using ImageMagick; +using OpenCvSharp; +using Microsoft.AspNetCore.Http; +using System.Runtime.InteropServices.ComTypes; +using Sdcb.PaddleInference; +using Sdcb.PaddleOCR.Models; +using Sdcb.PaddleOCR.Models.LocalV3; +using Sdcb.PaddleOCR; +using System.Threading.Tasks; +using BotSharp.Abstraction.Knowledges; +using static System.Net.WebRequestMethods; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig; +using System.Linq; +using static System.Net.Mime.MediaTypeNames; + +using Docnet; +using Docnet.Core.Models; +using Docnet.Core; +using Docnet.Core.Converters; +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; + +namespace BotSharp.Plugin.PaddleSharp.Providers; + +public class Pdf2TextConverter : IPdf2TextConverter +{ + private Dictionary _mappings = new Dictionary(); + private FullOcrModel _model = LocalFullModels.EnglishV3; + private string? _tempFolderPath; + private PaddleOcrAll _paddleSettings; + private MagickReadSettings _magicReadSettings; + private int _consumerCount; + private int _boundedCapacity; + private double _acceptScore; + + public Pdf2TextConverter(PaddleOcrAll paddleSettings, MagickReadSettings magicReadSettings, + double acceptScore = 0.8, int consumerCount = 1, int boundedCapacity = 64) + { + _paddleSettings = paddleSettings; + _magicReadSettings = magicReadSettings; + _consumerCount = consumerCount; + _boundedCapacity = boundedCapacity; + _acceptScore = acceptScore; + } + + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel = true) + { + string pdfContent; + if (paddleModel) + { + await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); + pdfContent = LocalImageToTextsAsync().Result; + } + else + { + pdfContent = await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); + } + return pdfContent; + } + + public async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + if (formFile.Length <= 0) + { + return await Task.FromResult(string.Empty); + } + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + var document = PdfDocument.Open(filePath); + var content = ""; + foreach (Page page in document.GetPages()) + { + if (startPageNum.HasValue && page.Number < startPageNum.Value) + { + continue; + } + + if (endPageNum.HasValue && page.Number > endPageNum.Value) + { + continue; + } + + content += page.Text; + } + + return content; + } + + public async Task LocalImageToTextsAsync() + { + string loadPath; + string contents = ""; + if (!System.IO.File.Exists(_tempFolderPath)) + { + throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); + } + + using QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model) + { + AllowRotateDetection = _paddleSettings.AllowRotateDetection, + Enable180Classification = _paddleSettings.Enable180Classification, + }, consumerCount: _consumerCount, boundedCapacity: _boundedCapacity); + + foreach (var item in _mappings.OrderBy(x => x.Key)) + { + loadPath = Path.Combine(_tempFolderPath, item.Value); + using (Mat src = Cv2.ImRead(loadPath)) + { + PaddleOcrResult result = await all.Run(src); + + foreach (PaddleOcrResultRegion region in result.Regions) + { + if (region.Score > _acceptScore) + { + contents += region.Text; + } + } + } + } + + DeleteTempFolder(); + + return contents; + } + + public void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum) + { + // This function is pending. I am considering if we could include Both "ImageMagick" and "Docnet.Core" + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + formFile.CopyTo(stream); + } + } + + private static void AddBytes(Bitmap bmp, byte[] rawBytes) + { + var rect = new Rectangle(0, 0, bmp.Width, bmp.Height); + + var bmpData = bmp.LockBits(rect, ImageLockMode.WriteOnly, bmp.PixelFormat); + var pNative = bmpData.Scan0; + + Marshal.Copy(rawBytes, 0, pNative, rawBytes.Length); + bmp.UnlockBits(bmpData); + } + + public void DocnetConverter(string filePath, int width = 1080, int height = 1920) + { + var pageSettings = new PageDimensions(width, height); + + // using (var docReader = DocLib.Instance.GetDocReader("C:\\Users\\104199\\Postman\\files\\WM2077CW.pdf", new PageDimensions(1080, 1920))) + using (var docReader = DocLib.Instance.GetDocReader(filePath, pageSettings)) + { + using (var pageReader = docReader.GetPageReader(17)) + { + var rawBytes = pageReader.GetImage(); + var pageWidth = pageReader.GetPageWidth(); + var pageHeight = pageReader.GetPageHeight(); + var characters = pageReader.GetCharacters(); + + using (var bmp = new Bitmap(pageWidth, pageHeight, PixelFormat.Format32bppArgb)) + { + AddBytes(bmp, rawBytes); + + using (var imageStream = new MemoryStream()) + { + //saving and exporting + bmp.Save(imageStream, ImageFormat.Png); + System.IO.File.WriteAllBytes(filePath, imageStream.ToArray()); + }; + } + } + }; + } + + public async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + string rootFileName; + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + using var images = new MagickImageCollection(); + // _magicReadSettings.Density = new Density((double)300); + /* + using var images = new MagickImageCollection(); + MagickNET.SetGhostscriptDirectory("C:\\Users\\104199\\Downloads\\ghostpcl-10.01.2-win64\\ghostpcl-10.01.2-win64"); + + images.Read("C:\\Users\\104199\\Postman\\files\\page12.pdf", new MagickReadSettings + { + Density = new Density(300, 300) + }); + */ + images.Read(filePath, new MagickReadSettings + { + Density = new Density(300, 300) + }); + + if (images.Count == 0) + { + throw new Exception("PDF loading failed. Please check if the PDF format is correct!"); + } + + startPageNum = startPageNum.HasValue ? startPageNum : 1; + endPageNum = endPageNum.HasValue ? endPageNum : images.Count; + + for (int page = (int)startPageNum; page <= (int)endPageNum; page++) + { + string tempFileName = Path.GetTempFileName(); + rootFileName = Path.Combine(_tempFolderPath, $"{tempFileName}_Page{page}.png"); + + // image.Format = MagickFormat.Jpg; Set to "Jpg" format + images[page].Write(rootFileName); + + _mappings[page] = rootFileName; + } + } + + public void DeleteTempFolder(string filePath = "") + { + if (!string.IsNullOrEmpty(filePath)) + { + Directory.Delete(_tempFolderPath); + } + else + { + Directory.Delete(_tempFolderPath); + _tempFolderPath = string.Empty; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs new file mode 100644 index 00000000..05592e91 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Sdcb.PaddleOCR; +using ImageMagick; + +namespace BotSharp.Plugin.PaddleSharp.Settings +{ + public class PaddleSharpSettings + { + public MagickReadSettings magickReadSettings { get; set; } + public PaddleOcrAll paddleOcrAll { get; set; } + } +} diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 606ec0b2..0acf233d 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -81,14 +81,15 @@ "WeixinAppSecret": "#{WeixinAppSecret}#" }, - "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - // "VectorDb": "QdrantDb", - "TextEmbedding": "fastTextEmbeddingProvider", - // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider" - // "TextCompletion": "LLamaSharp.TextCompletionProvider" - }, + "KnowledgeBase": { + "VectorDb": "MemVectorDatabase", + // "VectorDb": "QdrantDb", + "TextEmbedding": "fastTextEmbeddingProvider", + // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", + "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", + // "TextCompletion": "LLamaSharp.TextCompletionProvider", + "Pdf2TextConverter": "" + }, "PluginLoader": { "Assemblies": [ From 76ca3ffdb57bae95955372df3a6f12b6c359252b Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Wed, 23 Aug 2023 12:42:36 -0500 Subject: [PATCH 2/3] Add pdf2textconverter using paddlesharp --- .../Knowledges/IPdf2TextConverter.cs | 6 +- .../Controllers/KnowledgeController.cs | 37 ++------ .../BotSharp.Plugin.PaddleSharp.csproj | 3 + .../PaddleSharpPlugin.cs | 8 +- .../Providers/Pdf2TextConverter.cs | 91 ++++++++----------- .../Settings/PaddleSharpSettings.cs | 23 +++++ 6 files changed, 82 insertions(+), 86 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index af6da85d..48241a18 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -7,11 +7,11 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel); + Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum); Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); Task LocalImageToTextsAsync(); Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); - void DeleteTempFolder(string filePath = ""); + // void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); + void DeleteTempFile(string filePath); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs index 4d63885e..8506c0b4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs @@ -11,11 +11,12 @@ namespace BotSharp.OpenAPI.Controllers; public class KnowledgeController : ControllerBase, IApiAdapter { private readonly IKnowledgeService _knowledgeService; - public KnowledgeController(IKnowledgeService knowledgeService) + private readonly IPdf2TextConverter _pdf2TextConverter; + public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter) { _knowledgeService = knowledgeService; + _pdf2TextConverter = pdf2TextConverter; } - [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) { @@ -27,44 +28,18 @@ public class KnowledgeController : ControllerBase, IApiAdapter } [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) { long size = files.Sum(f => f.Length); foreach (var formFile in files) { - if (formFile.Length <= 0) - { - continue; - } - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - await formFile.CopyToAsync(stream); - } - - var document = PdfDocument.Open(filePath); var content = ""; - foreach (Page page in document.GetPages()) - { - if (startPageNum.HasValue && page.Number < startPageNum.Value) - { - continue; - } - - if (endPageNum.HasValue && page.Number > endPageNum.Value) - { - continue; - } - - content += page.Text; - } + + content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); // Process uploaded files // Don't rely on or trust the FileName property without validation. - await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 5d111ae1..7693c483 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -12,9 +12,12 @@ + + + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs index f679c4e7..94796b64 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs @@ -1,4 +1,7 @@ +using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.Plugins; +using BotSharp.Plugin.PaddleSharp.Providers; +using BotSharp.Plugin.PaddleSharp.Settings; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using System; @@ -9,6 +12,9 @@ public class PaddleSharpPlugin : IBotSharpPlugin { public void RegisterDI(IServiceCollection services, IConfiguration config) { - + var settings = new PaddleSharpSettings(); + config.Bind("PaddleSharp", settings); + services.AddSingleton(x => settings); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 8c6dbfed..87fb0efe 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -17,7 +17,6 @@ using UglyToad.PdfPig.Content; using UglyToad.PdfPig; using System.Linq; using static System.Net.Mime.MediaTypeNames; - using Docnet; using Docnet.Core.Models; using Docnet.Core; @@ -25,37 +24,39 @@ using Docnet.Core.Converters; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter { + // private readonly IServiceProvider _service; + private Dictionary _mappings = new Dictionary(); + /* + // private FullOcrModel _model; + private string? _tempFolderPath = Path.GetTempPath(); private FullOcrModel _model = LocalFullModels.EnglishV3; - private string? _tempFolderPath; - private PaddleOcrAll _paddleSettings; private MagickReadSettings _magicReadSettings; private int _consumerCount; private int _boundedCapacity; private double _acceptScore; - - public Pdf2TextConverter(PaddleOcrAll paddleSettings, MagickReadSettings magicReadSettings, - double acceptScore = 0.8, int consumerCount = 1, int boundedCapacity = 64) + */ + private FullOcrModel _model = LocalFullModels.EnglishV3; + private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { - _paddleSettings = paddleSettings; - _magicReadSettings = magicReadSettings; - _consumerCount = consumerCount; - _boundedCapacity = boundedCapacity; - _acceptScore = acceptScore; + _paddleSharpSettings = paddleSharpSettings; } - public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel = true) + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) { string pdfContent; - if (paddleModel) + if (_paddleSharpSettings.paddleModel) { await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); - pdfContent = LocalImageToTextsAsync().Result; + pdfContent = await LocalImageToTextsAsync(); } else { @@ -102,51 +103,47 @@ public class Pdf2TextConverter : IPdf2TextConverter { string loadPath; string contents = ""; - if (!System.IO.File.Exists(_tempFolderPath)) + if (!Directory.Exists(_paddleSharpSettings.tempFolderPath)) { throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); } - using QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model) - { - AllowRotateDetection = _paddleSettings.AllowRotateDetection, - Enable180Classification = _paddleSettings.Enable180Classification, - }, consumerCount: _consumerCount, boundedCapacity: _boundedCapacity); + // var converter = _service.GetRequiredService(); + QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn()) + { + AllowRotateDetection = true, + Enable180Classification = false, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + + foreach (var item in _mappings.OrderBy(x => x.Key)) { - loadPath = Path.Combine(_tempFolderPath, item.Value); + loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value); + // var pdfContent = converter.ConvertImageToText(loadPath); + // contents += pdfContent; + using (Mat src = Cv2.ImRead(loadPath)) { PaddleOcrResult result = await all.Run(src); foreach (PaddleOcrResultRegion region in result.Regions) { - if (region.Score > _acceptScore) + if (region.Score > _paddleSharpSettings.acceptScore) { - contents += region.Text; + contents += region.Text + " "; } } } + + // Delete related Temp files after converting image to texts + // DeleteTempFile(loadPath); } - - DeleteTempFolder(); - + // await Console.Out.WriteLineAsync("Finished!"); + // all.Dispose(); return contents; } - public void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum) - { - // This function is pending. I am considering if we could include Both "ImageMagick" and "Docnet.Core" - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - formFile.CopyTo(stream); - } - } - private static void AddBytes(Bitmap bmp, byte[] rawBytes) { var rect = new Rectangle(0, 0, bmp.Width, bmp.Height); @@ -224,26 +221,18 @@ public class Pdf2TextConverter : IPdf2TextConverter for (int page = (int)startPageNum; page <= (int)endPageNum; page++) { - string tempFileName = Path.GetTempFileName(); - rootFileName = Path.Combine(_tempFolderPath, $"{tempFileName}_Page{page}.png"); + string tempFileName = Path.GetRandomFileName(); + tempFileName = Path.ChangeExtension(tempFileName, "png"); + rootFileName = Path.Combine(_paddleSharpSettings.tempFolderPath, tempFileName); // image.Format = MagickFormat.Jpg; Set to "Jpg" format images[page].Write(rootFileName); - _mappings[page] = rootFileName; } } - public void DeleteTempFolder(string filePath = "") + public void DeleteTempFile(string filePath) { - if (!string.IsNullOrEmpty(filePath)) - { - Directory.Delete(_tempFolderPath); - } - else - { - Directory.Delete(_tempFolderPath); - _tempFolderPath = string.Empty; - } + System.IO.File.Delete(filePath); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs index 05592e91..39c99343 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.Text; using Sdcb.PaddleOCR; using ImageMagick; +using Sdcb.PaddleOCR.Models; +using System.IO; +using Sdcb.PaddleInference; namespace BotSharp.Plugin.PaddleSharp.Settings { @@ -10,5 +13,25 @@ namespace BotSharp.Plugin.PaddleSharp.Settings { public MagickReadSettings magickReadSettings { get; set; } public PaddleOcrAll paddleOcrAll { get; set; } + public string tempFolderPath { get; set; } = Path.GetTempPath(); + public PaddleOcrAll paddleSettings { get; set; } + public MagickReadSettings magicReadSettings + { + get + { + return magicReadSettings; + } + set + { + magicReadSettings.Density = new Density(300, 300); + } + } + public int consumerCount { get; set; } = 1; + public int boundedCapacity { get; set; } = 64; + public double acceptScore { get; set; } + public Action device { get; set; } = PaddleDevice.Mkldnn(); + public bool allowRotateDetection { get; set; } + public bool enable180Classification { get; set; } + public bool paddleModel { get; set; } = true; } } From 14d9066f657cce48ad14b34bebe6a0c8fbd0005e Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 28 Aug 2023 10:24:43 -0500 Subject: [PATCH 3/3] Add PaddleOcrConverter --- .../Knowledges/IPaddleOcrConverter.cs | 12 +++ .../Knowledges/IPdf2TextConverter.cs | 5 -- .../BotSharp.Core/BotSharp.Core.csproj | 1 + .../BotSharpServiceCollectionExtensions.cs | 3 + .../Knowledges/KnowledgeBaseSettings.cs | 1 + .../Services/PigPdf2TextConverter.cs | 49 +++++++++++ .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 4 +- .../Controllers/KnowledgeController.cs | 17 ++-- .../BotSharp.Plugin.PaddleSharp.csproj | 1 - .../Providers/PaddleOcrConverter.cs | 69 +++++++++++++++ .../Providers/Pdf2TextConverter.cs | 84 ++----------------- src/WebStarter/appsettings.json | 2 +- 12 files changed, 155 insertions(+), 93 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs create mode 100644 src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs create mode 100644 src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs new file mode 100644 index 00000000..6c913348 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Abstraction.Knowledges +{ + public interface IPaddleOcrConverter + { + // void LoadModel(); + Task ConvertImageToText(string loadPath); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index 48241a18..0ae85b30 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -8,10 +8,5 @@ namespace BotSharp.Abstraction.Knowledges public interface IPdf2TextConverter { Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum); - Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - Task LocalImageToTextsAsync(); - Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - // void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); - void DeleteTempFile(string filePath); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6b871db4..e39e4a2e 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -76,6 +76,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 9081ca05..08679545 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Functions; using BotSharp.Core.Functions; +using BotSharp.Core.Plugins.Knowledges.Services; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; @@ -85,5 +86,7 @@ public static class BotSharpServiceCollectionExtensions loader.Load(); services.AddSingleton(loader); + + services.AddSingleton(); } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs index a56767e9..0cb08a48 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs @@ -5,4 +5,5 @@ public class KnowledgeBaseSettings public string VectorDb { get; set; } public string TextEmbedding { get; set; } public string TextCompletion { get; set; } + public string Pdf2TextConverter { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs new file mode 100644 index 00000000..c19ddc77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Http; +using UglyToad.PdfPig; +using UglyToad.PdfPig.Content; + +namespace BotSharp.Core.Plugins.Knowledges.Services; + +public class PigPdf2TextConverter : IPdf2TextConverter +{ + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) + { + return await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); + } + + private async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + if (formFile.Length <= 0) + { + return await Task.FromResult(string.Empty); + } + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + var document = PdfDocument.Open(filePath); + var content = ""; + foreach (Page page in document.GetPages()) + { + if (startPageNum.HasValue && page.Number < startPageNum.Value) + { + continue; + } + + if (endPageNum.HasValue && page.Number > endPageNum.Value) + { + continue; + } + content += page.Text; + } + return content; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index d1a16a68..7b8290d6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -9,11 +9,11 @@ - + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs index 8506c0b4..3824f476 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs @@ -3,6 +3,8 @@ using BotSharp.Abstraction.Knowledges.Models; using Microsoft.AspNetCore.Http; using UglyToad.PdfPig.Content; using UglyToad.PdfPig; +using BotSharp.Core.Plugins.Knowledges; + namespace BotSharp.OpenAPI.Controllers; @@ -11,11 +13,12 @@ namespace BotSharp.OpenAPI.Controllers; public class KnowledgeController : ControllerBase, IApiAdapter { private readonly IKnowledgeService _knowledgeService; - private readonly IPdf2TextConverter _pdf2TextConverter; - public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter) + private readonly IServiceProvider _services; + + public KnowledgeController(IKnowledgeService knowledgeService, IServiceProvider services) { _knowledgeService = knowledgeService; - _pdf2TextConverter = pdf2TextConverter; + _services = services; } [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) @@ -30,16 +33,20 @@ public class KnowledgeController : ControllerBase, IApiAdapter [HttpPost("/knowledge/{agentId}")] public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); long size = files.Sum(f => f.Length); foreach (var formFile in files) { var content = ""; - - content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); + + content = await textConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); // Process uploaded files // Don't rely on or trust the FileName property without validation. + + // Add FeedWithMetaData await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 7693c483..3ad42c16 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs new file mode 100644 index 00000000..4cf8edb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs @@ -0,0 +1,69 @@ +/* +using System; +using System.Collections.Generic; +using System.Text; +using Sdcb.PaddleOCR; +using Sdcb.PaddleOCR.Models; +using Sdcb.PaddleInference; +using Sdcb.PaddleOCR.Models.LocalV3; +using OpenCvSharp; +using System.Threading.Tasks; +using BotSharp.Abstraction.Knowledges; +using BotSharp.Plugin.PaddleSharp.Settings; + +namespace BotSharp.Plugin.PaddleSharp.Providers; + +public class PaddleOcrConverter : IPaddleOcrConverter +{ + private FullOcrModel _paddleFullOcrmodel; + private QueuedPaddleOcrAll _allModel; + private readonly PaddleSharpSettings _paddleSharpSettings; + + public PaddleOcrConverter(FullOcrModel paddleFullOcrmodel, QueuedPaddleOcrAll allModel, PaddleSharpSettings paddleSharpSettings) + { + _paddleFullOcrmodel = paddleFullOcrmodel; + _allModel = allModel; + _paddleSharpSettings = paddleSharpSettings; + } + + private void LoadModel() + { + _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device) + { + AllowRotateDetection = _paddleSharpSettings.allowRotateDetection, + Enable180Classification = _paddleSharpSettings.enable180Classification, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + } + + private void DisposeModel() + { + _allModel.Dispose(); + } + + public async Task ConvertImageToText(string loadPath) + { + _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device) + { + AllowRotateDetection = _paddleSharpSettings.allowRotateDetection, + Enable180Classification = _paddleSharpSettings.enable180Classification, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + + var contents = ""; + using (Mat src = Cv2.ImRead(loadPath)) + { + PaddleOcrResult result = await _allModel.Run(src); + + foreach (PaddleOcrResultRegion region in result.Regions) + { + if (region.Score > _paddleSharpSettings.acceptScore) + { + contents += region.Text + " "; + } + } + } + + _allModel.Dispose(); + return contents; + } +} +*/ \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 87fb0efe..ab9d6576 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -5,18 +5,13 @@ using System.IO; using ImageMagick; using OpenCvSharp; using Microsoft.AspNetCore.Http; -using System.Runtime.InteropServices.ComTypes; using Sdcb.PaddleInference; using Sdcb.PaddleOCR.Models; using Sdcb.PaddleOCR.Models.LocalV3; using Sdcb.PaddleOCR; using System.Threading.Tasks; using BotSharp.Abstraction.Knowledges; -using static System.Net.WebRequestMethods; -using UglyToad.PdfPig.Content; -using UglyToad.PdfPig; using System.Linq; -using static System.Net.Mime.MediaTypeNames; using Docnet; using Docnet.Core.Models; using Docnet.Core; @@ -24,25 +19,13 @@ using Docnet.Core.Converters; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; -using Microsoft.Extensions.DependencyInjection; using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ - // private readonly IServiceProvider _service; - +{ private Dictionary _mappings = new Dictionary(); - /* - // private FullOcrModel _model; - private string? _tempFolderPath = Path.GetTempPath(); - private FullOcrModel _model = LocalFullModels.EnglishV3; - private MagickReadSettings _magicReadSettings; - private int _consumerCount; - private int _boundedCapacity; - private double _acceptScore; - */ private FullOcrModel _model = LocalFullModels.EnglishV3; private PaddleSharpSettings _paddleSharpSettings; public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) @@ -52,54 +35,11 @@ public class Pdf2TextConverter : IPdf2TextConverter public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) { - string pdfContent; - if (_paddleSharpSettings.paddleModel) - { - await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); - pdfContent = await LocalImageToTextsAsync(); - } - else - { - pdfContent = await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); - } - return pdfContent; + await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); + return await LocalImageToTextsAsync(); } - public async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) - { - if (formFile.Length <= 0) - { - return await Task.FromResult(string.Empty); - } - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - await formFile.CopyToAsync(stream); - } - - var document = PdfDocument.Open(filePath); - var content = ""; - foreach (Page page in document.GetPages()) - { - if (startPageNum.HasValue && page.Number < startPageNum.Value) - { - continue; - } - - if (endPageNum.HasValue && page.Number > endPageNum.Value) - { - continue; - } - - content += page.Text; - } - - return content; - } - - public async Task LocalImageToTextsAsync() + private async Task LocalImageToTextsAsync() { string loadPath; string contents = ""; @@ -108,8 +48,6 @@ public class Pdf2TextConverter : IPdf2TextConverter throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); } - // var converter = _service.GetRequiredService(); - QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn()) { AllowRotateDetection = true, @@ -120,8 +58,6 @@ public class Pdf2TextConverter : IPdf2TextConverter foreach (var item in _mappings.OrderBy(x => x.Key)) { loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value); - // var pdfContent = converter.ConvertImageToText(loadPath); - // contents += pdfContent; using (Mat src = Cv2.ImRead(loadPath)) { @@ -135,12 +71,7 @@ public class Pdf2TextConverter : IPdf2TextConverter } } } - - // Delete related Temp files after converting image to texts - // DeleteTempFile(loadPath); } - // await Console.Out.WriteLineAsync("Finished!"); - // all.Dispose(); return contents; } @@ -184,7 +115,7 @@ public class Pdf2TextConverter : IPdf2TextConverter }; } - public async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + private async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) { string rootFileName; @@ -230,9 +161,4 @@ public class Pdf2TextConverter : IPdf2TextConverter _mappings[page] = rootFileName; } } - - public void DeleteTempFile(string filePath) - { - System.IO.File.Delete(filePath); - } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 0acf233d..9657f072 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -88,7 +88,7 @@ // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", // "TextCompletion": "LLamaSharp.TextCompletionProvider", - "Pdf2TextConverter": "" + "Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter" }, "PluginLoader": {