diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index af6da85d..48241a18 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -7,11 +7,11 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel); + Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum); Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); Task LocalImageToTextsAsync(); Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); - void DeleteTempFolder(string filePath = ""); + // void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); + void DeleteTempFile(string filePath); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs index 4d63885e..8506c0b4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs @@ -11,11 +11,12 @@ namespace BotSharp.OpenAPI.Controllers; public class KnowledgeController : ControllerBase, IApiAdapter { private readonly IKnowledgeService _knowledgeService; - public KnowledgeController(IKnowledgeService knowledgeService) + private readonly IPdf2TextConverter _pdf2TextConverter; + public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter) { _knowledgeService = knowledgeService; + _pdf2TextConverter = pdf2TextConverter; } - [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) { @@ -27,44 +28,18 @@ public class KnowledgeController : ControllerBase, IApiAdapter } [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) { long size = files.Sum(f => f.Length); foreach (var formFile in files) { - if (formFile.Length <= 0) - { - continue; - } - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - await formFile.CopyToAsync(stream); - } - - var document = PdfDocument.Open(filePath); var content = ""; - foreach (Page page in document.GetPages()) - { - if (startPageNum.HasValue && page.Number < startPageNum.Value) - { - continue; - } - - if (endPageNum.HasValue && page.Number > endPageNum.Value) - { - continue; - } - - content += page.Text; - } + + content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); // Process uploaded files // Don't rely on or trust the FileName property without validation. - await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 5d111ae1..7693c483 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -12,9 +12,12 @@ + + + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs index f679c4e7..94796b64 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs @@ -1,4 +1,7 @@ +using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.Plugins; +using BotSharp.Plugin.PaddleSharp.Providers; +using BotSharp.Plugin.PaddleSharp.Settings; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using System; @@ -9,6 +12,9 @@ public class PaddleSharpPlugin : IBotSharpPlugin { public void RegisterDI(IServiceCollection services, IConfiguration config) { - + var settings = new PaddleSharpSettings(); + config.Bind("PaddleSharp", settings); + services.AddSingleton(x => settings); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 8c6dbfed..87fb0efe 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -17,7 +17,6 @@ using UglyToad.PdfPig.Content; using UglyToad.PdfPig; using System.Linq; using static System.Net.Mime.MediaTypeNames; - using Docnet; using Docnet.Core.Models; using Docnet.Core; @@ -25,37 +24,39 @@ using Docnet.Core.Converters; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter { + // private readonly IServiceProvider _service; + private Dictionary _mappings = new Dictionary(); + /* + // private FullOcrModel _model; + private string? _tempFolderPath = Path.GetTempPath(); private FullOcrModel _model = LocalFullModels.EnglishV3; - private string? _tempFolderPath; - private PaddleOcrAll _paddleSettings; private MagickReadSettings _magicReadSettings; private int _consumerCount; private int _boundedCapacity; private double _acceptScore; - - public Pdf2TextConverter(PaddleOcrAll paddleSettings, MagickReadSettings magicReadSettings, - double acceptScore = 0.8, int consumerCount = 1, int boundedCapacity = 64) + */ + private FullOcrModel _model = LocalFullModels.EnglishV3; + private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { - _paddleSettings = paddleSettings; - _magicReadSettings = magicReadSettings; - _consumerCount = consumerCount; - _boundedCapacity = boundedCapacity; - _acceptScore = acceptScore; + _paddleSharpSettings = paddleSharpSettings; } - public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel = true) + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) { string pdfContent; - if (paddleModel) + if (_paddleSharpSettings.paddleModel) { await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); - pdfContent = LocalImageToTextsAsync().Result; + pdfContent = await LocalImageToTextsAsync(); } else { @@ -102,51 +103,47 @@ public class Pdf2TextConverter : IPdf2TextConverter { string loadPath; string contents = ""; - if (!System.IO.File.Exists(_tempFolderPath)) + if (!Directory.Exists(_paddleSharpSettings.tempFolderPath)) { throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); } - using QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model) - { - AllowRotateDetection = _paddleSettings.AllowRotateDetection, - Enable180Classification = _paddleSettings.Enable180Classification, - }, consumerCount: _consumerCount, boundedCapacity: _boundedCapacity); + // var converter = _service.GetRequiredService(); + QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn()) + { + AllowRotateDetection = true, + Enable180Classification = false, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + + foreach (var item in _mappings.OrderBy(x => x.Key)) { - loadPath = Path.Combine(_tempFolderPath, item.Value); + loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value); + // var pdfContent = converter.ConvertImageToText(loadPath); + // contents += pdfContent; + using (Mat src = Cv2.ImRead(loadPath)) { PaddleOcrResult result = await all.Run(src); foreach (PaddleOcrResultRegion region in result.Regions) { - if (region.Score > _acceptScore) + if (region.Score > _paddleSharpSettings.acceptScore) { - contents += region.Text; + contents += region.Text + " "; } } } + + // Delete related Temp files after converting image to texts + // DeleteTempFile(loadPath); } - - DeleteTempFolder(); - + // await Console.Out.WriteLineAsync("Finished!"); + // all.Dispose(); return contents; } - public void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum) - { - // This function is pending. I am considering if we could include Both "ImageMagick" and "Docnet.Core" - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - formFile.CopyTo(stream); - } - } - private static void AddBytes(Bitmap bmp, byte[] rawBytes) { var rect = new Rectangle(0, 0, bmp.Width, bmp.Height); @@ -224,26 +221,18 @@ public class Pdf2TextConverter : IPdf2TextConverter for (int page = (int)startPageNum; page <= (int)endPageNum; page++) { - string tempFileName = Path.GetTempFileName(); - rootFileName = Path.Combine(_tempFolderPath, $"{tempFileName}_Page{page}.png"); + string tempFileName = Path.GetRandomFileName(); + tempFileName = Path.ChangeExtension(tempFileName, "png"); + rootFileName = Path.Combine(_paddleSharpSettings.tempFolderPath, tempFileName); // image.Format = MagickFormat.Jpg; Set to "Jpg" format images[page].Write(rootFileName); - _mappings[page] = rootFileName; } } - public void DeleteTempFolder(string filePath = "") + public void DeleteTempFile(string filePath) { - if (!string.IsNullOrEmpty(filePath)) - { - Directory.Delete(_tempFolderPath); - } - else - { - Directory.Delete(_tempFolderPath); - _tempFolderPath = string.Empty; - } + System.IO.File.Delete(filePath); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs index 05592e91..39c99343 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.Text; using Sdcb.PaddleOCR; using ImageMagick; +using Sdcb.PaddleOCR.Models; +using System.IO; +using Sdcb.PaddleInference; namespace BotSharp.Plugin.PaddleSharp.Settings { @@ -10,5 +13,25 @@ namespace BotSharp.Plugin.PaddleSharp.Settings { public MagickReadSettings magickReadSettings { get; set; } public PaddleOcrAll paddleOcrAll { get; set; } + public string tempFolderPath { get; set; } = Path.GetTempPath(); + public PaddleOcrAll paddleSettings { get; set; } + public MagickReadSettings magicReadSettings + { + get + { + return magicReadSettings; + } + set + { + magicReadSettings.Density = new Density(300, 300); + } + } + public int consumerCount { get; set; } = 1; + public int boundedCapacity { get; set; } = 64; + public double acceptScore { get; set; } + public Action device { get; set; } = PaddleDevice.Mkldnn(); + public bool allowRotateDetection { get; set; } + public bool enable180Classification { get; set; } + public bool paddleModel { get; set; } = true; } }