diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs new file mode 100644 index 00000000..6c913348 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Abstraction.Knowledges +{ + public interface IPaddleOcrConverter + { + // void LoadModel(); + Task ConvertImageToText(string loadPath); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index 48241a18..0ae85b30 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -8,10 +8,5 @@ namespace BotSharp.Abstraction.Knowledges public interface IPdf2TextConverter { Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum); - Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - Task LocalImageToTextsAsync(); - Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - // void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); - void DeleteTempFile(string filePath); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6b871db4..e39e4a2e 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -76,6 +76,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 9081ca05..08679545 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Functions; using BotSharp.Core.Functions; +using BotSharp.Core.Plugins.Knowledges.Services; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; @@ -85,5 +86,7 @@ public static class BotSharpServiceCollectionExtensions loader.Load(); services.AddSingleton(loader); + + services.AddSingleton(); } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs index a56767e9..0cb08a48 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs @@ -5,4 +5,5 @@ public class KnowledgeBaseSettings public string VectorDb { get; set; } public string TextEmbedding { get; set; } public string TextCompletion { get; set; } + public string Pdf2TextConverter { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs new file mode 100644 index 00000000..c19ddc77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Http; +using UglyToad.PdfPig; +using UglyToad.PdfPig.Content; + +namespace BotSharp.Core.Plugins.Knowledges.Services; + +public class PigPdf2TextConverter : IPdf2TextConverter +{ + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) + { + return await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); + } + + private async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + if (formFile.Length <= 0) + { + return await Task.FromResult(string.Empty); + } + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + var document = PdfDocument.Open(filePath); + var content = ""; + foreach (Page page in document.GetPages()) + { + if (startPageNum.HasValue && page.Number < startPageNum.Value) + { + continue; + } + + if (endPageNum.HasValue && page.Number > endPageNum.Value) + { + continue; + } + content += page.Text; + } + return content; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index d1a16a68..7b8290d6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -9,11 +9,11 @@ - + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs index 8506c0b4..3824f476 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs @@ -3,6 +3,8 @@ using BotSharp.Abstraction.Knowledges.Models; using Microsoft.AspNetCore.Http; using UglyToad.PdfPig.Content; using UglyToad.PdfPig; +using BotSharp.Core.Plugins.Knowledges; + namespace BotSharp.OpenAPI.Controllers; @@ -11,11 +13,12 @@ namespace BotSharp.OpenAPI.Controllers; public class KnowledgeController : ControllerBase, IApiAdapter { private readonly IKnowledgeService _knowledgeService; - private readonly IPdf2TextConverter _pdf2TextConverter; - public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter) + private readonly IServiceProvider _services; + + public KnowledgeController(IKnowledgeService knowledgeService, IServiceProvider services) { _knowledgeService = knowledgeService; - _pdf2TextConverter = pdf2TextConverter; + _services = services; } [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) @@ -30,16 +33,20 @@ public class KnowledgeController : ControllerBase, IApiAdapter [HttpPost("/knowledge/{agentId}")] public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); long size = files.Sum(f => f.Length); foreach (var formFile in files) { var content = ""; - - content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); + + content = await textConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); // Process uploaded files // Don't rely on or trust the FileName property without validation. + + // Add FeedWithMetaData await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 7693c483..3ad42c16 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs new file mode 100644 index 00000000..4cf8edb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs @@ -0,0 +1,69 @@ +/* +using System; +using System.Collections.Generic; +using System.Text; +using Sdcb.PaddleOCR; +using Sdcb.PaddleOCR.Models; +using Sdcb.PaddleInference; +using Sdcb.PaddleOCR.Models.LocalV3; +using OpenCvSharp; +using System.Threading.Tasks; +using BotSharp.Abstraction.Knowledges; +using BotSharp.Plugin.PaddleSharp.Settings; + +namespace BotSharp.Plugin.PaddleSharp.Providers; + +public class PaddleOcrConverter : IPaddleOcrConverter +{ + private FullOcrModel _paddleFullOcrmodel; + private QueuedPaddleOcrAll _allModel; + private readonly PaddleSharpSettings _paddleSharpSettings; + + public PaddleOcrConverter(FullOcrModel paddleFullOcrmodel, QueuedPaddleOcrAll allModel, PaddleSharpSettings paddleSharpSettings) + { + _paddleFullOcrmodel = paddleFullOcrmodel; + _allModel = allModel; + _paddleSharpSettings = paddleSharpSettings; + } + + private void LoadModel() + { + _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device) + { + AllowRotateDetection = _paddleSharpSettings.allowRotateDetection, + Enable180Classification = _paddleSharpSettings.enable180Classification, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + } + + private void DisposeModel() + { + _allModel.Dispose(); + } + + public async Task ConvertImageToText(string loadPath) + { + _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device) + { + AllowRotateDetection = _paddleSharpSettings.allowRotateDetection, + Enable180Classification = _paddleSharpSettings.enable180Classification, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + + var contents = ""; + using (Mat src = Cv2.ImRead(loadPath)) + { + PaddleOcrResult result = await _allModel.Run(src); + + foreach (PaddleOcrResultRegion region in result.Regions) + { + if (region.Score > _paddleSharpSettings.acceptScore) + { + contents += region.Text + " "; + } + } + } + + _allModel.Dispose(); + return contents; + } +} +*/ \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 87fb0efe..ab9d6576 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -5,18 +5,13 @@ using System.IO; using ImageMagick; using OpenCvSharp; using Microsoft.AspNetCore.Http; -using System.Runtime.InteropServices.ComTypes; using Sdcb.PaddleInference; using Sdcb.PaddleOCR.Models; using Sdcb.PaddleOCR.Models.LocalV3; using Sdcb.PaddleOCR; using System.Threading.Tasks; using BotSharp.Abstraction.Knowledges; -using static System.Net.WebRequestMethods; -using UglyToad.PdfPig.Content; -using UglyToad.PdfPig; using System.Linq; -using static System.Net.Mime.MediaTypeNames; using Docnet; using Docnet.Core.Models; using Docnet.Core; @@ -24,25 +19,13 @@ using Docnet.Core.Converters; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; -using Microsoft.Extensions.DependencyInjection; using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ - // private readonly IServiceProvider _service; - +{ private Dictionary _mappings = new Dictionary(); - /* - // private FullOcrModel _model; - private string? _tempFolderPath = Path.GetTempPath(); - private FullOcrModel _model = LocalFullModels.EnglishV3; - private MagickReadSettings _magicReadSettings; - private int _consumerCount; - private int _boundedCapacity; - private double _acceptScore; - */ private FullOcrModel _model = LocalFullModels.EnglishV3; private PaddleSharpSettings _paddleSharpSettings; public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) @@ -52,54 +35,11 @@ public class Pdf2TextConverter : IPdf2TextConverter public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) { - string pdfContent; - if (_paddleSharpSettings.paddleModel) - { - await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); - pdfContent = await LocalImageToTextsAsync(); - } - else - { - pdfContent = await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); - } - return pdfContent; + await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); + return await LocalImageToTextsAsync(); } - public async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) - { - if (formFile.Length <= 0) - { - return await Task.FromResult(string.Empty); - } - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - await formFile.CopyToAsync(stream); - } - - var document = PdfDocument.Open(filePath); - var content = ""; - foreach (Page page in document.GetPages()) - { - if (startPageNum.HasValue && page.Number < startPageNum.Value) - { - continue; - } - - if (endPageNum.HasValue && page.Number > endPageNum.Value) - { - continue; - } - - content += page.Text; - } - - return content; - } - - public async Task LocalImageToTextsAsync() + private async Task LocalImageToTextsAsync() { string loadPath; string contents = ""; @@ -108,8 +48,6 @@ public class Pdf2TextConverter : IPdf2TextConverter throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); } - // var converter = _service.GetRequiredService(); - QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn()) { AllowRotateDetection = true, @@ -120,8 +58,6 @@ public class Pdf2TextConverter : IPdf2TextConverter foreach (var item in _mappings.OrderBy(x => x.Key)) { loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value); - // var pdfContent = converter.ConvertImageToText(loadPath); - // contents += pdfContent; using (Mat src = Cv2.ImRead(loadPath)) { @@ -135,12 +71,7 @@ public class Pdf2TextConverter : IPdf2TextConverter } } } - - // Delete related Temp files after converting image to texts - // DeleteTempFile(loadPath); } - // await Console.Out.WriteLineAsync("Finished!"); - // all.Dispose(); return contents; } @@ -184,7 +115,7 @@ public class Pdf2TextConverter : IPdf2TextConverter }; } - public async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + private async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) { string rootFileName; @@ -230,9 +161,4 @@ public class Pdf2TextConverter : IPdf2TextConverter _mappings[page] = rootFileName; } } - - public void DeleteTempFile(string filePath) - { - System.IO.File.Delete(filePath); - } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 0acf233d..9657f072 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -88,7 +88,7 @@ // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", // "TextCompletion": "LLamaSharp.TextCompletionProvider", - "Pdf2TextConverter": "" + "Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter" }, "PluginLoader": {