SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add pdf2textconverter using paddlesharp

Browse source
This commit is contained in:
Wenbo Cao 2023-08-23 12:42:36 -05:00
parent 89cbc4adfc
commit 76ca3ffdb5
6 changed files with 82 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs
View file

@ -7,11 +7,11 @@ namespace BotSharp.Abstraction.Knowledges
{
public interface IPdf2TextConverter
{
Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel);
Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum);
Task<string> OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum);
Task<string> LocalImageToTextsAsync();
Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum);
void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum);
void DeleteTempFolder(string filePath = "");
// void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum);
void DeleteTempFile(string filePath);
}
}

37
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs
View file

@ -11,11 +11,12 @@ namespace BotSharp.OpenAPI.Controllers;
public class KnowledgeController : ControllerBase, IApiAdapter
{
private readonly IKnowledgeService _knowledgeService;
public KnowledgeController(IKnowledgeService knowledgeService)
private readonly IPdf2TextConverter _pdf2TextConverter;
public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter)
{
_knowledgeService = knowledgeService;
_pdf2TextConverter = pdf2TextConverter;
}
[HttpGet("/knowledge/{agentId}")]
public async Task<List<RetrievedResult>> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question)
{
@ -27,44 +28,18 @@ public class KnowledgeController : ControllerBase, IApiAdapter
}
[HttpPost("/knowledge/{agentId}")]
public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, List<IFormFile> files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum)
public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, List<IFormFile> files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel)
{
long size = files.Sum(f => f.Length);
foreach (var formFile in files)
{
if (formFile.Length <= 0)
{
continue;
}
var filePath = Path.GetTempFileName();
using (var stream = System.IO.File.Create(filePath))
{
await formFile.CopyToAsync(stream);
}
var document = PdfDocument.Open(filePath);
var content = "";
foreach (Page page in document.GetPages())
{
if (startPageNum.HasValue && page.Number < startPageNum.Value)
{
continue;
}
if (endPageNum.HasValue && page.Number > endPageNum.Value)
{
continue;
}
content += page.Text;
}
content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum);
// Process uploaded files
// Don't rely on or trust the FileName property without validation.
await _knowledgeService.Feed(new KnowledgeFeedModel
{
AgentId = agentId,

3
src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj
View file

@ -12,9 +12,12 @@
<PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.2.0" />
<PackageReference Include="Magick.NET.Core" Version="13.2.0" />
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.7.0.20230115" />
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20230806-4a480" />
<PackageReference Include="Sdcb.PaddleInference" Version="2.4.1.3" />
<PackageReference Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
<PackageReference Include="Sdcb.PaddleOCR" Version="2.6.0.5" />
<PackageReference Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.6.0.5" />
<PackageReference Include="System.Drawing.Common" Version="8.0.0-preview.7.23375.5" />
</ItemGroup>
<ItemGroup>

8
src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs
View file

@ -1,4 +1,7 @@
using BotSharp.Abstraction.Knowledges;
using BotSharp.Abstraction.Plugins;
using BotSharp.Plugin.PaddleSharp.Providers;
using BotSharp.Plugin.PaddleSharp.Settings;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using System;
@ -9,6 +12,9 @@ public class PaddleSharpPlugin : IBotSharpPlugin
{
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
var settings = new PaddleSharpSettings();
config.Bind("PaddleSharp", settings);
services.AddSingleton(x => settings);
services.AddSingleton<IPdf2TextConverter, Pdf2TextConverter>();
}
}

91
src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs
View file

@ -17,7 +17,6 @@ using UglyToad.PdfPig.Content;
using UglyToad.PdfPig;
using System.Linq;
using static System.Net.Mime.MediaTypeNames;
using Docnet;
using Docnet.Core.Models;
using Docnet.Core;
@ -25,37 +24,39 @@ using Docnet.Core.Converters;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Microsoft.Extensions.DependencyInjection;
using BotSharp.Plugin.PaddleSharp.Settings;
namespace BotSharp.Plugin.PaddleSharp.Providers;
public class Pdf2TextConverter : IPdf2TextConverter
{
// private readonly IServiceProvider _service;
private Dictionary<int, string> _mappings = new Dictionary<int, string>();
/*
// private FullOcrModel _model;
private string? _tempFolderPath = Path.GetTempPath();
private FullOcrModel _model = LocalFullModels.EnglishV3;
private string? _tempFolderPath;
private PaddleOcrAll _paddleSettings;
private MagickReadSettings _magicReadSettings;
private int _consumerCount;
private int _boundedCapacity;
private double _acceptScore;
public Pdf2TextConverter(PaddleOcrAll paddleSettings, MagickReadSettings magicReadSettings,
double acceptScore = 0.8, int consumerCount = 1, int boundedCapacity = 64)
*/
private FullOcrModel _model = LocalFullModels.EnglishV3;
private PaddleSharpSettings _paddleSharpSettings;
public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings)
{
_paddleSettings = paddleSettings;
_magicReadSettings = magicReadSettings;
_consumerCount = consumerCount;
_boundedCapacity = boundedCapacity;
_acceptScore = acceptScore;
_paddleSharpSettings = paddleSharpSettings;
}
public async Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel = true)
public async Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum)
{
string pdfContent;
if (paddleModel)
if (_paddleSharpSettings.paddleModel)
{
await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum);
pdfContent = LocalImageToTextsAsync().Result;
pdfContent = await LocalImageToTextsAsync();
}
else
{
@ -102,51 +103,47 @@ public class Pdf2TextConverter : IPdf2TextConverter
{
string loadPath;
string contents = "";
if (!System.IO.File.Exists(_tempFolderPath))
if (!Directory.Exists(_paddleSharpSettings.tempFolderPath))
{
throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\".");
}
using QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model)
{
AllowRotateDetection = _paddleSettings.AllowRotateDetection,
Enable180Classification = _paddleSettings.Enable180Classification,
}, consumerCount: _consumerCount, boundedCapacity: _boundedCapacity);
// var converter = _service.GetRequiredService<IPaddleOcrConverter>();
QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn())
{
AllowRotateDetection = true,
Enable180Classification = false,
}, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity);
foreach (var item in _mappings.OrderBy(x => x.Key))
{
loadPath = Path.Combine(_tempFolderPath, item.Value);
loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value);
// var pdfContent = converter.ConvertImageToText(loadPath);
// contents += pdfContent;
using (Mat src = Cv2.ImRead(loadPath))
{
PaddleOcrResult result = await all.Run(src);
foreach (PaddleOcrResultRegion region in result.Regions)
{
if (region.Score > _acceptScore)
if (region.Score > _paddleSharpSettings.acceptScore)
{
contents += region.Text;
contents += region.Text + " ";
}
}
}
// Delete related Temp files after converting image to texts
// DeleteTempFile(loadPath);
}
DeleteTempFolder();
// await Console.Out.WriteLineAsync("Finished!");
// all.Dispose();
return contents;
}
public void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum)
{
// This function is pending. I am considering if we could include Both "ImageMagick" and "Docnet.Core"
var filePath = Path.GetTempFileName();
using (var stream = System.IO.File.Create(filePath))
{
formFile.CopyTo(stream);
}
}
private static void AddBytes(Bitmap bmp, byte[] rawBytes)
{
var rect = new Rectangle(0, 0, bmp.Width, bmp.Height);
@ -224,26 +221,18 @@ public class Pdf2TextConverter : IPdf2TextConverter
for (int page = (int)startPageNum; page <= (int)endPageNum; page++)
{
string tempFileName = Path.GetTempFileName();
rootFileName = Path.Combine(_tempFolderPath, $"{tempFileName}_Page{page}.png");
string tempFileName = Path.GetRandomFileName();
tempFileName = Path.ChangeExtension(tempFileName, "png");
rootFileName = Path.Combine(_paddleSharpSettings.tempFolderPath, tempFileName);
// image.Format = MagickFormat.Jpg; Set to "Jpg" format
images[page].Write(rootFileName);
_mappings[page] = rootFileName;
}
}
public void DeleteTempFolder(string filePath = "")
public void DeleteTempFile(string filePath)
{
if (!string.IsNullOrEmpty(filePath))
{
Directory.Delete(_tempFolderPath);
}
else
{
Directory.Delete(_tempFolderPath);
_tempFolderPath = string.Empty;
}
System.IO.File.Delete(filePath);
}
}

23
src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs
View file

@ -3,6 +3,9 @@ using System.Collections.Generic;
using System.Text;
using Sdcb.PaddleOCR;
using ImageMagick;
using Sdcb.PaddleOCR.Models;
using System.IO;
using Sdcb.PaddleInference;
namespace BotSharp.Plugin.PaddleSharp.Settings
{
@ -10,5 +13,25 @@ namespace BotSharp.Plugin.PaddleSharp.Settings
{
public MagickReadSettings magickReadSettings { get; set; }
public PaddleOcrAll paddleOcrAll { get; set; }
public string tempFolderPath { get; set; } = Path.GetTempPath();
public PaddleOcrAll paddleSettings { get; set; }
public MagickReadSettings magicReadSettings
{
get
{
return magicReadSettings;
}
set
{
magicReadSettings.Density = new Density(300, 300);
}
}
public int consumerCount { get; set; } = 1;
public int boundedCapacity { get; set; } = 64;
public double acceptScore { get; set; }
public Action<PaddleConfig> device { get; set; } = PaddleDevice.Mkldnn();
public bool allowRotateDetection { get; set; }
public bool enable180Classification { get; set; }
public bool paddleModel { get; set; } = true;
}
}