SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add PaddleOcrConverter

Browse source
This commit is contained in:
Wenbo Cao 2023-08-28 10:24:43 -05:00
parent 37e0a164fd
commit 14d9066f65
12 changed files with 155 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs Normal file
View file

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Abstraction.Knowledges
{
public interface IPaddleOcrConverter
{
// void LoadModel();
Task<string> ConvertImageToText(string loadPath);
}
}

5
src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs
View file

@ -8,10 +8,5 @@ namespace BotSharp.Abstraction.Knowledges
public interface IPdf2TextConverter
{
Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum);
Task<string> OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum);
Task<string> LocalImageToTextsAsync();
Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum);
// void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum);
void DeleteTempFile(string filePath);
}
}

1
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -76,6 +76,7 @@
<PackageReference Include="EntityFrameworkCore.BootKit" Version="6.2.1" />
<PackageReference Include="Fluid.Core" Version="2.4.0" />
<PackageReference Include="TensorFlow.Keras" Version="0.11.2" />
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20230806-4a480" />
</ItemGroup>
<ItemGroup>

3
src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Core.Functions;
using BotSharp.Core.Plugins.Knowledges.Services;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Configuration;
@ -85,5 +86,7 @@ public static class BotSharpServiceCollectionExtensions
loader.Load();
services.AddSingleton(loader);
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
}
}

1
src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs
View file

@ -5,4 +5,5 @@ public class KnowledgeBaseSettings
public string VectorDb { get; set; }
public string TextEmbedding { get; set; }
public string TextCompletion { get; set; }
public string Pdf2TextConverter { get; set; }
}

49
src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs Normal file
View file

@ -0,0 +1,49 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Microsoft.AspNetCore.Http;
using UglyToad.PdfPig;
using UglyToad.PdfPig.Content;
namespace BotSharp.Core.Plugins.Knowledges.Services;
public class PigPdf2TextConverter : IPdf2TextConverter
{
public async Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum)
{
return await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum);
}
private async Task<string> OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum)
{
if (formFile.Length <= 0)
{
return await Task.FromResult(string.Empty);
}
var filePath = Path.GetTempFileName();
using (var stream = System.IO.File.Create(filePath))
{
await formFile.CopyToAsync(stream);
}
var document = PdfDocument.Open(filePath);
var content = "";
foreach (Page page in document.GetPages())
{
if (startPageNum.HasValue && page.Number < startPageNum.Value)
{
continue;
}
if (endPageNum.HasValue && page.Number > endPageNum.Value)
{
continue;
}
content += page.Text;
}
return content;
}
}

4
src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
@ -9,11 +9,11 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
<PackageReference Include="PdfPig" Version="0.1.8" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
</Project>

17
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs
View file

@ -3,6 +3,8 @@ using BotSharp.Abstraction.Knowledges.Models;
using Microsoft.AspNetCore.Http;
using UglyToad.PdfPig.Content;
using UglyToad.PdfPig;
using BotSharp.Core.Plugins.Knowledges;
namespace BotSharp.OpenAPI.Controllers;
@ -11,11 +13,12 @@ namespace BotSharp.OpenAPI.Controllers;
public class KnowledgeController : ControllerBase, IApiAdapter
{
private readonly IKnowledgeService _knowledgeService;
private readonly IPdf2TextConverter _pdf2TextConverter;
public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter)
private readonly IServiceProvider _services;
public KnowledgeController(IKnowledgeService knowledgeService, IServiceProvider services)
{
_knowledgeService = knowledgeService;
_pdf2TextConverter = pdf2TextConverter;
_services = services;
}
[HttpGet("/knowledge/{agentId}")]
public async Task<List<RetrievedResult>> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question)
@ -30,16 +33,20 @@ public class KnowledgeController : ControllerBase, IApiAdapter
[HttpPost("/knowledge/{agentId}")]
public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, List<IFormFile> files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel)
{
var setttings = _services.GetRequiredService<KnowledgeBaseSettings>();
var textConverter = _services.GetServices<IPdf2TextConverter>().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter));
long size = files.Sum(f => f.Length);
foreach (var formFile in files)
{
var content = "";
content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum);
content = await textConverter.ConvertPdfToText(formFile, startPageNum, endPageNum);
// Process uploaded files
// Don't rely on or trust the FileName property without validation.
// Add FeedWithMetaData
await _knowledgeService.Feed(new KnowledgeFeedModel
{
AgentId = agentId,

1
src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj
View file

@ -12,7 +12,6 @@
<PackageReference Include="Magick.NET-Q16-AnyCPU" Version="13.2.0" />
<PackageReference Include="Magick.NET.Core" Version="13.2.0" />
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.7.0.20230115" />
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20230806-4a480" />
<PackageReference Include="Sdcb.PaddleInference" Version="2.4.1.3" />
<PackageReference Include="Sdcb.PaddleInference.runtime.win64.mkl" Version="2.5.1" />
<PackageReference Include="Sdcb.PaddleOCR" Version="2.6.0.5" />

69
src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs Normal file
View file

@ -0,0 +1,69 @@
/*
using System;
using System.Collections.Generic;
using System.Text;
using Sdcb.PaddleOCR;
using Sdcb.PaddleOCR.Models;
using Sdcb.PaddleInference;
using Sdcb.PaddleOCR.Models.LocalV3;
using OpenCvSharp;
using System.Threading.Tasks;
using BotSharp.Abstraction.Knowledges;
using BotSharp.Plugin.PaddleSharp.Settings;
namespace BotSharp.Plugin.PaddleSharp.Providers;
public class PaddleOcrConverter : IPaddleOcrConverter
{
private FullOcrModel _paddleFullOcrmodel;
private QueuedPaddleOcrAll _allModel;
private readonly PaddleSharpSettings _paddleSharpSettings;
public PaddleOcrConverter(FullOcrModel paddleFullOcrmodel, QueuedPaddleOcrAll allModel, PaddleSharpSettings paddleSharpSettings)
{
_paddleFullOcrmodel = paddleFullOcrmodel;
_allModel = allModel;
_paddleSharpSettings = paddleSharpSettings;
}
private void LoadModel()
{
_allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device)
{
AllowRotateDetection = _paddleSharpSettings.allowRotateDetection,
Enable180Classification = _paddleSharpSettings.enable180Classification,
}, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity);
}
private void DisposeModel()
{
_allModel.Dispose();
}
public async Task<string> ConvertImageToText(string loadPath)
{
_allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device)
{
AllowRotateDetection = _paddleSharpSettings.allowRotateDetection,
Enable180Classification = _paddleSharpSettings.enable180Classification,
}, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity);
var contents = "";
using (Mat src = Cv2.ImRead(loadPath))
{
PaddleOcrResult result = await _allModel.Run(src);
foreach (PaddleOcrResultRegion region in result.Regions)
{
if (region.Score > _paddleSharpSettings.acceptScore)
{
contents += region.Text + " ";
}
}
}
_allModel.Dispose();
return contents;
}
}
*/

84
src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs
View file

@ -5,18 +5,13 @@ using System.IO;
using ImageMagick;
using OpenCvSharp;
using Microsoft.AspNetCore.Http;
using System.Runtime.InteropServices.ComTypes;
using Sdcb.PaddleInference;
using Sdcb.PaddleOCR.Models;
using Sdcb.PaddleOCR.Models.LocalV3;
using Sdcb.PaddleOCR;
using System.Threading.Tasks;
using BotSharp.Abstraction.Knowledges;
using static System.Net.WebRequestMethods;
using UglyToad.PdfPig.Content;
using UglyToad.PdfPig;
using System.Linq;
using static System.Net.Mime.MediaTypeNames;
using Docnet;
using Docnet.Core.Models;
using Docnet.Core;
@ -24,25 +19,13 @@ using Docnet.Core.Converters;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Microsoft.Extensions.DependencyInjection;
using BotSharp.Plugin.PaddleSharp.Settings;
namespace BotSharp.Plugin.PaddleSharp.Providers;
public class Pdf2TextConverter : IPdf2TextConverter
{
// private readonly IServiceProvider _service;
{
private Dictionary<int, string> _mappings = new Dictionary<int, string>();
/*
// private FullOcrModel _model;
private string? _tempFolderPath = Path.GetTempPath();
private FullOcrModel _model = LocalFullModels.EnglishV3;
private MagickReadSettings _magicReadSettings;
private int _consumerCount;
private int _boundedCapacity;
private double _acceptScore;
*/
private FullOcrModel _model = LocalFullModels.EnglishV3;
private PaddleSharpSettings _paddleSharpSettings;
public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings)
@ -52,54 +35,11 @@ public class Pdf2TextConverter : IPdf2TextConverter
public async Task<string> ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum)
{
string pdfContent;
if (_paddleSharpSettings.paddleModel)
{
await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum);
pdfContent = await LocalImageToTextsAsync();
}
else
{
pdfContent = await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum);
}
return pdfContent;
await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum);
return await LocalImageToTextsAsync();
}
public async Task<string> OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum)
{
if (formFile.Length <= 0)
{
return await Task.FromResult(string.Empty);
}
var filePath = Path.GetTempFileName();
using (var stream = System.IO.File.Create(filePath))
{
await formFile.CopyToAsync(stream);
}
var document = PdfDocument.Open(filePath);
var content = "";
foreach (Page page in document.GetPages())
{
if (startPageNum.HasValue && page.Number < startPageNum.Value)
{
continue;
}
if (endPageNum.HasValue && page.Number > endPageNum.Value)
{
continue;
}
content += page.Text;
}
return content;
}
public async Task<string> LocalImageToTextsAsync()
private async Task<string> LocalImageToTextsAsync()
{
string loadPath;
string contents = "";
@ -108,8 +48,6 @@ public class Pdf2TextConverter : IPdf2TextConverter
throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\".");
}
// var converter = _service.GetRequiredService<IPaddleOcrConverter>();
QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn())
{
AllowRotateDetection = true,
@ -120,8 +58,6 @@ public class Pdf2TextConverter : IPdf2TextConverter
foreach (var item in _mappings.OrderBy(x => x.Key))
{
loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value);
// var pdfContent = converter.ConvertImageToText(loadPath);
// contents += pdfContent;
using (Mat src = Cv2.ImRead(loadPath))
{
@ -135,12 +71,7 @@ public class Pdf2TextConverter : IPdf2TextConverter
}
}
}
// Delete related Temp files after converting image to texts
// DeleteTempFile(loadPath);
}
// await Console.Out.WriteLineAsync("Finished!");
// all.Dispose();
return contents;
}
@ -184,7 +115,7 @@ public class Pdf2TextConverter : IPdf2TextConverter
};
}
public async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum)
private async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum)
{
string rootFileName;
@ -230,9 +161,4 @@ public class Pdf2TextConverter : IPdf2TextConverter
_mappings[page] = rootFileName;
}
}
public void DeleteTempFile(string filePath)
{
System.IO.File.Delete(filePath);
}
}

2
src/WebStarter/appsettings.json
View file

@ -88,7 +88,7 @@
// "TextEmbedding": "LLamaSharp.TextEmbeddingProvider",
"TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider",
// "TextCompletion": "LLamaSharp.TextCompletionProvider",
"Pdf2TextConverter": ""
"Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter"
},
"PluginLoader": {