SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #319 from waelhosn/master

Browse source 
Resolved issue related to convertPdfToText
This commit is contained in:
C. Oceania 2024-02-27 15:07:16 -06:00 committed by GitHub
parent 32d5fc05bc fce5186490
commit 6e89dc6969
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 23 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.Knowledges.Settings;
using Microsoft.AspNetCore.Http;
namespace BotSharp.OpenAPI.Controllers;
@ -16,13 +17,6 @@ public class KnowledgeBaseController : ControllerBase
_services = services;
}
[HttpPost("/knowledge-base/embed")]
public async Task EmbedKnowledge()
{
var chunks = await _knowledgeService.CollectChunkedKnowledge();
await _knowledgeService.EmbedKnowledge(chunks);
}
[HttpGet("/knowledge/{agentId}")]
public async Task<List<RetrievedResult>> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question)
{
@ -70,8 +64,13 @@ public class KnowledgeBaseController : ControllerBase
foreach (var formFile in files)
{
var filePath = Path.GetTempFileName();
using var stream = System.IO.File.Create(filePath);
await formFile.CopyToAsync(stream);
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None))
{
await formFile.CopyToAsync(stream);
await stream.FlushAsync(); // Ensure all data is written to the file
}
var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum);
@ -84,6 +83,9 @@ public class KnowledgeBaseController : ControllerBase
AgentId = agentId,
Content = content
});
// Delete the temp file after processing to clean up
System.IO.File.Delete(filePath);
}
return Ok(new { count = files.Count, size });

28
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs
View file

@ -5,28 +5,24 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
public class PigPdf2TextConverter : IPdf2TextConverter
{
public async Task<string> ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum)
public Task<string> ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum)
{
return await OpenPdfDocumentAsync(filePath, startPageNum, endPageNum);
// since PdfDocument.Open is not async, we dont need to make this method async
// if you need this method to be async, consider wrapping the call in Task.Run for CPU-bound work
return Task.FromResult(OpenPdfDocument(filePath, startPageNum, endPageNum));
}
private async Task<string> OpenPdfDocumentAsync(string filePath, int? startPageNum, int? endPageNum)
private string OpenPdfDocument(string filePath, int? startPageNum, int? endPageNum)
{
var document = PdfDocument.Open(filePath);
var content = "";
using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var document = PdfDocument.Open(fileStream);
var content = new StringBuilder();
foreach (Page page in document.GetPages())
{
if (startPageNum.HasValue && page.Number < startPageNum.Value)
{
continue;
}
if (endPageNum.HasValue && page.Number > endPageNum.Value)
{
continue;
}
content += page.Text;
if (startPageNum.HasValue && page.Number < startPageNum.Value) continue;
if (endPageNum.HasValue && page.Number > endPageNum.Value) continue;
content.Append(page.Text);
}
return content;
return content.ToString();
}
}