From 89cbc4adfca9decf6ba5a8a14b2736b951d1c679 Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 21 Aug 2023 15:53:49 -0500 Subject: [PATCH 1/6] MR for PdfToTextConverter --- .../Knowledges/IPdf2TextConverter.cs | 17 ++ .../BotSharp.Plugin.PaddleSharp.csproj | 3 + .../Providers/Pdf2TextConverter.cs | 249 ++++++++++++++++++ .../Settings/PaddleSharpSettings.cs | 14 + src/WebStarter/appsettings.json | 17 +- 5 files changed, 292 insertions(+), 8 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs create mode 100644 src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs create mode 100644 src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs new file mode 100644 index 00000000..af6da85d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.AspNetCore.Http; + +namespace BotSharp.Abstraction.Knowledges +{ + public interface IPdf2TextConverter + { + Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel); + Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); + Task LocalImageToTextsAsync(); + Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); + void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); + void DeleteTempFolder(string filePath = ""); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 0069f97f..5d111ae1 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -8,6 +8,9 @@ + + + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs new file mode 100644 index 00000000..8c6dbfed --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using ImageMagick; +using OpenCvSharp; +using Microsoft.AspNetCore.Http; +using System.Runtime.InteropServices.ComTypes; +using Sdcb.PaddleInference; +using Sdcb.PaddleOCR.Models; +using Sdcb.PaddleOCR.Models.LocalV3; +using Sdcb.PaddleOCR; +using System.Threading.Tasks; +using BotSharp.Abstraction.Knowledges; +using static System.Net.WebRequestMethods; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig; +using System.Linq; +using static System.Net.Mime.MediaTypeNames; + +using Docnet; +using Docnet.Core.Models; +using Docnet.Core; +using Docnet.Core.Converters; +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; + +namespace BotSharp.Plugin.PaddleSharp.Providers; + +public class Pdf2TextConverter : IPdf2TextConverter +{ + private Dictionary _mappings = new Dictionary(); + private FullOcrModel _model = LocalFullModels.EnglishV3; + private string? _tempFolderPath; + private PaddleOcrAll _paddleSettings; + private MagickReadSettings _magicReadSettings; + private int _consumerCount; + private int _boundedCapacity; + private double _acceptScore; + + public Pdf2TextConverter(PaddleOcrAll paddleSettings, MagickReadSettings magicReadSettings, + double acceptScore = 0.8, int consumerCount = 1, int boundedCapacity = 64) + { + _paddleSettings = paddleSettings; + _magicReadSettings = magicReadSettings; + _consumerCount = consumerCount; + _boundedCapacity = boundedCapacity; + _acceptScore = acceptScore; + } + + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel = true) + { + string pdfContent; + if (paddleModel) + { + await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); + pdfContent = LocalImageToTextsAsync().Result; + } + else + { + pdfContent = await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); + } + return pdfContent; + } + + public async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + if (formFile.Length <= 0) + { + return await Task.FromResult(string.Empty); + } + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + var document = PdfDocument.Open(filePath); + var content = ""; + foreach (Page page in document.GetPages()) + { + if (startPageNum.HasValue && page.Number < startPageNum.Value) + { + continue; + } + + if (endPageNum.HasValue && page.Number > endPageNum.Value) + { + continue; + } + + content += page.Text; + } + + return content; + } + + public async Task LocalImageToTextsAsync() + { + string loadPath; + string contents = ""; + if (!System.IO.File.Exists(_tempFolderPath)) + { + throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); + } + + using QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model) + { + AllowRotateDetection = _paddleSettings.AllowRotateDetection, + Enable180Classification = _paddleSettings.Enable180Classification, + }, consumerCount: _consumerCount, boundedCapacity: _boundedCapacity); + + foreach (var item in _mappings.OrderBy(x => x.Key)) + { + loadPath = Path.Combine(_tempFolderPath, item.Value); + using (Mat src = Cv2.ImRead(loadPath)) + { + PaddleOcrResult result = await all.Run(src); + + foreach (PaddleOcrResultRegion region in result.Regions) + { + if (region.Score > _acceptScore) + { + contents += region.Text; + } + } + } + } + + DeleteTempFolder(); + + return contents; + } + + public void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum) + { + // This function is pending. I am considering if we could include Both "ImageMagick" and "Docnet.Core" + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + formFile.CopyTo(stream); + } + } + + private static void AddBytes(Bitmap bmp, byte[] rawBytes) + { + var rect = new Rectangle(0, 0, bmp.Width, bmp.Height); + + var bmpData = bmp.LockBits(rect, ImageLockMode.WriteOnly, bmp.PixelFormat); + var pNative = bmpData.Scan0; + + Marshal.Copy(rawBytes, 0, pNative, rawBytes.Length); + bmp.UnlockBits(bmpData); + } + + public void DocnetConverter(string filePath, int width = 1080, int height = 1920) + { + var pageSettings = new PageDimensions(width, height); + + // using (var docReader = DocLib.Instance.GetDocReader("C:\\Users\\104199\\Postman\\files\\WM2077CW.pdf", new PageDimensions(1080, 1920))) + using (var docReader = DocLib.Instance.GetDocReader(filePath, pageSettings)) + { + using (var pageReader = docReader.GetPageReader(17)) + { + var rawBytes = pageReader.GetImage(); + var pageWidth = pageReader.GetPageWidth(); + var pageHeight = pageReader.GetPageHeight(); + var characters = pageReader.GetCharacters(); + + using (var bmp = new Bitmap(pageWidth, pageHeight, PixelFormat.Format32bppArgb)) + { + AddBytes(bmp, rawBytes); + + using (var imageStream = new MemoryStream()) + { + //saving and exporting + bmp.Save(imageStream, ImageFormat.Png); + System.IO.File.WriteAllBytes(filePath, imageStream.ToArray()); + }; + } + } + }; + } + + public async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + string rootFileName; + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + using var images = new MagickImageCollection(); + // _magicReadSettings.Density = new Density((double)300); + /* + using var images = new MagickImageCollection(); + MagickNET.SetGhostscriptDirectory("C:\\Users\\104199\\Downloads\\ghostpcl-10.01.2-win64\\ghostpcl-10.01.2-win64"); + + images.Read("C:\\Users\\104199\\Postman\\files\\page12.pdf", new MagickReadSettings + { + Density = new Density(300, 300) + }); + */ + images.Read(filePath, new MagickReadSettings + { + Density = new Density(300, 300) + }); + + if (images.Count == 0) + { + throw new Exception("PDF loading failed. Please check if the PDF format is correct!"); + } + + startPageNum = startPageNum.HasValue ? startPageNum : 1; + endPageNum = endPageNum.HasValue ? endPageNum : images.Count; + + for (int page = (int)startPageNum; page <= (int)endPageNum; page++) + { + string tempFileName = Path.GetTempFileName(); + rootFileName = Path.Combine(_tempFolderPath, $"{tempFileName}_Page{page}.png"); + + // image.Format = MagickFormat.Jpg; Set to "Jpg" format + images[page].Write(rootFileName); + + _mappings[page] = rootFileName; + } + } + + public void DeleteTempFolder(string filePath = "") + { + if (!string.IsNullOrEmpty(filePath)) + { + Directory.Delete(_tempFolderPath); + } + else + { + Directory.Delete(_tempFolderPath); + _tempFolderPath = string.Empty; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs new file mode 100644 index 00000000..05592e91 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Sdcb.PaddleOCR; +using ImageMagick; + +namespace BotSharp.Plugin.PaddleSharp.Settings +{ + public class PaddleSharpSettings + { + public MagickReadSettings magickReadSettings { get; set; } + public PaddleOcrAll paddleOcrAll { get; set; } + } +} diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 606ec0b2..0acf233d 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -81,14 +81,15 @@ "WeixinAppSecret": "#{WeixinAppSecret}#" }, - "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - // "VectorDb": "QdrantDb", - "TextEmbedding": "fastTextEmbeddingProvider", - // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider" - // "TextCompletion": "LLamaSharp.TextCompletionProvider" - }, + "KnowledgeBase": { + "VectorDb": "MemVectorDatabase", + // "VectorDb": "QdrantDb", + "TextEmbedding": "fastTextEmbeddingProvider", + // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", + "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", + // "TextCompletion": "LLamaSharp.TextCompletionProvider", + "Pdf2TextConverter": "" + }, "PluginLoader": { "Assemblies": [ From 76ca3ffdb57bae95955372df3a6f12b6c359252b Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Wed, 23 Aug 2023 12:42:36 -0500 Subject: [PATCH 2/6] Add pdf2textconverter using paddlesharp --- .../Knowledges/IPdf2TextConverter.cs | 6 +- .../Controllers/KnowledgeController.cs | 37 ++------ .../BotSharp.Plugin.PaddleSharp.csproj | 3 + .../PaddleSharpPlugin.cs | 8 +- .../Providers/Pdf2TextConverter.cs | 91 ++++++++----------- .../Settings/PaddleSharpSettings.cs | 23 +++++ 6 files changed, 82 insertions(+), 86 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index af6da85d..48241a18 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -7,11 +7,11 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel); + Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum); Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); Task LocalImageToTextsAsync(); Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); - void DeleteTempFolder(string filePath = ""); + // void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); + void DeleteTempFile(string filePath); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs index 4d63885e..8506c0b4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs @@ -11,11 +11,12 @@ namespace BotSharp.OpenAPI.Controllers; public class KnowledgeController : ControllerBase, IApiAdapter { private readonly IKnowledgeService _knowledgeService; - public KnowledgeController(IKnowledgeService knowledgeService) + private readonly IPdf2TextConverter _pdf2TextConverter; + public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter) { _knowledgeService = knowledgeService; + _pdf2TextConverter = pdf2TextConverter; } - [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) { @@ -27,44 +28,18 @@ public class KnowledgeController : ControllerBase, IApiAdapter } [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) { long size = files.Sum(f => f.Length); foreach (var formFile in files) { - if (formFile.Length <= 0) - { - continue; - } - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - await formFile.CopyToAsync(stream); - } - - var document = PdfDocument.Open(filePath); var content = ""; - foreach (Page page in document.GetPages()) - { - if (startPageNum.HasValue && page.Number < startPageNum.Value) - { - continue; - } - - if (endPageNum.HasValue && page.Number > endPageNum.Value) - { - continue; - } - - content += page.Text; - } + + content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); // Process uploaded files // Don't rely on or trust the FileName property without validation. - await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 5d111ae1..7693c483 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -12,9 +12,12 @@ + + + diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs index f679c4e7..94796b64 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/PaddleSharpPlugin.cs @@ -1,4 +1,7 @@ +using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.Plugins; +using BotSharp.Plugin.PaddleSharp.Providers; +using BotSharp.Plugin.PaddleSharp.Settings; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using System; @@ -9,6 +12,9 @@ public class PaddleSharpPlugin : IBotSharpPlugin { public void RegisterDI(IServiceCollection services, IConfiguration config) { - + var settings = new PaddleSharpSettings(); + config.Bind("PaddleSharp", settings); + services.AddSingleton(x => settings); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 8c6dbfed..87fb0efe 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -17,7 +17,6 @@ using UglyToad.PdfPig.Content; using UglyToad.PdfPig; using System.Linq; using static System.Net.Mime.MediaTypeNames; - using Docnet; using Docnet.Core.Models; using Docnet.Core; @@ -25,37 +24,39 @@ using Docnet.Core.Converters; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; +using Microsoft.Extensions.DependencyInjection; +using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter { + // private readonly IServiceProvider _service; + private Dictionary _mappings = new Dictionary(); + /* + // private FullOcrModel _model; + private string? _tempFolderPath = Path.GetTempPath(); private FullOcrModel _model = LocalFullModels.EnglishV3; - private string? _tempFolderPath; - private PaddleOcrAll _paddleSettings; private MagickReadSettings _magicReadSettings; private int _consumerCount; private int _boundedCapacity; private double _acceptScore; - - public Pdf2TextConverter(PaddleOcrAll paddleSettings, MagickReadSettings magicReadSettings, - double acceptScore = 0.8, int consumerCount = 1, int boundedCapacity = 64) + */ + private FullOcrModel _model = LocalFullModels.EnglishV3; + private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { - _paddleSettings = paddleSettings; - _magicReadSettings = magicReadSettings; - _consumerCount = consumerCount; - _boundedCapacity = boundedCapacity; - _acceptScore = acceptScore; + _paddleSharpSettings = paddleSharpSettings; } - public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum, bool paddleModel = true) + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) { string pdfContent; - if (paddleModel) + if (_paddleSharpSettings.paddleModel) { await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); - pdfContent = LocalImageToTextsAsync().Result; + pdfContent = await LocalImageToTextsAsync(); } else { @@ -102,51 +103,47 @@ public class Pdf2TextConverter : IPdf2TextConverter { string loadPath; string contents = ""; - if (!System.IO.File.Exists(_tempFolderPath)) + if (!Directory.Exists(_paddleSharpSettings.tempFolderPath)) { throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); } - using QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model) - { - AllowRotateDetection = _paddleSettings.AllowRotateDetection, - Enable180Classification = _paddleSettings.Enable180Classification, - }, consumerCount: _consumerCount, boundedCapacity: _boundedCapacity); + // var converter = _service.GetRequiredService(); + QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn()) + { + AllowRotateDetection = true, + Enable180Classification = false, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + + foreach (var item in _mappings.OrderBy(x => x.Key)) { - loadPath = Path.Combine(_tempFolderPath, item.Value); + loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value); + // var pdfContent = converter.ConvertImageToText(loadPath); + // contents += pdfContent; + using (Mat src = Cv2.ImRead(loadPath)) { PaddleOcrResult result = await all.Run(src); foreach (PaddleOcrResultRegion region in result.Regions) { - if (region.Score > _acceptScore) + if (region.Score > _paddleSharpSettings.acceptScore) { - contents += region.Text; + contents += region.Text + " "; } } } + + // Delete related Temp files after converting image to texts + // DeleteTempFile(loadPath); } - - DeleteTempFolder(); - + // await Console.Out.WriteLineAsync("Finished!"); + // all.Dispose(); return contents; } - public void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum) - { - // This function is pending. I am considering if we could include Both "ImageMagick" and "Docnet.Core" - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - formFile.CopyTo(stream); - } - } - private static void AddBytes(Bitmap bmp, byte[] rawBytes) { var rect = new Rectangle(0, 0, bmp.Width, bmp.Height); @@ -224,26 +221,18 @@ public class Pdf2TextConverter : IPdf2TextConverter for (int page = (int)startPageNum; page <= (int)endPageNum; page++) { - string tempFileName = Path.GetTempFileName(); - rootFileName = Path.Combine(_tempFolderPath, $"{tempFileName}_Page{page}.png"); + string tempFileName = Path.GetRandomFileName(); + tempFileName = Path.ChangeExtension(tempFileName, "png"); + rootFileName = Path.Combine(_paddleSharpSettings.tempFolderPath, tempFileName); // image.Format = MagickFormat.Jpg; Set to "Jpg" format images[page].Write(rootFileName); - _mappings[page] = rootFileName; } } - public void DeleteTempFolder(string filePath = "") + public void DeleteTempFile(string filePath) { - if (!string.IsNullOrEmpty(filePath)) - { - Directory.Delete(_tempFolderPath); - } - else - { - Directory.Delete(_tempFolderPath); - _tempFolderPath = string.Empty; - } + System.IO.File.Delete(filePath); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs index 05592e91..39c99343 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Settings/PaddleSharpSettings.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.Text; using Sdcb.PaddleOCR; using ImageMagick; +using Sdcb.PaddleOCR.Models; +using System.IO; +using Sdcb.PaddleInference; namespace BotSharp.Plugin.PaddleSharp.Settings { @@ -10,5 +13,25 @@ namespace BotSharp.Plugin.PaddleSharp.Settings { public MagickReadSettings magickReadSettings { get; set; } public PaddleOcrAll paddleOcrAll { get; set; } + public string tempFolderPath { get; set; } = Path.GetTempPath(); + public PaddleOcrAll paddleSettings { get; set; } + public MagickReadSettings magicReadSettings + { + get + { + return magicReadSettings; + } + set + { + magicReadSettings.Density = new Density(300, 300); + } + } + public int consumerCount { get; set; } = 1; + public int boundedCapacity { get; set; } = 64; + public double acceptScore { get; set; } + public Action device { get; set; } = PaddleDevice.Mkldnn(); + public bool allowRotateDetection { get; set; } + public bool enable180Classification { get; set; } + public bool paddleModel { get; set; } = true; } } From e4d8524376f8991c885f5a95924dafd2da272a64 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Sun, 27 Aug 2023 06:07:22 -0500 Subject: [PATCH 3/6] Add disabled property to routing table. --- .../BotSharp.Abstraction/Agents/Models/RoutingRecord.cs | 3 +++ src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj | 4 ++++ src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs index 8f83c74b..3d8e1868 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs @@ -19,6 +19,9 @@ public class RoutingRecord [JsonPropertyName("redirect_to")] public string RedirectTo { get; set; } + [JsonPropertyName("disabled")] + public bool Disabled { get; set; } + public override string ToString() { return Name; diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ab77723b..20b036e4 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -82,4 +82,8 @@ + + + + diff --git a/src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs b/src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs index e030a904..559445a9 100644 --- a/src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs @@ -10,7 +10,9 @@ public class AgentHook : AgentHookBase public override bool OnInstructionLoaded(string template, Dictionary dict) { var router = _services.GetRequiredService(); - dict["routing_records"] = router.GetRoutingRecords(); + dict["routing_records"] = router.GetRoutingRecords() + .Where(x => !x.Disabled) + .ToList(); return true; } } From 6f5cf2fcae9711259dfac32105aab70290482097 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Sun, 27 Aug 2023 22:50:10 -0500 Subject: [PATCH 4/6] Draft of Reasoning --- .../Agents/IAgentRouting.cs | 5 +- .../Agents/Settings/AgentSettings.cs | 7 +- .../Conversations/Models/RoleDialogModel.cs | 2 +- .../Settings/ConversationSetting.cs | 2 + .../Functions/Models/FunctionCallFromLlm.cs | 14 + .../Routing/Models/RetrievalArgs.cs | 16 ++ .../{Agents => Routing}/Models/RoutingArgs.cs | 2 +- .../Models/RoutingRecord.cs | 2 +- .../Routing/Settings/GPT4Settings.cs | 8 + .../Agents/Services/AgentRouter.cs | 53 ---- .../BotSharp.Core/BotSharp.Core.csproj | 4 - .../BotSharpServiceCollectionExtensions.cs | 14 +- ...vice.GetChatCompletionsAsyncRecursively.cs | 5 +- .../ConversationService.SendMessage.cs | 43 +++- .../Services/ConversationStorage.cs | 1 + .../BotSharp.Core/Functions/RouteToAgentFn.cs | 5 +- .../BotSharp.Core/Hooks/ReasoningHook.cs | 14 + .../Hooks/{AgentHook.cs => RoutingHook.cs} | 4 +- .../BotSharp.Core/Routing/Reasoner.cs | 12 + .../BotSharp.Core/Routing/Router.cs | 43 ++++ .../BotSharp.Core/Routing/Simulator.cs | 133 ++++++++++ .../Templating/TemplateRender.cs | 3 +- .../AzureOpenAiPlugin.cs | 1 + .../Providers/GPT4CompletionProvider.cs | 240 ++++++++++++++++++ .../Settings/AzureOpenAiSettings.cs | 4 + 25 files changed, 560 insertions(+), 77 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs rename src/Infrastructure/BotSharp.Abstraction/{Agents => Routing}/Models/RoutingArgs.cs (82%) rename src/Infrastructure/BotSharp.Abstraction/{Agents => Routing}/Models/RoutingRecord.cs (92%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/Settings/GPT4Settings.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs create mode 100644 src/Infrastructure/BotSharp.Core/Hooks/ReasoningHook.cs rename src/Infrastructure/BotSharp.Core/Hooks/{AgentHook.cs => RoutingHook.cs} (77%) create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Reasoner.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Router.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Simulator.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs index 4df491db..d1086c26 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Routing.Models; + namespace BotSharp.Abstraction.Agents; public interface IAgentRouting { + string AgentId { get; } Task LoadRouter(); - Task LoadCurrentAgent(); RoutingRecord[] GetRoutingRecords(); + RoutingRecord GetRecordByName(string name); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs index 84ff0274..60307439 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs @@ -6,7 +6,12 @@ public class AgentSettings /// Router Agent Id /// public string RouterId { get; set; } + + /// + /// Reasoner Agent Id + /// + public string ReasonerId { get; set; } + public string DataDir { get; set; } public string TemplateFormat { get; set; } - public int MaxRecursiveDepth { get; set; } = 3; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index b681c2c3..e80c9427 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -45,7 +45,7 @@ public class RoleDialogModel { if (Role == AgentRole.Function) { - return $"{Role}: {FunctionName}"; + return $"{Role}: {FunctionName} => {ExecutionResult}"; } else { diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs index 80897657..c9836d4b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs @@ -6,4 +6,6 @@ public class ConversationSetting public string ChatCompletion { get; set; } public bool EnableKnowledgeBase { get; set; } public bool ShowVerboseLog { get; set; } + public int MaxRecursiveDepth { get; set; } = 3; + public bool EnableReasoning { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs new file mode 100644 index 00000000..5d9b9881 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -0,0 +1,14 @@ +using BotSharp.Abstraction.Routing.Models; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace BotSharp.Abstraction.Functions.Models; + +public class FunctionCallFromLlm +{ + [JsonPropertyName("function")] + public string Function { get; set; } + + [JsonPropertyName("parameters")] + public RetrievalArgs Parameters { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs new file mode 100644 index 00000000..835cda46 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs @@ -0,0 +1,16 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace BotSharp.Abstraction.Routing.Models; + +public class RetrievalArgs : RoutingArgs +{ + [JsonPropertyName("question")] + public string Question { get; set; } + + [JsonPropertyName("reason")] + public string Reason { get; set; } + + [JsonPropertyName("args")] + public JsonDocument Arguments { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs similarity index 82% rename from src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingArgs.cs rename to src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index e02b1c7b..e70d787d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace BotSharp.Abstraction.Agents.Models; +namespace BotSharp.Abstraction.Routing.Models; public class RoutingArgs { diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRecord.cs similarity index 92% rename from src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs rename to src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRecord.cs index 3d8e1868..f0d6d742 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/RoutingRecord.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingRecord.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace BotSharp.Abstraction.Agents.Models; +namespace BotSharp.Abstraction.Routing.Models; public class RoutingRecord { diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/GPT4Settings.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/GPT4Settings.cs new file mode 100644 index 00000000..21925c1c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/GPT4Settings.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Routing.Settings; + +public class GPT4Settings +{ + public string ApiKey { get; set; } + public string Endpoint { get; set; } + public string DeploymentModel { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs deleted file mode 100644 index 0aed1531..00000000 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentRouter.cs +++ /dev/null @@ -1,53 +0,0 @@ -using BotSharp.Abstraction.Agents.Models; -using System.IO; - -namespace BotSharp.Core.Agents.Services; - -public class AgentRouter : IAgentRouting -{ - private readonly IServiceProvider _services; - private readonly ILogger _logger; - private readonly AgentSettings _settings; - - public AgentRouter(IServiceProvider services, - ILogger logger, - AgentSettings settings) - { - _services = services; - _logger = logger; - _settings = settings; - } - - public async Task LoadRouter() - { - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(_settings.RouterId); - return agent; - } - - public async Task LoadCurrentAgent() - { - // Load current agent from state - var state = _services.GetRequiredService(); - var currentAgentId = state.GetState("agent_id"); - if (string.IsNullOrEmpty(currentAgentId)) - { - currentAgentId = _settings.RouterId; - } - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(currentAgentId); - - // Set agent and trigger state changed - state.SetState("agent_id", currentAgentId); - - return agent; - } - - public RoutingRecord[] GetRoutingRecords() - { - var agentSettings = _services.GetRequiredService(); - var dbSettings = _services.GetRequiredService(); - var filePath = Path.Combine(dbSettings.FileRepository, agentSettings.DataDir, agentSettings.RouterId, "route.json"); - return JsonSerializer.Deserialize(File.ReadAllText(filePath)); - } -} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 20b036e4..ab77723b 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -82,8 +82,4 @@ - - - - diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 1b38659c..b200c729 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions; using BotSharp.Core.Functions; using BotSharp.Core.Hooks; +using BotSharp.Core.Routing; using BotSharp.Core.Templating; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; @@ -41,13 +43,21 @@ public static class BotSharpServiceCollectionExtensions services.AddSingleton(); // Register router - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(p => + { + var setting = p.GetRequiredService(); + return setting.EnableReasoning ? p.GetRequiredService() : p.GetRequiredService(); + }); // Register function callback services.AddScoped(); // Register Hooks - services.AddScoped(); + services.AddScoped(); + + services.AddScoped(); return services; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs index 40856809..4e5a0d23 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs @@ -13,13 +13,12 @@ public partial class ConversationService string conversationId, Agent agent, List wholeDialogs, - int maxRecursiveDepth, Func onMessageReceived, Func onFunctionExecuting, Func onFunctionExecuted) { currentRecursiveDepth++; - if (currentRecursiveDepth > maxRecursiveDepth) + if (currentRecursiveDepth > _settings.MaxRecursiveDepth) { _logger.LogWarning($"Exceeded max recursive depth."); @@ -65,7 +64,6 @@ public partial class ConversationService fn.Content = fn.FunctionArgs.Replace("\r", " ").Replace("\n", " ").Trim() + " => " + fn.ExecutionResult; // Agent has been transferred - var agentSettings = _services.GetRequiredService(); if (fn.CurrentAgentId != preAgentId) { var agentService = _services.GetRequiredService(); @@ -83,7 +81,6 @@ public partial class ConversationService conversationId, agent, wholeDialogs, - maxRecursiveDepth, onMessageReceived, onFunctionExecuting, onFunctionExecuted); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 8d53971a..c05406a5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,5 +1,8 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.MLTasks; +using BotSharp.Core.Routing; namespace BotSharp.Core.Conversations.Services; @@ -31,7 +34,7 @@ public partial class ConversationService stateService.SetState("channel", lastDialog.Channel); var router = _services.GetRequiredService(); - var agent = await router.LoadRouter(); + Agent agent = await router.LoadRouter(); _logger.LogInformation($"[{agent.Name}] {lastDialog.Role}: {lastDialog.Content}"); @@ -65,14 +68,42 @@ public partial class ConversationService await hook.BeforeCompletion(); } - var agentSettings = _services.GetRequiredService(); + // reasoning + if (_settings.EnableReasoning) + { + var simulator = _services.GetRequiredService(); + var reasonedContext = await simulator.Enter(agent, wholeDialogs); + + if (reasonedContext.FunctionName == "interrupt_task_execution") + { + await HandleAssistantMessage(new RoleDialogModel(AgentRole.Assistant, reasonedContext.Content) + { + CurrentAgentId = agent.Id, + Channel = lastDialog.Channel + }, onMessageReceived); + return true; + } + else if (reasonedContext.FunctionName == "continue_execute_task") + { + if (reasonedContext.CurrentAgentId != agent.Id) + { + var agentService = _services.GetRequiredService(); + agent = await agentService.LoadAgent(reasonedContext.CurrentAgentId); + } + } + + simulator.Dialogs.ForEach(x => + { + wholeDialogs.Add(x); + _storage.Append(conversationId, agent.Id, x); + }); + } var chatCompletion = GetChatCompletion(); var result = await GetChatCompletionsAsyncRecursively(chatCompletion, conversationId, agent, wholeDialogs, - agentSettings.MaxRecursiveDepth, onMessageReceived, onFunctionExecuting, onFunctionExecuted); @@ -101,4 +132,10 @@ public partial class ConversationService var completions = _services.GetServices(); return completions.FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.ChatCompletion)); } + + public IChatCompletion GetGpt4ChatCompletion() + { + var completions = _services.GetServices(); + return completions.FirstOrDefault(x => x.GetType().FullName.EndsWith("GPT4CompletionProvider")); + } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 8e0b08dd..7525532b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -72,6 +72,7 @@ public class ConversationStorage : IConversationStorage CurrentAgentId = currentAgentId, FunctionName = funcName, FunctionArgs = funcArgs, + ExecutionResult = text, CreatedAt = createdAt }); } diff --git a/src/Infrastructure/BotSharp.Core/Functions/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Functions/RouteToAgentFn.cs index 3834b035..4991dc99 100644 --- a/src/Infrastructure/BotSharp.Core/Functions/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Functions/RouteToAgentFn.cs @@ -1,6 +1,6 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Routing.Models; using System.IO; namespace BotSharp.Core.Functions; @@ -51,8 +51,7 @@ public class RouteToAgentFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); var router = _services.GetRequiredService(); - var records = router.GetRoutingRecords(); - var routingRule = records.FirstOrDefault(x => x.Name.ToLower() == args.AgentName.ToLower()); + var routingRule = router.GetRecordByName(args.AgentName); if (routingRule == null) { diff --git a/src/Infrastructure/BotSharp.Core/Hooks/ReasoningHook.cs b/src/Infrastructure/BotSharp.Core/Hooks/ReasoningHook.cs new file mode 100644 index 00000000..f754733f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Hooks/ReasoningHook.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Core.Hooks; + +public class ReasoningHook : AgentHookBase +{ + public ReasoningHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + + public override bool OnInstructionLoaded(string template, Dictionary dict) + { + return true; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs b/src/Infrastructure/BotSharp.Core/Hooks/RoutingHook.cs similarity index 77% rename from src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs rename to src/Infrastructure/BotSharp.Core/Hooks/RoutingHook.cs index 559445a9..aba0ffc2 100644 --- a/src/Infrastructure/BotSharp.Core/Hooks/AgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Hooks/RoutingHook.cs @@ -1,8 +1,8 @@ namespace BotSharp.Core.Hooks; -public class AgentHook : AgentHookBase +public class RoutingHook : AgentHookBase { - public AgentHook(IServiceProvider services, AgentSettings settings) + public RoutingHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoner.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoner.cs new file mode 100644 index 00000000..c08f83ac --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoner.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Core.Routing; + +public class Reasoner : Router +{ + public override string AgentId => _settings.ReasonerId; + + public Reasoner(IServiceProvider services, + ILogger logger, + AgentSettings settings) : base(services, logger, settings) + { + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Router.cs b/src/Infrastructure/BotSharp.Core/Routing/Router.cs new file mode 100644 index 00000000..594ee734 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Router.cs @@ -0,0 +1,43 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Routing.Models; +using System.IO; +using static Tensorflow.ApiDef.Types; + +namespace BotSharp.Core.Routing; + +public class Router : IAgentRouting +{ + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + protected readonly AgentSettings _settings; + + public virtual string AgentId => _settings.RouterId; + + public Router(IServiceProvider services, + ILogger logger, + AgentSettings settings) + { + _services = services; + _logger = logger; + _settings = settings; + } + + public virtual async Task LoadRouter() + { + var agentService = _services.GetRequiredService(); + return await agentService.LoadAgent(AgentId); + } + + public RoutingRecord[] GetRoutingRecords() + { + var agentSettings = _services.GetRequiredService(); + var dbSettings = _services.GetRequiredService(); + var filePath = Path.Combine(dbSettings.FileRepository, agentSettings.DataDir, agentSettings.RouterId, "route.json"); + return JsonSerializer.Deserialize(File.ReadAllText(filePath)); + } + + public RoutingRecord GetRecordByName(string name) + { + return GetRoutingRecords().First(x => x.Name.ToLower() == name.ToLower()); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs b/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs new file mode 100644 index 00000000..d5c3d935 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs @@ -0,0 +1,133 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Routing; + +/// +/// Simulate the dialogue between different agents. +/// +public class Simulator +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private List _dialogs; + public List Dialogs => _dialogs; + + public Simulator(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Enter(Agent agent, List whileDialogs) + { + _dialogs = new List(); + + foreach (var dialog in whileDialogs.TakeLast(10)) + { + agent.Instruction += $"\r\n{dialog.Role}: {dialog.Content}"; + } + + var response = await SendMessageToReasoner(agent); + var args = JsonSerializer.Deserialize(response.Content); + response.FunctionName = args.Function; + response.Content = args.Parameters.Reason; + if (args.Function == "continue_execute_task") + { + response.FunctionArgs = JsonSerializer.Serialize(args.Parameters.Arguments); + + var router = _services.GetRequiredService(); + var record = router.GetRecordByName(args.Parameters.AgentName); + response.CurrentAgentId = record.AgentId; + } + + return response; + } + + private async Task SendMessageToReasoner(Agent reasoner) + { + var wholeDialogs = new List + { + new RoleDialogModel(AgentRole.User, @"What's the next step, your response must be in JSON format with ""function"" and ""parameters"". ") + }; + + var chatCompletion = GetGpt4ChatCompletion(); + + RoleDialogModel response = null; + await chatCompletion.GetChatCompletionsAsync(reasoner, wholeDialogs, async msg + => response = msg, fn + => Task.CompletedTask); + + var args = JsonSerializer.Deserialize(response.Content); + + SaveStateByArgs(args.Parameters.Arguments); + + // Retrieve information from specific agent + var router = _services.GetRequiredService(); + var record = router.GetRecordByName(args.Parameters.AgentName); + response = await SendMessageToAgent(record.AgentId, new List + { + new RoleDialogModel(AgentRole.User, args.Parameters.Question) + }); + + _dialogs.Add(new RoleDialogModel(AgentRole.Function, $"{record.Name}: {response.Content}") + { + FunctionName = args.Function, + FunctionArgs = JsonSerializer.Serialize(args.Parameters.Arguments), + ExecutionResult = response.Content + }); + + reasoner.Instruction += $"\r\n{record.Name}: {response.Content}"; + // Got the response from agent, then send to reasoner again to make the decision + await chatCompletion.GetChatCompletionsAsync(reasoner, wholeDialogs, async msg + => response = msg, fn + => Task.CompletedTask); + + return response; + } + + private async Task SendMessageToAgent(string agentId, List wholeDialogs) + { + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(agentId); + + var chatCompletion = GetChatCompletion(); + + RoleDialogModel response = null; + await chatCompletion.GetChatCompletionsAsync(agent, wholeDialogs, async msg + => response = msg, fn + => Task.CompletedTask); + return response; + } + + public IChatCompletion GetChatCompletion() + { + var completions = _services.GetServices(); + var settings = _services.GetRequiredService(); + return completions.FirstOrDefault(x => x.GetType().FullName.EndsWith(settings.ChatCompletion)); + } + + public IChatCompletion GetGpt4ChatCompletion() + { + var completions = _services.GetServices(); + return completions.FirstOrDefault(x => x.GetType().FullName.EndsWith("GPT4CompletionProvider")); + } + + private void SaveStateByArgs(JsonDocument args) + { + var stateService = _services.GetRequiredService(); + if (args.RootElement is JsonElement root) + { + foreach (JsonProperty property in root.EnumerateObject()) + { + if (!string.IsNullOrEmpty(property.Value.ToString())) + { + stateService.SetState(property.Name, property.Value.ToString()); + } + } + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs index ca80356a..97263119 100644 --- a/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs +++ b/src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Templating; using Fluid; using Microsoft.Extensions.Options; @@ -17,7 +18,7 @@ public class TemplateRender : ITemplateRender _services = services; _logger = logger; _options = new TemplateOptions(); - _options.MemberAccessStrategy.MemberNameStrategy = MemberNameStrategies.CamelCase; + _options.MemberAccessStrategy.MemberNameStrategy = MemberNameStrategies.SnakeCase; _options.MemberAccessStrategy.Register(); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index 1232e668..b966d167 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -23,5 +23,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs new file mode 100644 index 00000000..8767e7df --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs @@ -0,0 +1,240 @@ +using Azure; +using Azure.AI.OpenAI; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Conversations.Settings; +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class GPT4CompletionProvider : IChatCompletion +{ + private readonly AzureOpenAiSettings _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public GPT4CompletionProvider(AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) + { + _settings = settings; + _logger = logger; + _services = services; + } + + private OpenAIClient GetClient() + { + var client = new OpenAIClient(new Uri(_settings.GPT4.Endpoint), new AzureKeyCredential(_settings.GPT4.ApiKey)); + return client; + } + + public List GetChatSamples(string sampleText) + { + var samples = new List(); + if (string.IsNullOrEmpty(sampleText)) + { + return samples; + } + + var lines = sampleText.Split('\n'); + for (int i = 0; i < lines.Length; i++) + { + var line = lines[i]; + if (string.IsNullOrEmpty(line.Trim())) + { + continue; + } + var role = line.Substring(0, line.IndexOf(' ') - 1).Trim(); + var content = line.Substring(line.IndexOf(' ') + 1).Trim(); + + // comments + if (role == "##") + { + continue; + } + + samples.Add(new RoleDialogModel(role, content)); + } + + return samples; + } + + public List GetFunctions(string functionsJson) + { + var functions = new List(); + if (!string.IsNullOrEmpty(functionsJson)) + { + functions = JsonSerializer.Deserialize>(functionsJson, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + AllowTrailingCommas = true + }); + } + + return functions; + } + + public async Task GetChatCompletionsAsync(Agent agent, + List conversations, + Func onMessageReceived, + Func onFunctionExecuting) + { + var client = GetClient(); + var chatCompletionsOptions = PrepareOptions(agent, conversations); + + var response = await client.GetChatCompletionsAsync(_settings.GPT4.DeploymentModel, chatCompletionsOptions); + var choice = response.Value.Choices[0]; + var message = choice.Message; + + if (choice.FinishReason == CompletionsFinishReason.FunctionCall) + { + _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name} => {message.FunctionCall.Arguments}"); + + var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content) + { + CurrentAgentId = agent.Id, + FunctionName = message.FunctionCall.Name, + FunctionArgs = message.FunctionCall.Arguments, + Channel = conversations.Last().Channel + }; + + // Execute functions + await onFunctionExecuting(funcContextIn); + } + else + { + _logger.LogInformation($"[{agent.Name}] {message.Role}: {message.Content}"); + + var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) + { + CurrentAgentId= agent.Id, + Channel = conversations.Last().Channel + }; + + // Text response received + await onMessageReceived(msg); + } + + return true; + } + + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + { + var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey)); + var chatCompletionsOptions = PrepareOptions(agent, conversations); + + var response = await client.GetChatCompletionsStreamingAsync(_settings.DeploymentModel.ChatCompletionModel, chatCompletionsOptions); + using StreamingChatCompletions streaming = response.Value; + + string output = ""; + await foreach (var choice in streaming.GetChoicesStreaming()) + { + if (choice.FinishReason == CompletionsFinishReason.FunctionCall) + { + var args = ""; + await foreach (var message in choice.GetMessageStreaming()) + { + if (message.FunctionCall == null || message.FunctionCall.Arguments == null) + continue; + Console.Write(message.FunctionCall.Arguments); + args += message.FunctionCall.Arguments; + + } + await onMessageReceived(new RoleDialogModel(ChatRole.Assistant.ToString(), args)); + continue; + } + + await foreach (var message in choice.GetMessageStreaming()) + { + if (message.Content == null) + continue; + Console.Write(message.Content); + output += message.Content; + + _logger.LogInformation(message.Content); + + await onMessageReceived(new RoleDialogModel(message.Role.ToString(), message.Content)); + } + + output = ""; + } + + return true; + } + + + private ChatCompletionsOptions PrepareOptions(Agent agent, List conversations) + { + var chatCompletionsOptions = new ChatCompletionsOptions(); + + if (!string.IsNullOrEmpty(agent.Instruction)) + { + chatCompletionsOptions.Messages.Add(new ChatMessage(ChatRole.System, agent.Instruction)); + } + + if (!string.IsNullOrEmpty(agent.Knowledges)) + { + chatCompletionsOptions.Messages.Add(new ChatMessage(ChatRole.System, agent.Knowledges)); + } + + var samples = GetChatSamples(agent.Samples); + foreach (var message in samples) + { + chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content)); + } + + var functions = GetFunctions(agent.Functions); + foreach (var function in functions) + { + chatCompletionsOptions.Functions.Add(new FunctionDefinition + { + Name = function.Name, + Description = function.Description, + Parameters = BinaryData.FromObjectAsJson(function.Parameters) + }); + } + + foreach (var message in conversations) + { + if (message.Role == ChatRole.Function) + { + chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content) + { + Name = message.FunctionName + }); + } + else + { + chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content)); + } + } + + // https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683 + chatCompletionsOptions.Temperature = 0.5f; + chatCompletionsOptions.NucleusSamplingFactor = 0.5f; + + var convSetting = _services.GetRequiredService(); + if (convSetting.ShowVerboseLog) + { + var verbose = string.Join("\n", chatCompletionsOptions.Messages.Select(x => + { + return x.Role == ChatRole.Function ? + $"{x.Role}: {x.Name} {x.Content}" : + $"{x.Role}: {x.Content}"; + })); + _logger.LogInformation(verbose); + } + + return chatCompletionsOptions; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs index 2a3cbf6e..510bec5a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Routing.Settings; + namespace BotSharp.Plugin.AzureOpenAI.Settings; public class AzureOpenAiSettings @@ -6,4 +8,6 @@ public class AzureOpenAiSettings public string Endpoint { get; set; } = string.Empty; public DeploymentModelSetting DeploymentModel { get; set; } = new DeploymentModelSetting(); + + public GPT4Settings GPT4 { get; set; } } From 14d9066f657cce48ad14b34bebe6a0c8fbd0005e Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Mon, 28 Aug 2023 10:24:43 -0500 Subject: [PATCH 5/6] Add PaddleOcrConverter --- .../Knowledges/IPaddleOcrConverter.cs | 12 +++ .../Knowledges/IPdf2TextConverter.cs | 5 -- .../BotSharp.Core/BotSharp.Core.csproj | 1 + .../BotSharpServiceCollectionExtensions.cs | 3 + .../Knowledges/KnowledgeBaseSettings.cs | 1 + .../Services/PigPdf2TextConverter.cs | 49 +++++++++++ .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 4 +- .../Controllers/KnowledgeController.cs | 17 ++-- .../BotSharp.Plugin.PaddleSharp.csproj | 1 - .../Providers/PaddleOcrConverter.cs | 69 +++++++++++++++ .../Providers/Pdf2TextConverter.cs | 84 ++----------------- src/WebStarter/appsettings.json | 2 +- 12 files changed, 155 insertions(+), 93 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs create mode 100644 src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs create mode 100644 src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs new file mode 100644 index 00000000..6c913348 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPaddleOcrConverter.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Abstraction.Knowledges +{ + public interface IPaddleOcrConverter + { + // void LoadModel(); + Task ConvertImageToText(string loadPath); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index 48241a18..0ae85b30 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -8,10 +8,5 @@ namespace BotSharp.Abstraction.Knowledges public interface IPdf2TextConverter { Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum); - Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - Task LocalImageToTextsAsync(); - Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum); - // void ConvertPdfToLocalImages(IFormFile formFile, int? startPageNum, int? endPageNum); - void DeleteTempFile(string filePath); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 6b871db4..e39e4a2e 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -76,6 +76,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 9081ca05..08679545 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Functions; using BotSharp.Core.Functions; +using BotSharp.Core.Plugins.Knowledges.Services; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; @@ -85,5 +86,7 @@ public static class BotSharpServiceCollectionExtensions loader.Load(); services.AddSingleton(loader); + + services.AddSingleton(); } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs index a56767e9..0cb08a48 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs @@ -5,4 +5,5 @@ public class KnowledgeBaseSettings public string VectorDb { get; set; } public string TextEmbedding { get; set; } public string TextCompletion { get; set; } + public string Pdf2TextConverter { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs new file mode 100644 index 00000000..c19ddc77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/PigPdf2TextConverter.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Http; +using UglyToad.PdfPig; +using UglyToad.PdfPig.Content; + +namespace BotSharp.Core.Plugins.Knowledges.Services; + +public class PigPdf2TextConverter : IPdf2TextConverter +{ + public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) + { + return await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); + } + + private async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + { + if (formFile.Length <= 0) + { + return await Task.FromResult(string.Empty); + } + + var filePath = Path.GetTempFileName(); + + using (var stream = System.IO.File.Create(filePath)) + { + await formFile.CopyToAsync(stream); + } + + var document = PdfDocument.Open(filePath); + var content = ""; + foreach (Page page in document.GetPages()) + { + if (startPageNum.HasValue && page.Number < startPageNum.Value) + { + continue; + } + + if (endPageNum.HasValue && page.Number > endPageNum.Value) + { + continue; + } + content += page.Text; + } + return content; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index d1a16a68..7b8290d6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -9,11 +9,11 @@ - + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs index 8506c0b4..3824f476 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs @@ -3,6 +3,8 @@ using BotSharp.Abstraction.Knowledges.Models; using Microsoft.AspNetCore.Http; using UglyToad.PdfPig.Content; using UglyToad.PdfPig; +using BotSharp.Core.Plugins.Knowledges; + namespace BotSharp.OpenAPI.Controllers; @@ -11,11 +13,12 @@ namespace BotSharp.OpenAPI.Controllers; public class KnowledgeController : ControllerBase, IApiAdapter { private readonly IKnowledgeService _knowledgeService; - private readonly IPdf2TextConverter _pdf2TextConverter; - public KnowledgeController(IKnowledgeService knowledgeService, IPdf2TextConverter pdf2TextConverter) + private readonly IServiceProvider _services; + + public KnowledgeController(IKnowledgeService knowledgeService, IServiceProvider services) { _knowledgeService = knowledgeService; - _pdf2TextConverter = pdf2TextConverter; + _services = services; } [HttpGet("/knowledge/{agentId}")] public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) @@ -30,16 +33,20 @@ public class KnowledgeController : ControllerBase, IApiAdapter [HttpPost("/knowledge/{agentId}")] public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); long size = files.Sum(f => f.Length); foreach (var formFile in files) { var content = ""; - - content = await _pdf2TextConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); + + content = await textConverter.ConvertPdfToText(formFile, startPageNum, endPageNum); // Process uploaded files // Don't rely on or trust the FileName property without validation. + + // Add FeedWithMetaData await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj index 7693c483..3ad42c16 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/BotSharp.Plugin.PaddleSharp.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs new file mode 100644 index 00000000..4cf8edb7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/PaddleOcrConverter.cs @@ -0,0 +1,69 @@ +/* +using System; +using System.Collections.Generic; +using System.Text; +using Sdcb.PaddleOCR; +using Sdcb.PaddleOCR.Models; +using Sdcb.PaddleInference; +using Sdcb.PaddleOCR.Models.LocalV3; +using OpenCvSharp; +using System.Threading.Tasks; +using BotSharp.Abstraction.Knowledges; +using BotSharp.Plugin.PaddleSharp.Settings; + +namespace BotSharp.Plugin.PaddleSharp.Providers; + +public class PaddleOcrConverter : IPaddleOcrConverter +{ + private FullOcrModel _paddleFullOcrmodel; + private QueuedPaddleOcrAll _allModel; + private readonly PaddleSharpSettings _paddleSharpSettings; + + public PaddleOcrConverter(FullOcrModel paddleFullOcrmodel, QueuedPaddleOcrAll allModel, PaddleSharpSettings paddleSharpSettings) + { + _paddleFullOcrmodel = paddleFullOcrmodel; + _allModel = allModel; + _paddleSharpSettings = paddleSharpSettings; + } + + private void LoadModel() + { + _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device) + { + AllowRotateDetection = _paddleSharpSettings.allowRotateDetection, + Enable180Classification = _paddleSharpSettings.enable180Classification, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + } + + private void DisposeModel() + { + _allModel.Dispose(); + } + + public async Task ConvertImageToText(string loadPath) + { + _allModel = new(() => new PaddleOcrAll(_paddleFullOcrmodel, _paddleSharpSettings.device) + { + AllowRotateDetection = _paddleSharpSettings.allowRotateDetection, + Enable180Classification = _paddleSharpSettings.enable180Classification, + }, consumerCount: _paddleSharpSettings.consumerCount, boundedCapacity: _paddleSharpSettings.boundedCapacity); + + var contents = ""; + using (Mat src = Cv2.ImRead(loadPath)) + { + PaddleOcrResult result = await _allModel.Run(src); + + foreach (PaddleOcrResultRegion region in result.Regions) + { + if (region.Score > _paddleSharpSettings.acceptScore) + { + contents += region.Text + " "; + } + } + } + + _allModel.Dispose(); + return contents; + } +} +*/ \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 87fb0efe..ab9d6576 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -5,18 +5,13 @@ using System.IO; using ImageMagick; using OpenCvSharp; using Microsoft.AspNetCore.Http; -using System.Runtime.InteropServices.ComTypes; using Sdcb.PaddleInference; using Sdcb.PaddleOCR.Models; using Sdcb.PaddleOCR.Models.LocalV3; using Sdcb.PaddleOCR; using System.Threading.Tasks; using BotSharp.Abstraction.Knowledges; -using static System.Net.WebRequestMethods; -using UglyToad.PdfPig.Content; -using UglyToad.PdfPig; using System.Linq; -using static System.Net.Mime.MediaTypeNames; using Docnet; using Docnet.Core.Models; using Docnet.Core; @@ -24,25 +19,13 @@ using Docnet.Core.Converters; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; -using Microsoft.Extensions.DependencyInjection; using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ - // private readonly IServiceProvider _service; - +{ private Dictionary _mappings = new Dictionary(); - /* - // private FullOcrModel _model; - private string? _tempFolderPath = Path.GetTempPath(); - private FullOcrModel _model = LocalFullModels.EnglishV3; - private MagickReadSettings _magicReadSettings; - private int _consumerCount; - private int _boundedCapacity; - private double _acceptScore; - */ private FullOcrModel _model = LocalFullModels.EnglishV3; private PaddleSharpSettings _paddleSharpSettings; public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) @@ -52,54 +35,11 @@ public class Pdf2TextConverter : IPdf2TextConverter public async Task ConvertPdfToText(IFormFile formFile, int? startPageNum, int? endPageNum) { - string pdfContent; - if (_paddleSharpSettings.paddleModel) - { - await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); - pdfContent = await LocalImageToTextsAsync(); - } - else - { - pdfContent = await OpenPdfDocumentAsync(formFile, startPageNum, endPageNum); - } - return pdfContent; + await ConvertPdfToLocalImagesAsync(formFile, startPageNum, endPageNum); + return await LocalImageToTextsAsync(); } - public async Task OpenPdfDocumentAsync(IFormFile formFile, int? startPageNum, int? endPageNum) - { - if (formFile.Length <= 0) - { - return await Task.FromResult(string.Empty); - } - - var filePath = Path.GetTempFileName(); - - using (var stream = System.IO.File.Create(filePath)) - { - await formFile.CopyToAsync(stream); - } - - var document = PdfDocument.Open(filePath); - var content = ""; - foreach (Page page in document.GetPages()) - { - if (startPageNum.HasValue && page.Number < startPageNum.Value) - { - continue; - } - - if (endPageNum.HasValue && page.Number > endPageNum.Value) - { - continue; - } - - content += page.Text; - } - - return content; - } - - public async Task LocalImageToTextsAsync() + private async Task LocalImageToTextsAsync() { string loadPath; string contents = ""; @@ -108,8 +48,6 @@ public class Pdf2TextConverter : IPdf2TextConverter throw new Exception("No local temporary files found! Please convert PDF to local images first by \"ConvertPdfToLocalImages\"."); } - // var converter = _service.GetRequiredService(); - QueuedPaddleOcrAll all = new(() => new PaddleOcrAll(_model, PaddleDevice.Mkldnn()) { AllowRotateDetection = true, @@ -120,8 +58,6 @@ public class Pdf2TextConverter : IPdf2TextConverter foreach (var item in _mappings.OrderBy(x => x.Key)) { loadPath = Path.Combine(_paddleSharpSettings.tempFolderPath, item.Value); - // var pdfContent = converter.ConvertImageToText(loadPath); - // contents += pdfContent; using (Mat src = Cv2.ImRead(loadPath)) { @@ -135,12 +71,7 @@ public class Pdf2TextConverter : IPdf2TextConverter } } } - - // Delete related Temp files after converting image to texts - // DeleteTempFile(loadPath); } - // await Console.Out.WriteLineAsync("Finished!"); - // all.Dispose(); return contents; } @@ -184,7 +115,7 @@ public class Pdf2TextConverter : IPdf2TextConverter }; } - public async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) + private async Task ConvertPdfToLocalImagesAsync(IFormFile formFile, int? startPageNum, int? endPageNum) { string rootFileName; @@ -230,9 +161,4 @@ public class Pdf2TextConverter : IPdf2TextConverter _mappings[page] = rootFileName; } } - - public void DeleteTempFile(string filePath) - { - System.IO.File.Delete(filePath); - } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 0acf233d..9657f072 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -88,7 +88,7 @@ // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", // "TextCompletion": "LLamaSharp.TextCompletionProvider", - "Pdf2TextConverter": "" + "Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter" }, "PluginLoader": { From 5413f4f71b309529b656891b48a5c9158a77a3f9 Mon Sep 17 00:00:00 2001 From: hchen2020 <101423@smsassist.com> Date: Mon, 28 Aug 2023 10:58:35 -0500 Subject: [PATCH 6/6] response_to_user after reasoning. --- .../Routing/Models/RetrievalArgs.cs | 3 +++ .../ConversationService.SendMessage.cs | 9 ++++++++ .../BotSharp.Core/Routing/Simulator.cs | 21 +++++++++++++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs index 835cda46..e0b3f9a1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RetrievalArgs.cs @@ -8,6 +8,9 @@ public class RetrievalArgs : RoutingArgs [JsonPropertyName("question")] public string Question { get; set; } + [JsonPropertyName("answer")] + public string Answer { get; set; } + [JsonPropertyName("reason")] public string Reason { get; set; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index c05406a5..24376c21 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -83,6 +83,15 @@ public partial class ConversationService }, onMessageReceived); return true; } + else if (reasonedContext.FunctionName == "response_to_user") + { + await HandleAssistantMessage(new RoleDialogModel(AgentRole.Assistant, reasonedContext.Content) + { + CurrentAgentId = agent.Id, + Channel = lastDialog.Channel + }, onMessageReceived); + return true; + } else if (reasonedContext.FunctionName == "continue_execute_task") { if (reasonedContext.CurrentAgentId != agent.Id) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs b/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs index d5c3d935..4b8123d5 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs @@ -34,7 +34,7 @@ public class Simulator var response = await SendMessageToReasoner(agent); var args = JsonSerializer.Deserialize(response.Content); response.FunctionName = args.Function; - response.Content = args.Parameters.Reason; + if (args.Function == "continue_execute_task") { response.FunctionArgs = JsonSerializer.Serialize(args.Parameters.Arguments); @@ -43,6 +43,16 @@ public class Simulator var record = router.GetRecordByName(args.Parameters.AgentName); response.CurrentAgentId = record.AgentId; } + else if (args.Function == "interrupt_task_execution") + { + response.Content = args.Parameters.Reason; + response.ExecutionResult = args.Parameters.Reason; + } + else if (args.Function == "response_to_user") + { + response.Content = args.Parameters.Answer; + response.ExecutionResult = args.Parameters.Answer; + } return response; } @@ -63,7 +73,14 @@ public class Simulator var args = JsonSerializer.Deserialize(response.Content); - SaveStateByArgs(args.Parameters.Arguments); + if (args.Function == "retrieve_data_from_agent") + { + SaveStateByArgs(args.Parameters.Arguments); + } + else if (args.Function == "response_to_user") + { + return response; + } // Retrieve information from specific agent var router = _services.GetRequiredService();