diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 54ad3a6d..87df6137 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,6 +2,8 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { + public string Name { get; } + /// /// Convert pdf pages to images, and return a list of image file paths /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs new file mode 100644 index 00000000..10ccd1a0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Repositories.Enums; + +namespace BotSharp.Abstraction.Files; + +public class FileCoreSettings +{ + public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; + public string Pdf2TextConverter { get; set; } + public string Pdf2ImageConverter { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs deleted file mode 100644 index 23ba12c6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -using BotSharp.Abstraction.Repositories.Enums; - -namespace BotSharp.Abstraction.Files; - -public class FileStorageSettings -{ - public string Default { get; set; } = FileStorageEnum.LocalFileStorage; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs new file mode 100644 index 00000000..7d92e504 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgeCollectionName +{ + public static string BotSharp = nameof(BotSharp); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs new file mode 100644 index 00000000..9d95967f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgePayloadName +{ + public static string Text = "text"; + public static string Question = "question"; + public static string Answer = "answer"; + public static string Request = "request"; + public static string Response = "response"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs index aefcdec6..3a5ab788 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeHook diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 5b2b795b..827b2ba1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -4,11 +4,8 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> CollectChunkedKnowledge(); - Task EmbedKnowledge(List chunks); - - Task Feed(KnowledgeFeedModel knowledge); - Task EmbedKnowledge(KnowledgeCreationModel knowledge); - Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); - Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); + Task> SearchKnowledge(KnowledgeRetrievalModel model); + Task FeedKnowledge(KnowledgeCreationModel model); + Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); + Task DeleteKnowledgeCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index d2ca2940..b8f2d47b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.AspNetCore.Http; - namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { + public string Name { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs new file mode 100644 index 00000000..d013529f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionData +{ + public string Id { get; set; } + public string Question { get; set; } + public string Answer { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs index 33d09bd6..b43bce67 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs @@ -1,6 +1,9 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCreationModel { + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; public string Content { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs deleted file mode 100644 index 7e3a315e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeFeedModel -{ - public string AgentId { get; set; } = string.Empty; - public string Content { get; set; } = string.Empty; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs new file mode 100644 index 00000000..d2d9c490 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFilter : StringIdPagination +{ + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs index 03f66eb5..76e5e77d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs @@ -1,7 +1,13 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeRetrievalModel { - public string AgentId { get; set; } = string.Empty; - public string Question { get; set; } = string.Empty; + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + public string Text { get; set; } = string.Empty; + public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs new file mode 100644 index 00000000..b0deaa0f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchResult +{ + public IDictionary Data { get; set; } = new Dictionary(); + public double Score { get; set; } + public float[]? Vector { get; set; } +} + +public class KnowledgeRetrievalResult : KnowledgeSearchResult +{ +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs deleted file mode 100644 index 296e1ed6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Abstraction.Knowledges.Models; - -public class RetrievedResult -{ - public int Paragraph { get; set; } - - [JsonPropertyName("cite_source")] - public string CiteSource { get; set; } = "related text"; - - [JsonPropertyName("reasoning")] - public string Reasoning { get; set; } = ""; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 97f7f55c..ac3f0500 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -3,7 +3,11 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { public string VectorDb { get; set; } - public string TextEmbedding { get; set; } - public string TextCompletion { get; set; } - public string Pdf2TextConverter { get; set; } + public KnowledgeModelSetting TextEmbedding { get; set; } } + +public class KnowledgeModelSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 89c9f7db..825f7d8f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs new file mode 100644 index 00000000..d8e4d355 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.Utilities; + +public class StringIdPagination : Pagination +{ + [JsonPropertyName("start_id")] + public string? StartId { get; set; } +} + +public class StringIdPagedItems : PagedItems +{ + public new ulong Count { get; set; } + + [JsonPropertyName("next_id")] + public string? NextId { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 68aaaa78..a079aaef 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -2,8 +2,12 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - Task> GetCollections(); + string Name { get; } + + Task> GetCollections(); + Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false); + Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs similarity index 63% rename from src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs rename to src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs index 46eded0f..0b09d6aa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs @@ -4,22 +4,22 @@ using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Files; -public class FilePlugin : IBotSharpPlugin +public class FileCorePlugin : IBotSharpPlugin { public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; - public string Name => "File"; + public string Name => "File Core"; public string Description => "Provides file storage and analysis."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); - services.AddSingleton(myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); + services.AddSingleton(fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) + if (fileCoreSettings.Storage == FileStorageEnum.LocalFileStorage) { services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 6e6d4168..2aec257b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -97,7 +97,8 @@ public partial class FileInstructService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = _services.GetServices().FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 9e14677d..8c2ed831 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -275,8 +275,9 @@ public partial class LocalFileStorageService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 6ad463b3..5f73adb9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,6 +1,7 @@ +using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; -using Microsoft.AspNetCore.Http; +using BotSharp.OpenAPI.ViewModels.Knowledges; namespace BotSharp.OpenAPI.Controllers; @@ -17,77 +18,65 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpGet("/knowledge/{agentId}")] - public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) + [HttpPost("/knowledge/search")] + public async Task> SearchKnowledge([FromBody] SearchKnowledgeModel model) { - return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel + var searchModel = new KnowledgeRetrievalModel { - AgentId = agentId, - Question = question - }); + Collection = model.Collection, + Text = model.Text, + Fields = model.Fields, + Limit = model.Limit ?? 5, + Confidence = model.Confidence ?? 0.5f, + WithVector = model.WithVector + }; + + var results = await _knowledgeService.SearchKnowledge(searchModel); + return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList(); } - [HttpPost("/knowledge-base/upload")] - public async Task UploadKnowledge(IFormFile file, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + [HttpPost("/knowledge/{collection}/data")] + public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))? + .ToList() ?? new List(); + + return new StringIdPagedItems + { + Count = data.Count, + NextId = data.NextId, + Items = items + }; + } + + [HttpDelete("/knowledge/{collection}/data/{id}")] + public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) + { + return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); + } + + [HttpPost("/knowledge/upload")] + public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); var filePath = Path.GetTempFileName(); - using (var stream = System.IO.File.Create(filePath)) + using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) { await file.CopyToAsync(stream); + await stream.FlushAsync(); } var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.EmbedKnowledge(new KnowledgeCreationModel + await _knowledgeService.FeedKnowledge(new KnowledgeCreationModel { + Collection = collection ?? KnowledgeCollectionName.BotSharp, Content = content }); + System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } - - [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - long size = files.Sum(f => f.Length); - - foreach (var formFile in files) - { - var filePath = Path.GetTempFileName(); - - - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - { - await formFile.CopyToAsync(stream); - await stream.FlushAsync(); // Ensure all data is written to the file - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.Feed(new KnowledgeFeedModel - { - AgentId = agentId, - Content = content - }); - - // Delete the temp file after processing to clean up - System.IO.File.Delete(filePath); - } - - return Ok(new { count = files.Count, size }); - } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs new file mode 100644 index 00000000..f77b8777 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionDataViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("question")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Question { get; set; } + + [JsonPropertyName("answer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Answer { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeCollectionDataViewModel From(KnowledgeCollectionData data) + { + return new KnowledgeCollectionDataViewModel + { + Id = data.Id, + Question = data.Question, + Answer = data.Answer, + Vector = data.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs new file mode 100644 index 00000000..2e2b9e08 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeRetrivalViewModel +{ + [JsonPropertyName("data")] + public IDictionary Data { get; set; } + + [JsonPropertyName("score")] + public double Score { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeRetrivalViewModel From(KnowledgeRetrievalResult model) + { + return new KnowledgeRetrivalViewModel + { + Data = model.Data, + Score = model.Score, + Vector = model.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs new file mode 100644 index 00000000..afb311ed --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchKnowledgeModel +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index a12bdfd4..759f94b6 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs index 521dc221..bac82d2e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Messaging.Enums; -using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.Messaging; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class ConfirmKnowledgePersistenceFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 3eaa72f5..7d2af5b9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback @@ -20,20 +17,14 @@ public class KnowledgeRetrievalFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); - - var vector = await embedding.GetVectorsAsync(new List - { - args.Question - }); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); + var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetRequiredService(); + var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List { KnowledgePayloadName.Answer }); - var id = Utilities.HashTextMd5(args.Question); - var knowledges = await vectorDb.Search("lessen", vector[0], "answer"); - - if (knowledges.Count > 0) + if (!knowledges.IsNullOrEmpty()) { message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 23b49b94..619ea685 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback @@ -20,8 +17,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorsAsync(new List { @@ -29,15 +26,14 @@ public class MemorizeKnowledgeFn : IFunctionCallback }); var vectorDb = _services.GetRequiredService(); + await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); - await vectorDb.CreateCollection("lessen", vector[0].Length); - - var id = Utilities.HashTextMd5(args.Question); - var result = await vectorDb.Upsert("lessen", id, vector[0], + var id = Guid.NewGuid().ToString(); + var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], args.Question, new Dictionary { - { "answer", args.Answer } + { KnowledgePayloadName.Answer, args.Answer } }); message.Content = result ? "Saved to my brain" : "I forgot it"; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs index 2eebf75c..f8296520 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseAgentHook : AgentHookBase, IAgentHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index fd163cbf..cd428136 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs index 4710faea..df97bc34 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs @@ -9,6 +9,6 @@ public class MemVecDbPlugin : IBotSharpPlugin public string Description => "Store text embedding, search similar text from memory."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs new file mode 100644 index 00000000..ba5e6383 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -0,0 +1,71 @@ +using BotSharp.Plugin.KnowledgeBase.Utilities; +using Tensorflow.NumPy; + +namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; + +public class MemoryVectorDb : IVectorDb +{ + private readonly Dictionary _collections = new Dictionary(); + private readonly Dictionary> _vectors = new Dictionary>(); + + + public string Name => "MemoryVector"; + + public async Task CreateCollection(string collectionName, int dim) + { + _collections[collectionName] = dim; + _vectors[collectionName] = new List(); + } + + public async Task> GetCollections() + { + return _collections.Select(x => x.Key).ToList(); + } + + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + { + if (!_vectors.ContainsKey(collectionName)) + { + return new List(); + } + + var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + + var results = np.argsort(similarities).ToArray() + .Reverse() + .Take(limit) + .Select(i => new KnowledgeSearchResult + { + Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, + Score = similarities[i], + Vector = withVector ? _vectors[collectionName][i].Vector : null, + }) + .ToList(); + + return await Task.FromResult(results); + } + + public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) + { + _vectors[collectionName].Add(new VecRecord + { + Id = id, + Vector = vector, + Text = text + }); + + return true; + } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs new file mode 100644 index 00000000..81d923a3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task FeedKnowledge(KnowledgeCreationModel knowledge) + { + var index = 0; + var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + var db = GetVectorDb(); + var textEmbedding = GetTextEmbedding(); + + await db.CreateCollection(knowledge.Collection, textEmbedding.Dimension); + foreach (var line in lines) + { + var vec = await textEmbedding.GetVectorAsync(line); + var id = Guid.NewGuid().ToString(); + await db.Upsert(knowledge.Collection, id, vec, line); + index++; + Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs new file mode 100644 index 00000000..5020c529 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task DeleteKnowledgeCollectionData(string collectionName, string id) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionData(collectionName, id); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs new file mode 100644 index 00000000..648ba120 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(collectionName, filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new StringIdPagedItems(); + } + } + + public async Task> SearchKnowledge(KnowledgeRetrievalModel model) + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(model.Text); + + // Vector search + var db = GetVectorDb(); + var collection = !string.IsNullOrWhiteSpace(model.Collection) ? model.Collection : KnowledgeCollectionName.BotSharp; + var fields = !model.Fields.IsNullOrEmpty() ? model.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + var found = await db.Search(collection, vector, fields, limit: model.Limit ?? 5, confidence: model.Confidence ?? 0.5f, withVector: model.WithVector); + + var results = found.Select(x => new KnowledgeRetrievalResult + { + Data = x.Data, + Score = x.Score, + Vector = x.Vector + }).ToList(); + return results; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 7ac5cf4c..dff80134 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -5,114 +5,33 @@ public partial class KnowledgeService : IKnowledgeService private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; private readonly ITextChopper _textChopper; + private readonly ILogger _logger; - public KnowledgeService(IServiceProvider services, + public KnowledgeService( + IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper) + ITextChopper textChopper, + ILogger logger) { _services = services; _settings = settings; _textChopper = textChopper; + _logger = logger; } - public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) + private IVectorDb GetVectorDb() { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection("shared", textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert("shared", idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task Feed(KnowledgeFeedModel knowledge) - { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(knowledge.AgentId, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(knowledge.AgentId, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task GetKnowledges(KnowledgeRetrievalModel retrievalModel) - { - var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question); - - // Vector search - var db = GetVectorDb(); - var result = await db.Search("shared", vector, "answer", limit: 10); - - // Restore - return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); - } - - public async Task> GetAnswer(KnowledgeRetrievalModel retrievalModel) - { - // Restore - var prompt = await GetKnowledges(retrievalModel); - - var sb = new StringBuilder(prompt); - sb.AppendLine(); - sb.AppendLine(); - sb.AppendLine("------"); - sb.AppendLine("Answer question based on the given information above. Keep your answers concise. Please response with paragraph number, cite sources and reasoning in JSON format, if multiple paragraphs are found, put them in a JSON array. make sure the paragraph number is real. If you don't know the answer just output empty."); - sb.AppendLine("[" + JsonSerializer.Serialize(new RetrievedResult()) + "]"); - sb.AppendLine("------"); - sb.AppendLine($"QUESTION: \"{retrievalModel.Question}\""); - sb.AppendLine("Which paragraphs are relevant in order to answer the above question?"); - sb.AppendLine("ANSWER: "); - prompt = sb.ToString().Trim(); - - var completion = await GetTextCompletion().GetCompletion(prompt, Guid.Empty.ToString(), Guid.Empty.ToString()); - return JsonSerializer.Deserialize>(completion); - } - - public IVectorDb GetVectorDb() - { - var db = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.VectorDb)); + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); return db; } - public ITextEmbedding GetTextEmbedding() + private ITextEmbedding GetTextEmbedding() { - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + if (embedding != null) + { + embedding.SetModelName(_settings.TextEmbedding.Model); + } return embedding; } - - public ITextCompletion GetTextCompletion() - { - var textCompletion = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextCompletion)); - return textCompletion; - } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs deleted file mode 100644 index eccfed78..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CollectChunkedKnowledge() - { - throw new NotImplementedException(); - } - - public async Task EmbedKnowledge(List chunks) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 17706c13..09401f20 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,6 +5,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { + public string Name => "Pig"; + public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { // since PdfDocument.Open is not async, we dont need to make this method async diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs index 96dc021d..88c77641 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs @@ -16,7 +16,7 @@ public class TextChopperService : ITextChopper var chunks = new List(); var words = content.Split(' ') - .Where(x => !string.IsNullOrEmpty(x)) + .Where(x => !string.IsNullOrWhiteSpace(x)) .ToList(); var chunk = ""; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 60db13a6..b92f919f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -17,7 +17,18 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Agents.Settings; global using BotSharp.Abstraction.Conversations.Settings; global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.MLTasks; -global using BotSharp.Plugin.KnowledgeBase.Services; \ No newline at end of file +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Plugin.KnowledgeBase.Services; +global using BotSharp.Plugin.KnowledgeBase.Enum; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs similarity index 53% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs index c46ccdf5..f0538af0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs @@ -1,55 +1,12 @@ +using BotSharp.Plugin.KnowledgeBase.MemVecDb; using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; +namespace BotSharp.Plugin.KnowledgeBase.Utilities; -public class MemVectorDatabase : IVectorDb +public static class VectorUtility { - private readonly Dictionary _collections = new Dictionary(); - private readonly Dictionary> _vectors = new Dictionary>(); - public async Task CreateCollection(string collectionName, int dim) - { - _collections[collectionName] = dim; - _vectors[collectionName] = new List(); - } - - public async Task> GetCollections() - { - return _collections.Select(x => x.Key).ToList(); - } - - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) - { - if (!_vectors.ContainsKey(collectionName)) - { - return new List(); - } - - var similarities = CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]); - - var texts = np.argsort(similarities).ToArray() - .Reverse() - .Take(limit) - .Select(i => _vectors[collectionName][i].Text) - .ToList(); - - return texts; - } - - public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) - { - _vectors[collectionName].Add(new VecRecord - { - Id = id, - Vector = vector, - Text = text - }); - - return true; - } - - private float[] CalEuclideanDistance(float[] vec, List records) + public static float[] CalEuclideanDistance(float[] vec, List records) { var a = np.zeros((records.Count, vec.Length), np.float32); var b = np.zeros((records.Count, vec.Length), np.float32); @@ -64,7 +21,7 @@ public class MemVectorDatabase : IVectorDb return c.ToArray(); } - public NDArray CalCosineSimilarity(float[] vec, List records) + public static NDArray CalCosineSimilarity(float[] vec, List records) { var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); @@ -85,7 +42,7 @@ public class MemVectorDatabase : IVectorDb return simiMatix; } - public (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) + public static (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) { var simiMatix = CalCosineSimilarity(vec, records); @@ -108,7 +65,7 @@ public class MemVectorDatabase : IVectorDb return resIndex.ToArray(); } - public (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + private static (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) { var squaredX = np.sum(np.multiply(x, x), axis: 1); var normX = np.sqrt(squaredX); diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 48e2be17..785acd4d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -7,17 +9,25 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { + public string Name => "Faiss"; + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); } - public Task> GetCollections() + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> GetCollections() + { + throw new NotImplementedException(); + } + + public Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); } @@ -26,4 +36,9 @@ public class FaissDb : IVectorDb { throw new NotImplementedException(); } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index a66cb8f9..81c53f83 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -19,15 +19,18 @@ using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ +{ private Dictionary _mappings = new Dictionary(); private FullOcrModel _model; private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { _paddleSharpSettings = paddleSharpSettings; } + public string Name => "Paddle"; + public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { await ConvertPdfToLocalImagesAsync(filePath, startPageNum, endPageNum); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index e698d5e3..f03b4ab6 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,13 +1,6 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.VectorStorage; -using Microsoft.Extensions.DependencyInjection; +using BotSharp.Abstraction.Utilities; using Qdrant.Client; using Qdrant.Client.Grpc; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; namespace BotSharp.Plugin.Qdrant; @@ -17,14 +10,16 @@ public class QdrantDb : IVectorDb private readonly QdrantSetting _setting; private readonly IServiceProvider _services; - public QdrantDb(QdrantSetting setting, + public QdrantDb( + QdrantSetting setting, IServiceProvider services) { _setting = setting; _services = services; - } + public string Name => "Qdrant"; + private QdrantClient GetClient() { if (_client == null) @@ -39,20 +34,50 @@ public class QdrantDb : IVectorDb return _client; } - public async Task> GetCollections() + public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } + public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return new StringIdPagedItems(); + } + + var totalPointCount = await client.CountAsync(collectionName); + var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new KnowledgeCollectionData + { + Id = x.Id?.Uuid ?? string.Empty, + Question = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, + Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, + Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null + })?.ToList() ?? new List(); + + return new StringIdPagedItems + { + Count = totalPointCount, + NextId = response?.NextPageOffset?.Uuid, + Items = points + }; + } + public async Task CreateCollection(string collectionName, int dim) { - var collections = await GetCollections(); - if (!collections.Contains(collectionName)) + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { // Create a new collection - await GetClient().CreateCollectionAsync(collectionName, new VectorParams() + await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dim, Distance = Distance.Cosine @@ -60,7 +85,7 @@ public class QdrantDb : IVectorDb } // Get collection info - var collectionInfo = await _client.GetCollectionInfoAsync(collectionName); + var collectionInfo = await client.GetCollectionInfoAsync(collectionName); if (collectionInfo == null) { throw new Exception($"Create {collectionName} failed."); @@ -77,10 +102,9 @@ public class QdrantDb : IVectorDb Uuid = id }, Vectors = vector, - - Payload = + Payload = { - { "text", text } + { KnowledgePayloadName.Text, text } } }; @@ -93,7 +117,6 @@ public class QdrantDb : IVectorDb } var client = GetClient(); - var result = await client.UpsertAsync(collectionName, points: new List { point @@ -102,13 +125,54 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var client = GetClient(); - var points = await client.SearchAsync(collectionName, vector, - limit: (ulong)limit, - scoreThreshold: confidence); + var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); - return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); + var results = new List(); + foreach (var point in points) + { + var data = new Dictionary(); + foreach (var field in fields) + { + if (point.Payload.ContainsKey(field)) + { + data[field] = point.Payload[field].StringValue; + } + else + { + data[field] = ""; + } + } + + results.Add(new KnowledgeSearchResult + { + Data = data, + Score = point.Score, + Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null + }); + } + + return results; + } + + public async Task DeleteCollectionData(string collectionName, string id) + { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + + var client = GetClient(); + var result = await client.DeleteAsync(collectionName, guid); + return result.Status == UpdateStatus.Completed; + } + + + private async Task DoesCollectionExist(QdrantClient client, string collectionName) + { + return await client.CollectionExistsAsync(collectionName); } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs index 8cfd137b..a3bc116a 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Abstraction.VectorStorage; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs new file mode 100644 index 00000000..dc7e9747 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.VectorStorage; +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 089b5168..c52baf59 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -57,8 +57,8 @@ public class IntentClassifier return; } - var vector = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); + var vector = _services.GetServices().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.TextEmbedding.Provider); + vector.SetModelName(_knowledgeBaseSettings.TextEmbedding.Model); var layers = new List { @@ -136,8 +136,8 @@ public class IntentClassifier public NDArray GetTextEmbedding(string text) { var knowledgeSettings = _services.GetRequiredService(); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(knowledgeSettings.TextEmbedding)); + var embedding = _services.GetServices() .FirstOrDefault(x => x.Provider == knowledgeSettings.TextEmbedding.Provider); + embedding.SetModelName(knowledgeSettings.TextEmbedding.Model); var x = np.zeros((1, embedding.Dimension), dtype: np.float32); x[0] = embedding.GetVectorAsync(text).GetAwaiter().GetResult(); diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 20fdeecc..a5db6de8 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -1,8 +1,9 @@ +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; using System; using System.Collections.Generic; -using System.Text; using System.Threading.Tasks; namespace BotSharp.Plugin.SemanticKernel @@ -19,12 +20,21 @@ namespace BotSharp.Plugin.SemanticKernel { this._memoryStore = memoryStore; } + + + public string Name => "SemanticKernel"; + public async Task CreateCollection(string collectionName, int dim) { await _memoryStore.CreateCollectionAsync(collectionName); } - public async Task> GetCollections() + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + { + throw new System.NotImplementedException(); + } + + public async Task> GetCollections() { var result = new List(); await foreach (var collection in _memoryStore.GetCollectionsAsync()) @@ -34,18 +44,23 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); - await foreach (var (record, _) in results) + var resultTexts = new List(); + await foreach (var (record, score) in results) { - resultTexts.Add(record.Metadata.Text); + resultTexts.Add(new KnowledgeSearchResult + { + Data = new Dictionary { { "text", record.Metadata.Text } }, + Score = score, + Vector = withVector ? record.Embedding.ToArray() : null + }); } return resultTexts; - } public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload) @@ -55,5 +70,17 @@ namespace BotSharp.Plugin.SemanticKernel #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. return true; } + + public async Task DeleteCollectionData(string collectionName, string id) + { + var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + + if (exist) + { + await _memoryStore.RemoveAsync(collectionName, id); + return true; + } + return false; + } } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index f04d0be6..7e2462de 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; @@ -252,8 +253,9 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private string BuilFileUrl(string file) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 1dcb5edb..791e7f98 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -18,10 +18,10 @@ public class TencentCosPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.TencentCosStorage) + if (fileCoreSettings.Storage == FileStorageEnum.TencentCosStorage) { services.AddScoped(provider => { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b10e2160..155dae1b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -230,9 +230,13 @@ "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] }, - "FileStorage": { - "Default": "LocalFileStorage" + + "FileCore": { + "Storage": "LocalFileStorage", + "Pdf2TextConverter": "", + "Pdf2ImageConverter": "" }, + "TencentCos": { "AppId": "", "SecretId": "", @@ -254,10 +258,11 @@ }, "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - "TextEmbedding": "fastTextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", - "Pdf2TextConverter": "PigPdf2TextConverter" + "VectorDb": "Qdrant", + "TextEmbedding": { + "Provider": "openai", + "Model": "text-embedding-3-small" + } }, "SparkDesk": {