diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 54ad3a6d..87df6137 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,6 +2,8 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { + public string Name { get; } + /// /// Convert pdf pages to images, and return a list of image file paths /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs new file mode 100644 index 00000000..10ccd1a0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Repositories.Enums; + +namespace BotSharp.Abstraction.Files; + +public class FileCoreSettings +{ + public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; + public string Pdf2TextConverter { get; set; } + public string Pdf2ImageConverter { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs deleted file mode 100644 index 23ba12c6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -using BotSharp.Abstraction.Repositories.Enums; - -namespace BotSharp.Abstraction.Files; - -public class FileStorageSettings -{ - public string Default { get; set; } = FileStorageEnum.LocalFileStorage; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs index aefcdec6..3a5ab788 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeHook diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f145aa6f..827b2ba1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -4,17 +4,8 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> CollectChunkedKnowledge(); - Task EmbedKnowledge(List chunks); - - Task Feed(KnowledgeFeedModel knowledge); - Task EmbedKnowledge(KnowledgeCreationModel knowledge); - Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); - Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); - - #region List + Task> SearchKnowledge(KnowledgeRetrievalModel model); + Task FeedKnowledge(KnowledgeCreationModel model); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); - Task> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter); Task DeleteKnowledgeCollectionData(string collectionName, string id); - #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index d2ca2940..b8f2d47b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.AspNetCore.Http; - namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { + public string Name { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs deleted file mode 100644 index 714c2b3f..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeCollectionInfo -{ - public ulong DataCount { get; set; } - public ulong VectorCount { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs index 33d09bd6..b43bce67 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs @@ -1,6 +1,9 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCreationModel { + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; public string Content { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs deleted file mode 100644 index 7e3a315e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeFeedModel -{ - public string AgentId { get; set; } = string.Empty; - public string Content { get; set; } = string.Empty; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs index 03f66eb5..76e5e77d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs @@ -1,7 +1,13 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeRetrievalModel { - public string AgentId { get; set; } = string.Empty; - public string Question { get; set; } = string.Empty; + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + public string Text { get; set; } = string.Empty; + public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs deleted file mode 100644 index 131bacc0..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeRetrievalResult -{ - public string Id { get; set; } - public string Text { get; set; } - public float Score { get; set; } - public float[]? Vector { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs new file mode 100644 index 00000000..b0deaa0f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchResult +{ + public IDictionary Data { get; set; } = new Dictionary(); + public double Score { get; set; } + public float[]? Vector { get; set; } +} + +public class KnowledgeRetrievalResult : KnowledgeSearchResult +{ +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs deleted file mode 100644 index 18e55634..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class RetrievedResult -{ - public int Paragraph { get; set; } - - [JsonPropertyName("cite_source")] - public string CiteSource { get; set; } = "related text"; - - [JsonPropertyName("reasoning")] - public string Reasoning { get; set; } = ""; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 97f7f55c..0c3f8d45 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -3,7 +3,12 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { public string VectorDb { get; set; } - public string TextEmbedding { get; set; } - public string TextCompletion { get; set; } + public KnowledgeModelSetting TextEmbedding { get; set; } public string Pdf2TextConverter { get; set; } } + +public class KnowledgeModelSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 89c9f7db..825f7d8f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 413e998a..a079aaef 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,13 +1,13 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { + string Name { get; } + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs similarity index 63% rename from src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs rename to src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs index 46eded0f..0b09d6aa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs @@ -4,22 +4,22 @@ using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Files; -public class FilePlugin : IBotSharpPlugin +public class FileCorePlugin : IBotSharpPlugin { public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; - public string Name => "File"; + public string Name => "File Core"; public string Description => "Provides file storage and analysis."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); - services.AddSingleton(myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); + services.AddSingleton(fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) + if (fileCoreSettings.Storage == FileStorageEnum.LocalFileStorage) { services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 6e6d4168..2aec257b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -97,7 +97,8 @@ public partial class FileInstructService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = _services.GetServices().FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 9e14677d..8c2ed831 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -275,8 +275,9 @@ public partial class LocalFileStorageService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index debec995..7f6a06de 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -17,86 +18,28 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpGet("/knowledge/{agentId}")] - public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) + [HttpPost("/knowledge/search")] + public async Task> SearchKnowledge([FromBody] SearchKnowledgeModel model) { - return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel + var searchModel = new KnowledgeRetrievalModel { - AgentId = agentId, - Question = question - }); + Collection = model.Collection, + Text = model.Text, + Fields = model.Fields, + Limit = model.Limit ?? 5, + Confidence = model.Confidence ?? 0.5f, + WithVector = model.WithVector + }; + + var results = await _knowledgeService.SearchKnowledge(searchModel); + return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList(); } - [HttpPost("/knowledge-base/upload")] - public async Task UploadKnowledge(IFormFile file, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - - var filePath = Path.GetTempFileName(); - using (var stream = System.IO.File.Create(filePath)) - { - await file.CopyToAsync(stream); - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.EmbedKnowledge(new KnowledgeCreationModel - { - Content = content - }); - - return Ok(new { count = 1, file.Length }); - } - - [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - long size = files.Sum(f => f.Length); - - foreach (var formFile in files) - { - var filePath = Path.GetTempFileName(); - - - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - { - await formFile.CopyToAsync(stream); - await stream.FlushAsync(); // Ensure all data is written to the file - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.Feed(new KnowledgeFeedModel - { - AgentId = agentId, - Content = content - }); - - // Delete the temp file after processing to clean up - System.IO.File.Delete(filePath); - } - - return Ok(new { count = files.Count, size }); - } - - [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))? .ToList() ?? new List(); return new StringIdPagedItems @@ -107,19 +50,33 @@ public class KnowledgeBaseController : ControllerBase }; } - [HttpPost("/knowledge/{collection}/similar")] - public async Task> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) - { - var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? - .ToList() ?? new List(); - - return - } - [HttpDelete("/knowledge/{collection}/data/{id}")] public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) { return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); } + + [HttpPost("/knowledge/upload")] + public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + + var filePath = Path.GetTempFileName(); + using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await file.CopyToAsync(stream); + await stream.FlushAsync(); + } + + var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); + await _knowledgeService.FeedKnowledge(new KnowledgeCreationModel + { + Collection = collection ?? KnowledgeCollectionName.BotSharp, + Content = content + }); + + System.IO.File.Delete(filePath); + return Ok(new { count = 1, file.Length }); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 5ce43369..f77b8777 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -20,7 +20,7 @@ public class KnowledgeCollectionDataViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public float[]? Vector { get; set; } - public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data) + public static KnowledgeCollectionDataViewModel From(KnowledgeCollectionData data) { return new KnowledgeCollectionDataViewModel { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs new file mode 100644 index 00000000..2e2b9e08 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeRetrivalViewModel +{ + [JsonPropertyName("data")] + public IDictionary Data { get; set; } + + [JsonPropertyName("score")] + public double Score { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeRetrivalViewModel From(KnowledgeRetrievalResult model) + { + return new KnowledgeRetrivalViewModel + { + Data = model.Data, + Score = model.Score, + Vector = model.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs new file mode 100644 index 00000000..afb311ed --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchKnowledgeModel +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f8d83e0c..7d2af5b9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -17,14 +17,14 @@ public class KnowledgeRetrievalFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetRequiredService(); - var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer); + var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List { KnowledgePayloadName.Answer }); - if (knowledges.Count > 0) + if (!knowledges.IsNullOrEmpty()) { message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 0ef0f950..619ea685 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback @@ -19,8 +17,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorsAsync(new List { @@ -28,10 +26,9 @@ public class MemorizeKnowledgeFn : IFunctionCallback }); var vectorDb = _services.GetRequiredService(); - await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); - var id = Utilities.HashTextMd5(args.Question); + var id = Guid.NewGuid().ToString(); var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], args.Question, new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs index 4710faea..df97bc34 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs @@ -9,6 +9,6 @@ public class MemVecDbPlugin : IBotSharpPlugin public string Description => "Store text embedding, search similar text from memory."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs deleted file mode 100644 index f7c9276d..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ /dev/null @@ -1,139 +0,0 @@ -using Tensorflow.NumPy; -using static Tensorflow.Binding; - -namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; - -public class MemVectorDatabase : IVectorDb -{ - private readonly Dictionary _collections = new Dictionary(); - private readonly Dictionary> _vectors = new Dictionary>(); - - public async Task CreateCollection(string collectionName, int dim) - { - _collections[collectionName] = dim; - _vectors[collectionName] = new List(); - } - - public async Task> GetCollections() - { - return _collections.Select(x => x.Key).ToList(); - } - - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) - { - throw new NotImplementedException(); - } - - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) - { - if (!_vectors.ContainsKey(collectionName)) - { - return new List(); - } - - var similarities = CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]); - - var texts = np.argsort(similarities).ToArray() - .Reverse() - .Take(limit) - .Select(i => _vectors[collectionName][i].Text) - .ToList(); - - return texts; - } - - public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) - { - _vectors[collectionName].Add(new VecRecord - { - Id = id, - Vector = vector, - Text = text - }); - - return true; - } - - public Task DeleteCollectionData(string collectionName, string id) - { - throw new NotImplementedException(); - } - - #region Private methods - private float[] CalEuclideanDistance(float[] vec, List records) - { - var a = np.zeros((records.Count, vec.Length), np.float32); - var b = np.zeros((records.Count, vec.Length), np.float32); - for (var i = 0; i < records.Count; i++) - { - a[i] = vec; - b[i] = records[i].Vector; - } - - var c = np.sqrt(np.sum(np.square(a - b), axis: 1)); - // var c = -np.prod(np.linalg.norm(a, axis: 1) * np.linalg.norm(b, axis: 1), axis: 1); - return c.ToArray(); - } - - private NDArray CalCosineSimilarity(float[] vec, List records) - { - var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); - - for (int i = 0; i < records.Count; i++) - { - recordsArray[i] = records[i].Vector; - } - - var vecArray = np.expand_dims(np.array(vec, dtype: np.float32), axis: 0); // [1. 300] - - (var normVecArray, var _) = SafeNormalize(vecArray); - (var normRecordsArray, var _) = SafeNormalize(recordsArray); - - var simiMatix = tf.matmul(tf.cast(normVecArray, tf.float32), tf.transpose(tf.cast(normRecordsArray, tf.float32))).numpy(); // [1, num_records] - - simiMatix = np.squeeze(simiMatix, axis: 0); - - return simiMatix; - } - - public (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) - { - var simiMatix = CalCosineSimilarity(vec, records); - - topK = Math.Min(topK, records.Count); - var topIndex = np.argsort(simiMatix)["::-1"][$":{topK}"]; - - var resIndex = new List<(int, float)>(); - - for (int i = 0; i < topK; i++) - { - var index = topIndex[i]; - var value = simiMatix[index]; - - if (value > filterProb) - { - resIndex.Add((topIndex[i], value)); - } - } - - return resIndex.ToArray(); - } - - private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) - { - var squaredX = np.sum(np.multiply(x, x), axis: 1); - var normX = np.sqrt(squaredX); - - var epsTensor = tf.cast(tf.convert_to_tensor(eps), dtype: tf.float32); - var normXTensor = tf.cast(normX, tf.float32); - var contantMask = (normXTensor < epsTensor); - var divideTensor = tf.ones_like(normXTensor, dtype: tf.float32); - - normX = tf.where(contantMask, divideTensor, normXTensor).numpy(); - normX = np.expand_dims(normX, axis: 1); - - return (x / normX, normX); - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs new file mode 100644 index 00000000..ba5e6383 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -0,0 +1,71 @@ +using BotSharp.Plugin.KnowledgeBase.Utilities; +using Tensorflow.NumPy; + +namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; + +public class MemoryVectorDb : IVectorDb +{ + private readonly Dictionary _collections = new Dictionary(); + private readonly Dictionary> _vectors = new Dictionary>(); + + + public string Name => "MemoryVector"; + + public async Task CreateCollection(string collectionName, int dim) + { + _collections[collectionName] = dim; + _vectors[collectionName] = new List(); + } + + public async Task> GetCollections() + { + return _collections.Select(x => x.Key).ToList(); + } + + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + { + if (!_vectors.ContainsKey(collectionName)) + { + return new List(); + } + + var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + + var results = np.argsort(similarities).ToArray() + .Reverse() + .Take(limit) + .Select(i => new KnowledgeSearchResult + { + Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, + Score = similarities[i], + Vector = withVector ? _vectors[collectionName][i].Vector : null, + }) + .ToList(); + + return await Task.FromResult(results); + } + + public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) + { + _vectors[collectionName].Add(new VecRecord + { + Id = id, + Vector = vector, + Text = text + }); + + return true; + } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs new file mode 100644 index 00000000..81d923a3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task FeedKnowledge(KnowledgeCreationModel knowledge) + { + var index = 0; + var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + var db = GetVectorDb(); + var textEmbedding = GetTextEmbedding(); + + await db.CreateCollection(knowledge.Collection, textEmbedding.Dimension); + foreach (var line in lines) + { + var vec = await textEmbedding.GetVectorAsync(line); + var id = Guid.NewGuid().ToString(); + await db.Upsert(knowledge.Collection, id, vec, line); + index++; + Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs similarity index 50% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index d3b5ed84..5020c529 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,20 +2,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) - { - try - { - var db = GetVectorDb(); - return await db.GetCollectionData(collectionName, filter); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new StringIdPagedItems(); - } - } - public async Task DeleteKnowledgeCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs new file mode 100644 index 00000000..648ba120 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(collectionName, filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new StringIdPagedItems(); + } + } + + public async Task> SearchKnowledge(KnowledgeRetrievalModel model) + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(model.Text); + + // Vector search + var db = GetVectorDb(); + var collection = !string.IsNullOrWhiteSpace(model.Collection) ? model.Collection : KnowledgeCollectionName.BotSharp; + var fields = !model.Fields.IsNullOrEmpty() ? model.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + var found = await db.Search(collection, vector, fields, limit: model.Limit ?? 5, confidence: model.Confidence ?? 0.5f, withVector: model.WithVector); + + var results = found.Select(x => new KnowledgeRetrievalResult + { + Data = x.Data, + Score = x.Score, + Vector = x.Vector + }).ToList(); + return results; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index cdfff9db..dff80134 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -7,7 +7,8 @@ public partial class KnowledgeService : IKnowledgeService private readonly ITextChopper _textChopper; private readonly ILogger _logger; - public KnowledgeService(IServiceProvider services, + public KnowledgeService( + IServiceProvider services, KnowledgeBaseSettings settings, ITextChopper textChopper, ILogger logger) @@ -18,104 +19,19 @@ public partial class KnowledgeService : IKnowledgeService _logger = logger; } - public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) + private IVectorDb GetVectorDb() { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(KnowledgeCollectionName.BotSharp, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(KnowledgeCollectionName.BotSharp, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task Feed(KnowledgeFeedModel knowledge) - { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(knowledge.AgentId, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(knowledge.AgentId, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task GetKnowledges(KnowledgeRetrievalModel retrievalModel) - { - var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question); - - // Vector search - var db = GetVectorDb(); - var result = await db.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer, limit: 10); - - // Restore - return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); - } - - public async Task> GetAnswer(KnowledgeRetrievalModel retrievalModel) - { - // Restore - var prompt = await GetKnowledges(retrievalModel); - - var sb = new StringBuilder(prompt); - sb.AppendLine(); - sb.AppendLine(); - sb.AppendLine("------"); - sb.AppendLine("Answer question based on the given information above. Keep your answers concise. Please response with paragraph number, cite sources and reasoning in JSON format, if multiple paragraphs are found, put them in a JSON array. make sure the paragraph number is real. If you don't know the answer just output empty."); - sb.AppendLine("[" + JsonSerializer.Serialize(new RetrievedResult()) + "]"); - sb.AppendLine("------"); - sb.AppendLine($"QUESTION: \"{retrievalModel.Question}\""); - sb.AppendLine("Which paragraphs are relevant in order to answer the above question?"); - sb.AppendLine("ANSWER: "); - prompt = sb.ToString().Trim(); - - var completion = await GetTextCompletion().GetCompletion(prompt, Guid.Empty.ToString(), Guid.Empty.ToString()); - return JsonSerializer.Deserialize>(completion); - } - - public IVectorDb GetVectorDb() - { - var db = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.VectorDb)); + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); return db; } - public ITextEmbedding GetTextEmbedding() + private ITextEmbedding GetTextEmbedding() { - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + if (embedding != null) + { + embedding.SetModelName(_settings.TextEmbedding.Model); + } return embedding; } - - public ITextCompletion GetTextCompletion() - { - var textCompletion = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextCompletion)); - return textCompletion; - } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs deleted file mode 100644 index eccfed78..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CollectChunkedKnowledge() - { - throw new NotImplementedException(); - } - - public async Task EmbedKnowledge(List chunks) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 17706c13..09401f20 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,6 +5,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { + public string Name => "Pig"; + public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { // since PdfDocument.Open is not async, we dont need to make this method async diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs new file mode 100644 index 00000000..f0538af0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs @@ -0,0 +1,83 @@ +using BotSharp.Plugin.KnowledgeBase.MemVecDb; +using Tensorflow.NumPy; +using static Tensorflow.Binding; + +namespace BotSharp.Plugin.KnowledgeBase.Utilities; + +public static class VectorUtility +{ + public static float[] CalEuclideanDistance(float[] vec, List records) + { + var a = np.zeros((records.Count, vec.Length), np.float32); + var b = np.zeros((records.Count, vec.Length), np.float32); + for (var i = 0; i < records.Count; i++) + { + a[i] = vec; + b[i] = records[i].Vector; + } + + var c = np.sqrt(np.sum(np.square(a - b), axis: 1)); + // var c = -np.prod(np.linalg.norm(a, axis: 1) * np.linalg.norm(b, axis: 1), axis: 1); + return c.ToArray(); + } + + public static NDArray CalCosineSimilarity(float[] vec, List records) + { + var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); + + for (int i = 0; i < records.Count; i++) + { + recordsArray[i] = records[i].Vector; + } + + var vecArray = np.expand_dims(np.array(vec, dtype: np.float32), axis: 0); // [1. 300] + + (var normVecArray, var _) = SafeNormalize(vecArray); + (var normRecordsArray, var _) = SafeNormalize(recordsArray); + + var simiMatix = tf.matmul(tf.cast(normVecArray, tf.float32), tf.transpose(tf.cast(normRecordsArray, tf.float32))).numpy(); // [1, num_records] + + simiMatix = np.squeeze(simiMatix, axis: 0); + + return simiMatix; + } + + public static (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) + { + var simiMatix = CalCosineSimilarity(vec, records); + + topK = Math.Min(topK, records.Count); + var topIndex = np.argsort(simiMatix)["::-1"][$":{topK}"]; + + var resIndex = new List<(int, float)>(); + + for (int i = 0; i < topK; i++) + { + var index = topIndex[i]; + var value = simiMatix[index]; + + if (value > filterProb) + { + resIndex.Add((topIndex[i], value)); + } + } + + return resIndex.ToArray(); + } + + private static (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + { + var squaredX = np.sum(np.multiply(x, x), axis: 1); + var normX = np.sqrt(squaredX); + + var epsTensor = tf.cast(tf.convert_to_tensor(eps), dtype: tf.float32); + var normXTensor = tf.cast(normX, tf.float32); + var contantMask = (normXTensor < epsTensor); + var divideTensor = tf.ones_like(normXTensor, dtype: tf.float32); + + normX = tf.where(contantMask, divideTensor, normXTensor).numpy(); + normX = np.expand_dims(normX, axis: 1); + + return (x / normX, normX); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 7e884632..785acd4d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,6 +9,8 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { + public string Name => "Faiss"; + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); @@ -24,7 +26,8 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index a66cb8f9..81c53f83 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -19,15 +19,18 @@ using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ +{ private Dictionary _mappings = new Dictionary(); private FullOcrModel _model; private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { _paddleSharpSettings = paddleSharpSettings; } + public string Name => "Paddle"; + public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { await ConvertPdfToLocalImagesAsync(filePath, startPageNum, endPageNum); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f7bb25c5..f03b4ab6 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -10,14 +10,16 @@ public class QdrantDb : IVectorDb private readonly QdrantSetting _setting; private readonly IServiceProvider _services; - public QdrantDb(QdrantSetting setting, + public QdrantDb( + QdrantSetting setting, IServiceProvider services) { _setting = setting; _services = services; - } + public string Name => "Qdrant"; + private QdrantClient GetClient() { if (_client == null) @@ -42,9 +44,8 @@ public class QdrantDb : IVectorDb public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); - - var exists = await client.CollectionExistsAsync(collectionName); - if (!exists) + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { return new StringIdPagedItems(); } @@ -71,11 +72,12 @@ public class QdrantDb : IVectorDb public async Task CreateCollection(string collectionName, int dim) { - var collections = await GetCollections(); - if (!collections.Contains(collectionName)) + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { // Create a new collection - await GetClient().CreateCollectionAsync(collectionName, new VectorParams() + await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dim, Distance = Distance.Cosine @@ -83,7 +85,7 @@ public class QdrantDb : IVectorDb } // Get collection info - var collectionInfo = await _client.GetCollectionInfoAsync(collectionName); + var collectionInfo = await client.GetCollectionInfoAsync(collectionName); if (collectionInfo == null) { throw new Exception($"Create {collectionName} failed."); @@ -115,7 +117,6 @@ public class QdrantDb : IVectorDb } var client = GetClient(); - var result = await client.UpsertAsync(collectionName, points: new List { point @@ -124,19 +125,54 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var client = GetClient(); var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); - return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); + var results = new List(); + foreach (var point in points) + { + var data = new Dictionary(); + foreach (var field in fields) + { + if (point.Payload.ContainsKey(field)) + { + data[field] = point.Payload[field].StringValue; + } + else + { + data[field] = ""; + } + } + + results.Add(new KnowledgeSearchResult + { + Data = data, + Score = point.Score, + Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null + }); + } + + return results; } public async Task DeleteCollectionData(string collectionName, string id) { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + var client = GetClient(); - var guid = Guid.Parse(id); var result = await client.DeleteAsync(collectionName, guid); return result.Status == UpdateStatus.Completed; } + + + private async Task DoesCollectionExist(QdrantClient client, string collectionName) + { + return await client.CollectionExistsAsync(collectionName); + } } diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 089b5168..c52baf59 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -57,8 +57,8 @@ public class IntentClassifier return; } - var vector = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); + var vector = _services.GetServices().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.TextEmbedding.Provider); + vector.SetModelName(_knowledgeBaseSettings.TextEmbedding.Model); var layers = new List { @@ -136,8 +136,8 @@ public class IntentClassifier public NDArray GetTextEmbedding(string text) { var knowledgeSettings = _services.GetRequiredService(); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(knowledgeSettings.TextEmbedding)); + var embedding = _services.GetServices() .FirstOrDefault(x => x.Provider == knowledgeSettings.TextEmbedding.Provider); + embedding.SetModelName(knowledgeSettings.TextEmbedding.Model); var x = np.zeros((1, embedding.Dimension), dtype: np.float32); x[0] = embedding.GetVectorAsync(text).GetAwaiter().GetResult(); diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index b52ab71e..a5db6de8 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -20,6 +20,10 @@ namespace BotSharp.Plugin.SemanticKernel { this._memoryStore = memoryStore; } + + + public string Name => "SemanticKernel"; + public async Task CreateCollection(string collectionName, int dim) { await _memoryStore.CreateCollectionAsync(collectionName); @@ -40,18 +44,23 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); - await foreach (var (record, _) in results) + var resultTexts = new List(); + await foreach (var (record, score) in results) { - resultTexts.Add(record.Metadata.Text); + resultTexts.Add(new KnowledgeSearchResult + { + Data = new Dictionary { { "text", record.Metadata.Text } }, + Score = score, + Vector = withVector ? record.Embedding.ToArray() : null + }); } return resultTexts; - } public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload) @@ -62,9 +71,16 @@ namespace BotSharp.Plugin.SemanticKernel return true; } - public Task DeleteCollectionData(string collectionName, string id) + public async Task DeleteCollectionData(string collectionName, string id) { - throw new NotImplementedException(); + var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + + if (exist) + { + await _memoryStore.RemoveAsync(collectionName, id); + return true; + } + return false; } } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index f04d0be6..7e2462de 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; @@ -252,8 +253,9 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private string BuilFileUrl(string file) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 1dcb5edb..791e7f98 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -18,10 +18,10 @@ public class TencentCosPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.TencentCosStorage) + if (fileCoreSettings.Storage == FileStorageEnum.TencentCosStorage) { services.AddScoped(provider => { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b10e2160..155dae1b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -230,9 +230,13 @@ "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] }, - "FileStorage": { - "Default": "LocalFileStorage" + + "FileCore": { + "Storage": "LocalFileStorage", + "Pdf2TextConverter": "", + "Pdf2ImageConverter": "" }, + "TencentCos": { "AppId": "", "SecretId": "", @@ -254,10 +258,11 @@ }, "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - "TextEmbedding": "fastTextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", - "Pdf2TextConverter": "PigPdf2TextConverter" + "VectorDb": "Qdrant", + "TextEmbedding": { + "Provider": "openai", + "Model": "text-embedding-3-small" + } }, "SparkDesk": {