diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 1021ea1c..ca040086 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -19,6 +19,8 @@ public interface IKnowledgeService #region Graph Task SearchGraphKnowledge(string query, GraphSearchOptions options); - Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); + #endregion + + #region Document #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs deleted file mode 100644 index 771b070b..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs +++ /dev/null @@ -1,10 +0,0 @@ -using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.VectorStorage.Models; - -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeSearchResult -{ - public IEnumerable VectorResult { get; set; } - public GraphSearchResult GraphResult { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 827c2d7c..fd446bd2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -143,29 +143,7 @@ public class KnowledgeBaseController : ControllerBase #endregion - #region Knowledge - [HttpPost("/knowledge/search")] - public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) - { - var vectorOptions = new VectorSearchOptions - { - Fields = request.VectorParams.Fields, - Limit = request.VectorParams.Limit ?? 5, - Confidence = request.VectorParams.Confidence ?? 0.5f, - WithVector = request.VectorParams.WithVector - }; - - var graphOptions = new GraphSearchOptions - { - Method = request.GraphParams.Method - }; - - var result = await _knowledgeService.SearchKnowledge(request.Text, request.VectorParams.Collection, vectorOptions, graphOptions); - return new KnowledgeSearchViewModel - { - VectorResult = result?.VectorResult?.Select(x => VectorKnowledgeViewModel.From(x)), - GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null - }; - } + #region Document + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs deleted file mode 100644 index f862f184..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class KnowledgeSearchViewModel -{ - [JsonPropertyName("vector_result")] - public IEnumerable? VectorResult { get; set; } - - [JsonPropertyName("graph_result")] - public GraphKnowledgeViewModel? GraphResult { get; set; } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs deleted file mode 100644 index 20f80fec..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class SearchKnowledgeRequest -{ - [JsonPropertyName("text")] - public string Text { get; set; } = string.Empty; - - #region Vector - [JsonPropertyName("vector_params")] - public VectorParam VectorParams { get; set; } - #endregion - - #region Graph - [JsonPropertyName("graph_params")] - public GraphParam GraphParams { get; set; } - #endregion -} - -public class VectorParam -{ - [JsonPropertyName("collection")] - public string Collection { get; set; } - - [JsonPropertyName("fields")] - public IEnumerable? Fields { get; set; } - - [JsonPropertyName("limit")] - public int? Limit { get; set; } = 5; - - [JsonPropertyName("confidence")] - public float? Confidence { get; set; } = 0.5f; - - [JsonPropertyName("with_vector")] - public bool WithVector { get; set; } -} - -public class GraphParam -{ - [JsonPropertyName("method")] - public string Method { get; set; } = string.Empty; -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs deleted file mode 100644 index 2c4004fc..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ /dev/null @@ -1,69 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) - { - var index = 0; - var lines = TextChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(collectionName); - - await db.CreateCollection(collectionName, textEmbedding.GetDimension()); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(collectionName, Guid.NewGuid(), vec, line); - index++; - Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); - } - } - - public async Task CreateVectorCollection(string collectionName, int dimension) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var db = GetVectorDb(); - return await db.CreateCollection(collectionName, dimension); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } - - public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) - { - return false; - } - - var textEmbedding = GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(create.Text); - - var db = GetVectorDb(); - var guid = Guid.NewGuid(); - return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs deleted file mode 100644 index b0973a99..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task DeleteVectorCollection(string collectionName) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName)) - { - return false; - } - - var db = GetVectorDb(); - return await db.DeleteCollection(collectionName); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } - - public async Task DeleteVectorCollectionData(string collectionName, string id) - { - try - { - if (!Guid.TryParse(id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - return await db.DeleteCollectionData(collectionName, guid); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs new file mode 100644 index 00000000..0e4710bc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -0,0 +1,27 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) + { + var index = 0; + var lines = TextChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + var db = GetVectorDb(); + var textEmbedding = GetTextEmbedding(collectionName); + + await db.CreateCollection(collectionName, textEmbedding.GetDimension()); + foreach (var line in lines) + { + var vec = await textEmbedding.GetVectorAsync(line); + await db.Upsert(collectionName, Guid.NewGuid(), vec, line); + index++; + Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs deleted file mode 100644 index dcffa944..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ /dev/null @@ -1,103 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> GetVectorCollections() - { - try - { - var db = GetVectorDb(); - return await db.GetCollections(); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}"); - return Enumerable.Empty(); - } - } - - public async Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter) - { - try - { - var db = GetVectorDb(); - var pagedResult = await db.GetPagedCollectionData(collectionName, filter); - return new StringIdPagedItems - { - Count = pagedResult.Count, - Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)), - NextId = pagedResult.NextId, - }; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new StringIdPagedItems(); - } - } - - public async Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options) - { - try - { - var textEmbedding = GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(query); - - // Vector search - var db = GetVectorDb(); - var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); - - var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); - return results; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new List(); - } - } - - public async Task SearchGraphKnowledge(string query, GraphSearchOptions options) - { - try - { - var db = GetGraphDb(); - var found = await db.Search(query, options); - return new GraphSearchResult - { - Result = found.Result - }; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}"); - return new GraphSearchResult(); - } - } - - public async Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions) - { - try - { - var textEmbedding = GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(query); - - var vectorDb = GetVectorDb(); - var vectorRes = await vectorDb.Search(collectionName, vector, vectorOptions.Fields, limit: vectorOptions.Limit ?? 5, - confidence: vectorOptions.Confidence ?? 0.5f, withVector: vectorOptions.WithVector); - - var graphDb = GetGraphDb(); - var graphRes = await graphDb.Search(query, graphOptions); - return new KnowledgeSearchResult - { - VectorResult = vectorRes.Select(x => VectorSearchResult.CopyFrom(x)), - GraphResult = new GraphSearchResult { Result = graphRes.Result } - }; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when searching knowledge (Vector collection: {collectionName}) (Query: {query}). {ex.Message}\r\n{ex.InnerException}"); - return new KnowledgeSearchResult(); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs new file mode 100644 index 00000000..7c8f7168 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs @@ -0,0 +1,22 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task SearchGraphKnowledge(string query, GraphSearchOptions options) + { + try + { + var db = GetGraphDb(); + var found = await db.Search(query, options); + return new GraphSearchResult + { + Result = found.Result + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}"); + return new GraphSearchResult(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs deleted file mode 100644 index 0fa1f5cd..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update) - { - try - { - if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid)) - { - return false; - } - - var db = GetVectorDb(); - var found = await db.GetCollectionData(collectionName, new List { guid }); - if (found.IsNullOrEmpty()) - { - return false; - } - - var textEmbedding = GetTextEmbedding(collectionName); - var vector = await textEmbedding.GetVectorAsync(update.Text); - return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}"); - return false; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs new file mode 100644 index 00000000..9afd7c30 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -0,0 +1,170 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + #region Collection + public async Task CreateVectorCollection(string collectionName, int dimension) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.CreateCollection(collectionName, dimension); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public async Task> GetVectorCollections() + { + try + { + var db = GetVectorDb(); + return await db.GetCollections(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + + public async Task DeleteVectorCollection(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollection(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + #endregion + + #region Collection data + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text)) + { + return false; + } + + var textEmbedding = GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(create.Text); + + var db = GetVectorDb(); + var guid = Guid.NewGuid(); + return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public async Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid)) + { + return false; + } + + var db = GetVectorDb(); + var found = await db.GetCollectionData(collectionName, new List { guid }); + if (found.IsNullOrEmpty()) + { + return false; + } + + var textEmbedding = GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(update.Text); + return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public async Task DeleteVectorCollectionData(string collectionName, string id) + { + try + { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollectionData(collectionName, guid); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public async Task> GetPagedVectorCollectionData(string collectionName, VectorFilter filter) + { + try + { + var db = GetVectorDb(); + var pagedResult = await db.GetPagedCollectionData(collectionName, filter); + return new StringIdPagedItems + { + Count = pagedResult.Count, + Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)), + NextId = pagedResult.NextId, + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new StringIdPagedItems(); + } + } + + public async Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options) + { + try + { + var textEmbedding = GetTextEmbedding(collectionName); + var vector = await textEmbedding.GetVectorAsync(query); + + // Vector search + var db = GetVectorDb(); + var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); + + var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); + return results; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + #endregion +}