SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add vector collection details

Browse source
This commit is contained in:
Jicheng Lu 2025-04-30 11:55:25 -05:00
parent 9febf00795
commit 5fe29f60b1
7 changed files with 163 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -10,6 +10,7 @@ public interface IKnowledgeService
Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
Task<bool> DeleteVectorCollection(string collectionName);
Task<IEnumerable<VectorCollectionConfig>> GetVectorCollections(string? type = null);
Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
Task<bool> DeleteVectorCollectionData(string collectionName, string id);

2
src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
View file

@ -10,6 +10,8 @@ public interface IVectorDb
=> throw new NotImplementedException();
Task<IEnumerable<string>> GetCollections()
=> throw new NotImplementedException();
Task<VectorCollectionDetails?> GetCollectionDetails(string collectionName)
=> throw new NotImplementedException();
Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
=> throw new NotImplementedException();
Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,

51
src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionDetails.cs Normal file
View file

@ -0,0 +1,51 @@
namespace BotSharp.Abstraction.VectorStorage.Models;
public class VectorCollectionDetails
{
[JsonPropertyName("status")]
public string Status { get; set; }
[JsonPropertyName("optimizer_status")]
public string OptimizerStatus { get; set; }
[JsonPropertyName("segments_count")]
public ulong SegmentsCount { get; set; }
[JsonPropertyName("vectors_count")]
public ulong VectorsCount { get; set; }
[JsonPropertyName("indexed_vectors_count")]
public ulong IndexedVectorsCount { get; set; }
[JsonPropertyName("points_count")]
public ulong PointsCount { get; set; }
[JsonPropertyName("inner_config")]
public VectorCollectionDetailConfig? InnerConfig { get; set; }
[JsonPropertyName("basic_info")]
public VectorCollectionConfig? BasicInfo { get; set; }
}
public class VectorCollectionDetailConfig
{
public VectorCollectionDetailConfigParam? Param { get; set; }
}
public class VectorCollectionDetailConfigParam
{
[JsonPropertyName("shard_number")]
public uint? ShardNumber { get; set; }
[JsonPropertyName("sharding_method")]
public string? ShardingMethod { get; set; }
[JsonPropertyName("replication_factor")]
public uint? ReplicationFactor { get; set; }
[JsonPropertyName("write_consistency_factor")]
public uint? WriteConsistencyFactor { get; set; }
[JsonPropertyName("read_fan_out_factor")]
public uint? ReadFanOutFactor { get; set; }
}

7
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -34,6 +34,13 @@ public class KnowledgeBaseController : ControllerBase
return collections.Select(x => VectorCollectionConfigViewModel.From(x));
}
[HttpGet("knowledge/vector/{collection}/details")]
public async Task<VectorCollectionDetailsViewModel?> GetVectorCollectionDetails([FromRoute] string collection)
{
var details = await _knowledgeService.GetVectorCollectionDetails(collection);
return VectorCollectionDetailsViewModel.From(details);
}
[HttpPost("knowledge/vector/create-collection")]
public async Task<bool> CreateVectorCollection([FromBody] CreateVectorCollectionRequest request)
{

23
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/View/VectorCollectionDetailsViewModel.cs Normal file
View file

@ -0,0 +1,23 @@
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class VectorCollectionDetailsViewModel : VectorCollectionDetails
{
public static VectorCollectionDetailsViewModel? From(VectorCollectionDetails? model)
{
if (model == null) return null;
return new VectorCollectionDetailsViewModel
{
Status = model.Status,
OptimizerStatus = model.OptimizerStatus,
SegmentsCount = model.SegmentsCount,
VectorsCount = model.VectorsCount,
IndexedVectorsCount = model.IndexedVectorsCount,
PointsCount = model.PointsCount,
InnerConfig = model.InnerConfig,
BasicInfo = model.BasicInfo
};
}
}

63
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
View file

@ -64,7 +64,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when creating a vector collection ({collectionName}).");
return false;
}
}
@ -86,11 +86,38 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when getting vector db collections.");
return Enumerable.Empty<VectorCollectionConfig>();
}
}
public async Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName)) return null;
var db = _services.GetRequiredService<IBotSharpRepository>();
var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
{
CollectionNames = [collectionName]
}).ToList();
var vectorDb = GetVectorDb();
var details = await vectorDb.GetCollectionDetails(collectionName);
if (details != null)
{
details.BasicInfo = configs.FirstOrDefault();
}
return details;
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when getting vector db collection details.");
return null;
}
}
public async Task<bool> DeleteVectorCollection(string collectionName)
{
try
@ -118,7 +145,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when deleting collection ({collectionName}).");
return false;
}
}
@ -146,7 +173,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when creating vector collection data.");
return false;
}
}
@ -155,13 +182,15 @@ public partial class KnowledgeService
{
try
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(update.Text)
|| !Guid.TryParse(update.Id, out var guid))
{
return false;
}
var db = GetVectorDb();
var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
var found = await db.GetCollectionData(collectionName, [guid]);
if (found.IsNullOrEmpty())
{
return false;
@ -176,7 +205,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when updating vector collection data.");
return false;
}
}
@ -185,18 +214,18 @@ public partial class KnowledgeService
{
try
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(update.Text)
|| !Guid.TryParse(update.Id, out var guid))
{
return false;
}
var db = GetVectorDb();
var found = await db.GetCollectionData(collectionName, new List<Guid> { guid },
withVector: true,
withPayload: true);
var found = await db.GetCollectionData(collectionName, [guid], withVector: true, withPayload: true);
if (!found.IsNullOrEmpty())
{
if (found.First().Data["text"].ToString() == update.Text)
if (found.First().Data[KnowledgePayloadName.Text].ToString() == update.Text)
{
// Only update payload
return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload);
@ -212,7 +241,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when updating vector collection data.");
return false;
}
}
@ -231,7 +260,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}-{id}).");
return false;
}
}
@ -246,7 +275,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when deleting vector collection data ({collectionName}).");
return false;
}
}
@ -266,7 +295,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when getting vector knowledge collection data ({collectionName}).");
return new StringIdPagedItems<VectorSearchResult>();
}
}
@ -287,7 +316,7 @@ public partial class KnowledgeService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning(ex, $"Error when searching vector knowledge ({collectionName}).");
return Enumerable.Empty<VectorSearchResult>();
}
}

33
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -96,6 +96,39 @@ public class QdrantDb : IVectorDb
var collections = await GetClient().ListCollectionsAsync();
return collections.ToList();
}
public async Task<VectorCollectionDetails?> GetCollectionDetails(string collectionName)
{
var exist = await DoesCollectionExist(collectionName);
if (!exist) return null;
var client = GetClient();
var details = await client.GetCollectionInfoAsync(collectionName);
if (details == null) return null;
return new VectorCollectionDetails
{
Status = details.Status.ToString(),
OptimizerStatus = details.OptimizerStatus.ToString(),
SegmentsCount = details.SegmentsCount,
InnerConfig = new VectorCollectionDetailConfig
{
Param = new VectorCollectionDetailConfigParam
{
ShardNumber = details.Config?.Params?.ShardNumber,
ShardingMethod = details.Config?.Params?.ShardingMethod.ToString(),
ReplicationFactor = details.Config?.Params?.ReplicationFactor,
WriteConsistencyFactor = details.Config?.Params?.WriteConsistencyFactor,
ReadFanOutFactor = details.Config?.Params?.ReadFanOutFactor
}
},
VectorsCount = details.VectorsCount,
IndexedVectorsCount = details.IndexedVectorsCount,
PointsCount = details.PointsCount
};
}
#endregion
#region Collection data