Support more data type in Qdrant payload.
This commit is contained in:
parent
08d1a40916
commit
af484579f5
|
|
@ -12,7 +12,7 @@ public interface IVectorDb
|
|||
Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);
|
||||
Task<bool> CreateCollection(string collectionName, int dimension);
|
||||
Task<bool> DeleteCollection(string collectionName);
|
||||
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
|
||||
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null);
|
||||
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
|
||||
Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
|
||||
Task<bool> DeleteCollectionAllData(string collectionName);
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.VectorStorage.Models;
|
|||
public class VectorCollectionData
|
||||
{
|
||||
public string Id { get; set; }
|
||||
public Dictionary<string, string> Data { get; set; } = new();
|
||||
public Dictionary<string, object> Data { get; set; } = new();
|
||||
public double? Score { get; set; }
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
|
|
|
|||
|
|
@ -6,5 +6,5 @@ public class VectorCreateModel
|
|||
{
|
||||
public string Text { get; set; }
|
||||
public string DataSource { get; set; } = VectorDataSource.Api;
|
||||
public Dictionary<string, string>? Payload { get; set; }
|
||||
public Dictionary<string, object>? Payload { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,5 +11,5 @@ public class VectorKnowledgeCreateRequest
|
|||
public string DataSource { get; set; } = VectorDataSource.Api;
|
||||
|
||||
[JsonPropertyName("payload")]
|
||||
public Dictionary<string, string>? Payload { get; set; }
|
||||
public Dictionary<string, object>? Payload { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ public class VectorKnowledgeViewModel
|
|||
public string Id { get; set; }
|
||||
|
||||
[JsonPropertyName("data")]
|
||||
public IDictionary<string, string> Data { get; set; }
|
||||
public IDictionary<string, object> Data { get; set; }
|
||||
|
||||
[JsonPropertyName("score")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback
|
|||
var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel
|
||||
{
|
||||
Text = args.Question,
|
||||
Payload = new Dictionary<string, string>
|
||||
Payload = new Dictionary<string, object>
|
||||
{
|
||||
{ KnowledgePayloadName.Answer, args.Answer }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class MemoryVectorDb : IVectorDb
|
|||
.Take(limit)
|
||||
.Select(i => new VectorCollectionData
|
||||
{
|
||||
Data = new Dictionary<string, string> { { "text", _vectors[collectionName][i].Text } },
|
||||
Data = new Dictionary<string, object> { { "text", _vectors[collectionName][i].Text } },
|
||||
Score = similarities[i],
|
||||
Vector = withVector ? _vectors[collectionName][i].Vector : null,
|
||||
})
|
||||
|
|
@ -68,7 +68,7 @@ public class MemoryVectorDb : IVectorDb
|
|||
return await Task.FromResult(results);
|
||||
}
|
||||
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
|
||||
{
|
||||
_vectors[collectionName].Add(new VecRecord
|
||||
{
|
||||
|
|
|
|||
|
|
@ -398,7 +398,7 @@ public partial class KnowledgeService
|
|||
var vectorDb = GetVectorDb();
|
||||
var textEmbedding = GetTextEmbedding(collectionName);
|
||||
|
||||
var payload = new Dictionary<string, string>
|
||||
var payload = new Dictionary<string, object>
|
||||
{
|
||||
{ KnowledgePayloadName.DataSource, vectorDataSource },
|
||||
{ KnowledgePayloadName.FileId, fileId.ToString() },
|
||||
|
|
|
|||
|
|
@ -196,7 +196,7 @@ public partial class KnowledgeService
|
|||
withPayload: true);
|
||||
if (!found.IsNullOrEmpty())
|
||||
{
|
||||
if (found.First().Data["text"] == update.Text)
|
||||
if (found.First().Data["text"].ToString() == update.Text)
|
||||
{
|
||||
// Only update payload
|
||||
return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload);
|
||||
|
|
|
|||
|
|
@ -19,23 +19,23 @@ public class PrimaryStagePlanFn : IFunctionCallback
|
|||
{
|
||||
var agentService = _services.GetRequiredService<IAgentService>();
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
|
||||
var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
|
||||
// var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
|
||||
// var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
|
||||
|
||||
state.SetState("max_tokens", "4096");
|
||||
var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
|
||||
var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
|
||||
// var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
|
||||
|
||||
// Get knowledge from vectordb
|
||||
var hooks = _services.GetServices<IKnowledgeHook>();
|
||||
var knowledges = new List<string>();
|
||||
foreach (var question in task.Questions)
|
||||
{
|
||||
var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions
|
||||
/*var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions
|
||||
{
|
||||
Confidence = 0.4f
|
||||
});
|
||||
knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));
|
||||
knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));*/
|
||||
|
||||
foreach (var hook in hooks)
|
||||
{
|
||||
|
|
@ -43,6 +43,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
|
|||
knowledges.AddRange(k);
|
||||
}
|
||||
}
|
||||
knowledges = knowledges.Distinct().ToList();
|
||||
|
||||
// Get first stage planning prompt
|
||||
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Qdrant.Client" Version="1.10.0" />
|
||||
<PackageReference Include="Qdrant.Client" Version="1.11.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -142,7 +142,13 @@ public class QdrantDb : IVectorDb
|
|||
var points = response?.Result?.Select(x => new VectorCollectionData
|
||||
{
|
||||
Id = x.Id?.Uuid ?? string.Empty,
|
||||
Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue),
|
||||
Data = x.Payload.ToDictionary(p => p.Key, p => p.Value.KindCase switch
|
||||
{
|
||||
Value.KindOneofCase.StringValue => p.Value.StringValue,
|
||||
Value.KindOneofCase.BoolValue => p.Value.BoolValue,
|
||||
Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
|
||||
_ => new object()
|
||||
}),
|
||||
Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null
|
||||
})?.ToList() ?? new List<VectorCollectionData>();
|
||||
|
||||
|
|
@ -175,12 +181,18 @@ public class QdrantDb : IVectorDb
|
|||
return points.Select(x => new VectorCollectionData
|
||||
{
|
||||
Id = x.Id?.Uuid ?? string.Empty,
|
||||
Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(),
|
||||
Data = x.Payload?.ToDictionary(p => p.Key, p => p.Value.KindCase switch
|
||||
{
|
||||
Value.KindOneofCase.StringValue => p.Value.StringValue,
|
||||
Value.KindOneofCase.BoolValue => p.Value.BoolValue,
|
||||
Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
|
||||
_ => new object()
|
||||
}) ?? new(),
|
||||
Vector = x.Vectors?.Vector?.Data?.ToArray()
|
||||
});
|
||||
}
|
||||
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
|
||||
{
|
||||
// Insert vectors
|
||||
var point = new PointStruct()
|
||||
|
|
@ -200,7 +212,42 @@ public class QdrantDb : IVectorDb
|
|||
{
|
||||
foreach (var item in payload)
|
||||
{
|
||||
point.Payload[item.Key] = item.Value;
|
||||
if (item.Value is string str)
|
||||
{
|
||||
point.Payload[item.Key] = str;
|
||||
}
|
||||
else if (item.Value is bool b)
|
||||
{
|
||||
point.Payload[item.Key] = b;
|
||||
}
|
||||
else if (item.Value is byte int8)
|
||||
{
|
||||
point.Payload[item.Key] = int8;
|
||||
}
|
||||
else if (item.Value is short int16)
|
||||
{
|
||||
point.Payload[item.Key] = int16;
|
||||
}
|
||||
else if (item.Value is int int32)
|
||||
{
|
||||
point.Payload[item.Key] = int32;
|
||||
}
|
||||
else if (item.Value is long int64)
|
||||
{
|
||||
point.Payload[item.Key] = int64;
|
||||
}
|
||||
else if (item.Value is float f32)
|
||||
{
|
||||
point.Payload[item.Key] = f32;
|
||||
}
|
||||
else if (item.Value is double f64)
|
||||
{
|
||||
point.Payload[item.Key] = f64;
|
||||
}
|
||||
else if (item.Value is DateTime dt)
|
||||
{
|
||||
point.Payload[item.Key] = dt.ToUniversalTime().ToString("o");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -241,7 +288,13 @@ public class QdrantDb : IVectorDb
|
|||
results = points.Select(x => new VectorCollectionData
|
||||
{
|
||||
Id = x.Id.Uuid,
|
||||
Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue),
|
||||
Data = x.Payload.ToDictionary(p => p.Key, p => p.Value.KindCase switch
|
||||
{
|
||||
Value.KindOneofCase.StringValue => p.Value.StringValue,
|
||||
Value.KindOneofCase.BoolValue => p.Value.BoolValue,
|
||||
Value.KindOneofCase.IntegerValue => p.Value.IntegerValue,
|
||||
_ => new object()
|
||||
}),
|
||||
Score = x.Score,
|
||||
Vector = x.Vectors?.Vector?.Data?.ToArray()
|
||||
}).ToList();
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ public class DbKnowledgeService
|
|||
await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel
|
||||
{
|
||||
Text = item.Question,
|
||||
Payload = new Dictionary<string, string>
|
||||
Payload = new Dictionary<string, object>
|
||||
{
|
||||
{ KnowledgePayloadName.Answer, item.Answer }
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue