SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix vector db

Browse source
This commit is contained in:
Jicheng Lu 2024-10-07 13:10:48 -05:00
parent 41aaa7ea49
commit 70babdac7c
2 changed files with 45 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -60,7 +60,20 @@ public partial class KnowledgeService
}
// Save to vector db
var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents, file.FileSource);
var payload = new Dictionary<string, object>()
{
{ KnowledgePayloadName.DataSource, VectorDataSource.File },
{ KnowledgePayloadName.FileId, fileId.ToString() },
{ KnowledgePayloadName.FileName, file.FileName },
{ KnowledgePayloadName.FileSource, file.FileSource }
};
if (!string.IsNullOrWhiteSpace(file.FileUrl))
{
payload[KnowledgePayloadName.FileUrl] = file.FileUrl;
}
var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents, payload);
if (!dataIds.IsNullOrEmpty())
{
db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData
@ -119,7 +132,20 @@ public partial class KnowledgeService
var fileId = Guid.NewGuid();
var contentType = FileUtility.GetFileContentType(fileName);
var dataIds = await SaveToVectorDb(collectionName, fileId, fileName, contents, fileSource, fileUrl: refData?.Url);
var payload = new Dictionary<string, object>()
{
{ KnowledgePayloadName.DataSource, VectorDataSource.File },
{ KnowledgePayloadName.FileId, fileId.ToString() },
{ KnowledgePayloadName.FileName, fileName },
{ KnowledgePayloadName.FileSource, fileSource }
};
if (!string.IsNullOrWhiteSpace(refData?.Url))
{
payload[KnowledgePayloadName.FileUrl] = refData.Url;
}
var dataIds = await SaveToVectorDb(collectionName, fileId, fileName, contents, payload);
db.SaveKnolwedgeBaseFileMeta(new KnowledgeDocMetaData
{
Collection = collectionName,
@ -386,8 +412,7 @@ public partial class KnowledgeService
}
private async Task<IEnumerable<string>> SaveToVectorDb(
string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File, string? fileUrl = null)
string collectionName, Guid fileId, string fileName, IEnumerable<string> contents, Dictionary<string, object>? payload = null)
{
if (contents.IsNullOrEmpty())
{
@ -398,25 +423,12 @@ public partial class KnowledgeService
var vectorDb = GetVectorDb();
var textEmbedding = GetTextEmbedding(collectionName);
var payload = new Dictionary<string, object>
{
{ KnowledgePayloadName.DataSource, vectorDataSource },
{ KnowledgePayloadName.FileId, fileId.ToString() },
{ KnowledgePayloadName.FileName, fileName },
{ KnowledgePayloadName.FileSource, fileSource }
};
if (!string.IsNullOrWhiteSpace(fileUrl))
{
payload[KnowledgePayloadName.FileUrl] = fileUrl;
}
for (int i = 0; i < contents.Count(); i++)
{
var content = contents.ElementAt(i);
var vector = await textEmbedding.GetVectorAsync(content);
var dataId = Guid.NewGuid();
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload);
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, payload ?? new Dictionary<string, object>());
if (!saved) continue;

27
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -212,42 +212,45 @@ public class QdrantDb : IVectorDb
{
foreach (var item in payload)
{
if (item.Value is string str)
{
point.Payload[item.Key] = str;
}
else if (item.Value is bool b)
var value = item.Value?.ToString();
if (value == null) continue;
if (bool.TryParse(value, out var b))
{
point.Payload[item.Key] = b;
}
else if (item.Value is byte int8)
else if (byte.TryParse(value, out var int8))
{
point.Payload[item.Key] = int8;
}
else if (item.Value is short int16)
else if (short.TryParse(value, out var int16))
{
point.Payload[item.Key] = int16;
}
else if (item.Value is int int32)
else if (int.TryParse(value, out var int32))
{
point.Payload[item.Key] = int32;
}
else if (item.Value is long int64)
else if (long.TryParse(value, out var int64))
{
point.Payload[item.Key] = int64;
}
else if (item.Value is float f32)
else if (float.TryParse(value, out var f32))
{
point.Payload[item.Key] = f32;
}
else if (item.Value is double f64)
else if (double.TryParse(value, out var f64))
{
point.Payload[item.Key] = f64;
}
else if (item.Value is DateTime dt)
else if (DateTime.TryParse(value, out var dt))
{
point.Payload[item.Key] = dt.ToUniversalTime().ToString("o");
}
else
{
point.Payload[item.Key] = value;
}
}
}