1. split knowledge menu

2. refine vector db
This commit is contained in:
Jicheng Lu 2024-08-20 13:32:22 -05:00
parent 5029455050
commit e96b217dfe
10 changed files with 39 additions and 31 deletions

View file

@ -25,9 +25,9 @@ public class PluginMenuDef
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<PluginMenuDef>? SubMenu { get; set; }
public PluginMenuDef(string lable, string? link = null, string? icon = null, int weight = 0)
public PluginMenuDef(string label, string? link = null, string? icon = null, int weight = 0)
{
Label = lable;
Label = label;
Link = link;
Icon = icon;
Weight = weight;

View file

@ -9,7 +9,7 @@ public interface IVectorDb
Task<IEnumerable<string>> GetCollections();
Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
Task CreateCollection(string collectionName, int dim);
Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
Task<bool> DeleteCollectionData(string collectionName, string id);
Task<bool> DeleteCollectionData(string collectionName, Guid id);
}

View file

@ -29,9 +29,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback
var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
await vectorDb.CreateCollection(collectionName, vector[0].Length);
var id = Guid.NewGuid().ToString();
var result = await vectorDb.Upsert(collectionName, id, vector[0],
args.Question,
var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0],
args.Question,
new Dictionary<string, string>
{
{ KnowledgePayloadName.Answer, args.Answer }

View file

@ -20,8 +20,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
});
var a = config["KnowledgeBase"];
services.AddScoped<ITextChopper, TextChopperService>();
services.AddScoped<IKnowledgeService, KnowledgeService>();
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
@ -32,7 +30,14 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
public bool AttachMenu(List<PluginMenuDef> menu)
{
var section = menu.First(x => x.Label == "Apps");
menu.Add(new PluginMenuDef("Knowledge Base", link: "/page/knowledge-base", icon: "bx bx-book-open", weight: section.Weight + 1));
menu.Add(new PluginMenuDef("Knowledge Base", icon: "bx bx-book-open", weight: section.Weight + 1)
{
SubMenu = new List<PluginMenuDef>
{
new PluginMenuDef("Vector", link: "page/knowledge-base/vector"),
new PluginMenuDef("Graph", link: "page/knowledge-base/graph")
}
});
return true;
}
}

View file

@ -53,11 +53,11 @@ public class MemoryVectorDb : IVectorDb
return await Task.FromResult(results);
}
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
_vectors[collectionName].Add(new VecRecord
{
Id = id,
Id = id.ToString(),
Vector = vector,
Text = text
});
@ -65,7 +65,7 @@ public class MemoryVectorDb : IVectorDb
return true;
}
public async Task<bool> DeleteCollectionData(string collectionName, string id)
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
return await Task.FromResult(false);
}

View file

@ -19,8 +19,7 @@ public partial class KnowledgeService
foreach (var line in lines)
{
var vec = await textEmbedding.GetVectorAsync(line);
var id = Guid.NewGuid().ToString();
await db.Upsert(collectionName, id, vec, line);
await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
index++;
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
}

View file

@ -6,8 +6,13 @@ public partial class KnowledgeService
{
try
{
if (!Guid.TryParse(id, out var guid))
{
return false;
}
var db = GetVectorDb();
return await db.DeleteCollectionData(collectionName, id);
return await db.DeleteCollectionData(collectionName, guid);
}
catch (Exception ex)
{

View file

@ -32,12 +32,12 @@ public class FaissDb : IVectorDb
throw new NotImplementedException();
}
public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
throw new NotImplementedException();
}
public Task<bool> DeleteCollectionData(string collectionName, string id)
public Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
throw new NotImplementedException();
}

View file

@ -92,14 +92,14 @@ public class QdrantDb : IVectorDb
}
}
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
// Insert vectors
var point = new PointStruct()
{
Id = new PointId()
{
Uuid = id
Uuid = id.ToString()
},
Vectors = vector,
Payload =
@ -137,7 +137,11 @@ public class QdrantDb : IVectorDb
return results;
}
var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
var points = await client.SearchAsync(collectionName,
vector,
limit: (ulong)limit,
scoreThreshold: confidence,
vectorsSelector: new WithVectorsSelector { Enable = withVector });
var pickFields = fields != null;
foreach (var point in points)
@ -174,15 +178,10 @@ public class QdrantDb : IVectorDb
return results;
}
public async Task<bool> DeleteCollectionData(string collectionName, string id)
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
if (!Guid.TryParse(id, out var guid))
{
return false;
}
var client = GetClient();
var result = await client.DeleteAsync(collectionName, guid);
var result = await client.DeleteAsync(collectionName, id);
return result.Status == UpdateStatus.Completed;
}

View file

@ -2,6 +2,7 @@ using BotSharp.Abstraction.Utilities;
using BotSharp.Abstraction.VectorStorage;
using BotSharp.Abstraction.VectorStorage.Models;
using Microsoft.SemanticKernel.Memory;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
@ -62,7 +63,7 @@ namespace BotSharp.Plugin.SemanticKernel
return resultTexts;
}
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload)
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)
{
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
@ -70,13 +71,13 @@ namespace BotSharp.Plugin.SemanticKernel
return true;
}
public async Task<bool> DeleteCollectionData(string collectionName, string id)
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
{
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
if (exist)
{
await _memoryStore.RemoveAsync(collectionName, id);
await _memoryStore.RemoveAsync(collectionName, id.ToString());
return true;
}
return false;