1. split knowledge menu
2. refine vector db
This commit is contained in:
parent
5029455050
commit
e96b217dfe
|
|
@ -25,9 +25,9 @@ public class PluginMenuDef
|
|||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<PluginMenuDef>? SubMenu { get; set; }
|
||||
|
||||
public PluginMenuDef(string lable, string? link = null, string? icon = null, int weight = 0)
|
||||
public PluginMenuDef(string label, string? link = null, string? icon = null, int weight = 0)
|
||||
{
|
||||
Label = lable;
|
||||
Label = label;
|
||||
Link = link;
|
||||
Icon = icon;
|
||||
Weight = weight;
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ public interface IVectorDb
|
|||
Task<IEnumerable<string>> GetCollections();
|
||||
Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
|
||||
Task CreateCollection(string collectionName, int dim);
|
||||
Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
|
||||
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
|
||||
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
|
||||
Task<bool> DeleteCollectionData(string collectionName, string id);
|
||||
Task<bool> DeleteCollectionData(string collectionName, Guid id);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,9 +29,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback
|
|||
var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
|
||||
await vectorDb.CreateCollection(collectionName, vector[0].Length);
|
||||
|
||||
var id = Guid.NewGuid().ToString();
|
||||
var result = await vectorDb.Upsert(collectionName, id, vector[0],
|
||||
args.Question,
|
||||
var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0],
|
||||
args.Question,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ KnowledgePayloadName.Answer, args.Answer }
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
|
|||
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
|
||||
});
|
||||
|
||||
var a = config["KnowledgeBase"];
|
||||
|
||||
services.AddScoped<ITextChopper, TextChopperService>();
|
||||
services.AddScoped<IKnowledgeService, KnowledgeService>();
|
||||
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
|
||||
|
|
@ -32,7 +30,14 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
|
|||
public bool AttachMenu(List<PluginMenuDef> menu)
|
||||
{
|
||||
var section = menu.First(x => x.Label == "Apps");
|
||||
menu.Add(new PluginMenuDef("Knowledge Base", link: "/page/knowledge-base", icon: "bx bx-book-open", weight: section.Weight + 1));
|
||||
menu.Add(new PluginMenuDef("Knowledge Base", icon: "bx bx-book-open", weight: section.Weight + 1)
|
||||
{
|
||||
SubMenu = new List<PluginMenuDef>
|
||||
{
|
||||
new PluginMenuDef("Vector", link: "page/knowledge-base/vector"),
|
||||
new PluginMenuDef("Graph", link: "page/knowledge-base/graph")
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,11 +53,11 @@ public class MemoryVectorDb : IVectorDb
|
|||
return await Task.FromResult(results);
|
||||
}
|
||||
|
||||
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
{
|
||||
_vectors[collectionName].Add(new VecRecord
|
||||
{
|
||||
Id = id,
|
||||
Id = id.ToString(),
|
||||
Vector = vector,
|
||||
Text = text
|
||||
});
|
||||
|
|
@ -65,7 +65,7 @@ public class MemoryVectorDb : IVectorDb
|
|||
return true;
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteCollectionData(string collectionName, string id)
|
||||
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||
{
|
||||
return await Task.FromResult(false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@ public partial class KnowledgeService
|
|||
foreach (var line in lines)
|
||||
{
|
||||
var vec = await textEmbedding.GetVectorAsync(line);
|
||||
var id = Guid.NewGuid().ToString();
|
||||
await db.Upsert(collectionName, id, vec, line);
|
||||
await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
|
||||
index++;
|
||||
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,13 @@ public partial class KnowledgeService
|
|||
{
|
||||
try
|
||||
{
|
||||
if (!Guid.TryParse(id, out var guid))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var db = GetVectorDb();
|
||||
return await db.DeleteCollectionData(collectionName, id);
|
||||
return await db.DeleteCollectionData(collectionName, guid);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -32,12 +32,12 @@ public class FaissDb : IVectorDb
|
|||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public Task<bool> DeleteCollectionData(string collectionName, string id)
|
||||
public Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,14 +92,14 @@ public class QdrantDb : IVectorDb
|
|||
}
|
||||
}
|
||||
|
||||
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
|
||||
{
|
||||
// Insert vectors
|
||||
var point = new PointStruct()
|
||||
{
|
||||
Id = new PointId()
|
||||
{
|
||||
Uuid = id
|
||||
Uuid = id.ToString()
|
||||
},
|
||||
Vectors = vector,
|
||||
Payload =
|
||||
|
|
@ -137,7 +137,11 @@ public class QdrantDb : IVectorDb
|
|||
return results;
|
||||
}
|
||||
|
||||
var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
|
||||
var points = await client.SearchAsync(collectionName,
|
||||
vector,
|
||||
limit: (ulong)limit,
|
||||
scoreThreshold: confidence,
|
||||
vectorsSelector: new WithVectorsSelector { Enable = withVector });
|
||||
|
||||
var pickFields = fields != null;
|
||||
foreach (var point in points)
|
||||
|
|
@ -174,15 +178,10 @@ public class QdrantDb : IVectorDb
|
|||
return results;
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteCollectionData(string collectionName, string id)
|
||||
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||
{
|
||||
if (!Guid.TryParse(id, out var guid))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var client = GetClient();
|
||||
var result = await client.DeleteAsync(collectionName, guid);
|
||||
var result = await client.DeleteAsync(collectionName, id);
|
||||
return result.Status == UpdateStatus.Completed;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ using BotSharp.Abstraction.Utilities;
|
|||
using BotSharp.Abstraction.VectorStorage;
|
||||
using BotSharp.Abstraction.VectorStorage.Models;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
|
|
@ -62,7 +63,7 @@ namespace BotSharp.Plugin.SemanticKernel
|
|||
return resultTexts;
|
||||
}
|
||||
|
||||
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload)
|
||||
public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)
|
||||
{
|
||||
#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
|
||||
await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
|
||||
|
|
@ -70,13 +71,13 @@ namespace BotSharp.Plugin.SemanticKernel
|
|||
return true;
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteCollectionData(string collectionName, string id)
|
||||
public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
|
||||
{
|
||||
var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
|
||||
|
||||
if (exist)
|
||||
{
|
||||
await _memoryStore.RemoveAsync(collectionName, id);
|
||||
await _memoryStore.RemoveAsync(collectionName, id.ToString());
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
|
|
|||
Loading…
Reference in a new issue