From a7b263e2eb172b1331cd000e74b6e334918c99ba Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 26 Jun 2023 23:08:23 -0500 Subject: [PATCH] All user to config providers in KnowledgeBase.#77 --- .../Knowledges/KnowledgeBaseSettings.cs | 1 + .../Knowledges/Services/KnowledgeService.cs | 29 ++++++++-------- .../Plugins/MemVecDb/MemVectorDatabase.cs | 33 ++++++++++++++++++- src/WebStarter/WebStarter.csproj | 3 +- src/WebStarter/appsettings.json | 4 ++- 5 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs index b2b3acdd..a56767e9 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs @@ -4,4 +4,5 @@ public class KnowledgeBaseSettings { public string VectorDb { get; set; } public string TextEmbedding { get; set; } + public string TextCompletion { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs index cf12ae62..9e0113f0 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs @@ -9,17 +9,14 @@ public class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly ITextCompletion _textCompletion; private readonly ITextChopper _textChopper; public KnowledgeService(IServiceProvider services, KnowledgeBaseSettings settings, - ITextCompletion textCompletion, ITextChopper textChopper) { _services = services; _settings = settings; - _textCompletion = textCompletion; _textChopper = textChopper; } @@ -33,14 +30,14 @@ public class KnowledgeService : IKnowledgeService }); // Store chunks in local file system - var knowledgeStoreDir = Path.Combine("knowledge_chunks", knowledge.AgentId); + var knowledgeStoreDir = Path.Combine("knowledge_base"); if (!Directory.Exists(knowledgeStoreDir)) { Directory.CreateDirectory(knowledgeStoreDir); } - var knowledgePath = Path.Combine(knowledgeStoreDir, "chuncks"); - File.WriteAllLines(knowledgePath + ".txt", lines); + var knowledgePath = Path.Combine(knowledgeStoreDir, knowledge.AgentId + ".txt"); + File.WriteAllLines(knowledgePath, lines); var db = GetVectorDb(); var textEmbedding = GetTextEmbedding(); @@ -60,17 +57,10 @@ public class KnowledgeService : IKnowledgeService var vector = textEmbedding.GetVector(retrievalModel.Question); // Scan local knowledge directory - var knowledgeName = ""; - var chunks = new string[0]; - - foreach (var file in Directory.GetFiles(Path.Combine("knowledge_chunks", retrievalModel.AgentId))) - { - knowledgeName = new FileInfo(file).Name.Split('.').First(); - chunks = File.ReadAllLines(file); - } + var chunks = File.ReadAllLines(Path.Combine("knowledge_base", retrievalModel.AgentId + ".txt")); // Vector search - var result = await GetVectorDb().Search(knowledgeName, vector); + var result = await GetVectorDb().Search(retrievalModel.AgentId, vector); // Restore var prompt = ""; @@ -83,7 +73,7 @@ public class KnowledgeService : IKnowledgeService prompt += "Answer the user's question based on the content provided above, and your reply should be as concise and organized as possible.\r\n"; prompt += $"Question: {retrievalModel.Question}\r\nAnswer: "; - var completion = await _textCompletion.GetCompletion(prompt); + var completion = await GetTextCompletion().GetCompletion(prompt); return completion; } @@ -100,4 +90,11 @@ public class KnowledgeService : IKnowledgeService .FirstOrDefault(x => x.GetType().Name == _settings.TextEmbedding); return embedding; } + + public ITextCompletion GetTextCompletion() + { + var textCompletion = _services.GetServices() + .FirstOrDefault(x => x.GetType().Name == _settings.TextCompletion); + return textCompletion; + } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs b/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs index be09c97d..95b611fd 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.VectorStorage; +using Tensorflow.NumPy; namespace BotSharp.Core.Plugins.MemVecDb; @@ -20,7 +21,16 @@ public class MemVectorDatabase : IVectorDb public Task> Search(string collectionName, float[] vector, int limit = 10) { - throw new NotImplementedException(); + var cosineList = new List(); + for (int i = 0; i < _vectors[collectionName].Count; i++) + { + var p = CalCosineSimilarity(vector, _vectors[collectionName][i].Vector); + cosineList.Add(p); + } + var similarities = cosineList.ToArray(); + var indice = np.argsort(similarities).ToArray() + .Reverse().Take(limit).ToList(); + return Task.FromResult(indice); } public Task Upsert(string collectionName, int id, float[] vector) @@ -33,4 +43,25 @@ public class MemVectorDatabase : IVectorDb return Task.CompletedTask; } + + private double CalCosineSimilarity(float[] vector1, float[] vector2) + { + NDArray a = vector1; + NDArray b = vector2; + double num = np.dot(a, b); + if(num == 0) + { + return 0.0; + } + + b = np.square(a); + var x = np.sqrt(np.sum(b)); + var x3 = np.sum(np.square(vector2)); + double num2 = np.sqrt(x) * np.sqrt(x3); + if(num2 == 0) + { + return 0.0; + } + return num / num2; + } } diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index e2abd70d..63ac1254 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -31,6 +31,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 10509e05..e183aba7 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -68,7 +68,9 @@ }, "KnowledgeBase": { - "VectorDb": "MemVecDbProvider" + "VectorDb": "MemVectorDatabase", + "TextEmbedding": "fastTextEmbeddingProvider", + "TextCompletion": "TextCompletionProvider" }, "PluginLoader": {