From e5e322b3d76a2f13c2252675a02783674117754b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 27 Jun 2023 18:36:50 -0500 Subject: [PATCH] #81 --- .../VectorStorage/IVectorDb.cs | 4 +- .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Knowledges/Services/KnowledgeService.cs | 21 ++-------- .../LLamaSharp/ChatCompletionProvider.cs | 41 +++++-------------- .../Plugins/LLamaSharp/LLamaSharpPlugin.cs | 1 + .../Plugins/LLamaSharp/LlamaAiModel.cs | 12 +++--- .../LLamaSharp/TextCompletionProvider.cs | 33 +++++++++++++++ .../LLamaSharp/TextEmbeddingProvider.cs | 14 +++++-- .../Plugins/MemVecDb/MemVectorDatabase.cs | 14 ++++--- .../Providers/FaissDb.cs | 4 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 32 +++++++++++++-- .../BotSharp.Plugin.Qdrant/QdrantPlugin.cs | 2 +- src/WebStarter/Program.cs | 2 - src/WebStarter/appsettings.json | 3 +- 14 files changed, 109 insertions(+), 76 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextCompletionProvider.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ccd41e35..dd8e193f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,6 +4,6 @@ public interface IVectorDb { Task> GetCollections(); Task CreateCollection(string collectionName, int dim); - Task Upsert(string collectionName, int id, float[] vector); - Task> Search(string collectionName, float[] vector, int limit = 10); + Task Upsert(string collectionName, int id, float[] vector, string text); + Task> Search(string collectionName, float[] vector, int limit = 10); } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index eeaa3b2d..9e348243 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -64,7 +64,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs index b0459dc4..1e0d8a31 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Knowledges.Models; -using System.IO; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.VectorStorage; @@ -9,17 +8,14 @@ public class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly IAgentService _agentService; private readonly ITextChopper _textChopper; public KnowledgeService(IServiceProvider services, KnowledgeBaseSettings settings, - IAgentService agentService, ITextChopper textChopper) { _services = services; _settings = settings; - _agentService = agentService; _textChopper = textChopper; } @@ -32,11 +28,6 @@ public class KnowledgeService : IKnowledgeService Conjunction = 32 }); - // Store chunks in local file system - var agentDataDir = _agentService.GetAgentDataDir(knowledge.AgentId); - var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt"); - File.WriteAllLines(knowledgePath, lines); - var db = GetVectorDb(); var textEmbedding = GetTextEmbedding(); @@ -44,7 +35,7 @@ public class KnowledgeService : IKnowledgeService foreach (var line in lines) { var vec = textEmbedding.GetVector(line); - await db.Upsert(knowledge.AgentId, idStart, vec); + await db.Upsert(knowledge.AgentId, idStart, vec, line); idStart++; } } @@ -54,23 +45,19 @@ public class KnowledgeService : IKnowledgeService var textEmbedding = GetTextEmbedding(); var vector = textEmbedding.GetVector(retrievalModel.Question); - // Scan local knowledge directory - var agentDataDir = _agentService.GetAgentDataDir(retrievalModel.AgentId); - var chunks = File.ReadAllLines(Path.Combine(agentDataDir, "knowledge.txt")); - // Vector search var result = await GetVectorDb().Search(retrievalModel.AgentId, vector); // Restore var prompt = ""; - foreach (var r in result) + foreach (var knowledge in result) { - prompt += chunks[r] + "\n"; + prompt += knowledge + "\n"; } prompt += "\r\n###\r\n"; prompt += "Answer the user's question based on the content provided above, and your reply should be as concise and organized as possible.\r\n"; - prompt += $"Question: {retrievalModel.Question}\r\nAnswer: "; + prompt += $"\r\nQuestion: {retrievalModel.Question}\r\nAnswer: "; var completion = await GetTextCompletion().GetCompletion(prompt); return completion; diff --git a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs index 55b623c5..edf9b217 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs @@ -2,37 +2,16 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.MLTasks; using LLama; -using System.IO; +using LLama.Common; namespace BotSharp.Core.Plugins.LLamaSharp; public class ChatCompletionProvider : IChatCompletion { - private IChatModel _model; - - - public ChatCompletionProvider(LlamaAiModel model) + private readonly IServiceProvider _services; + public ChatCompletionProvider(IServiceProvider services) { - model.LoadModel(); - _model = model.Model; - // _model.InitChatPrompt(prompt, "UTF-8"); - // _model.InitChatAntiprompt(new string[] { "user:" }); - } - - public async Task GetChatCompletionsAsync(List conversations, - Func onChunkReceived) - { - string totalResponse = ""; - var content = string.Join("\n ", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim(); - content += "\n assistant: "; - foreach (var response in _model.Chat(content, "", "UTF-8")) - { - Console.Write(response); - totalResponse += response; - await onChunkReceived(response); - } - - Console.WriteLine(); + _services = services; } public Task GetChatCompletionsAsync(Agent agent, List conversations) @@ -40,12 +19,14 @@ public class ChatCompletionProvider : IChatCompletion string totalResponse = ""; var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim(); content += "\nassistant: "; - foreach (var response in _model.Chat(content, agent.Instruction, "UTF-8")) + + var llama = _services.GetRequiredService(); + llama.LoadModel(); + var executor = new StatelessExecutor(llama.Model); + var inferenceParams = new InferenceParams() { Temperature = 1.0f, AntiPrompts = new List { "user:" }, MaxTokens = 64 }; + + foreach (var response in executor.Infer(agent.Instruction, inferenceParams)) { - if (response == "\n") - { - break; - } Console.Write(response); totalResponse += response; } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LLamaSharpPlugin.cs b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LLamaSharpPlugin.cs index 7a2470d8..92db8244 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LLamaSharpPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LLamaSharpPlugin.cs @@ -13,6 +13,7 @@ public class LLamaSharpPlugin : IBotSharpPlugin services.AddSingleton(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LlamaAiModel.cs b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LlamaAiModel.cs index 3ee01589..a4f4336e 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LlamaAiModel.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LlamaAiModel.cs @@ -1,9 +1,12 @@ using LLama; +using LLama.Common; + namespace BotSharp.Core.Plugins.LLamaSharp; public class LlamaAiModel { private readonly LlamaSharpSettings _settings; + public LlamaSharpSettings Settings => _settings; LLamaModel _model; @@ -22,11 +25,8 @@ public class LlamaAiModel return; } - _model = new LLamaModel(new LLamaParams(model: _settings.ModelPath, - n_ctx: _settings.MaxContextLength, - interactive: _settings.Interactive, - repeat_penalty: _settings.RepeatPenalty, - verbose_prompt: _settings.VerbosePrompt, - n_gpu_layers: _settings.NumberOfGpuLayer)); + _model = new LLamaModel(new ModelParams(_settings.ModelPath, + contextSize: _settings.MaxContextLength, + gpuLayerCount: _settings.NumberOfGpuLayer)); } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextCompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextCompletionProvider.cs new file mode 100644 index 00000000..b7ac3cb7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextCompletionProvider.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.MLTasks; +using LLama; +using LLama.Common; + +namespace BotSharp.Core.Plugins.LLamaSharp; + +public class TextCompletionProvider : ITextCompletion +{ + private readonly IServiceProvider _services; + + public TextCompletionProvider(IServiceProvider services) + { + _services = services; + } + + public Task GetCompletion(string text) + { + var llama = _services.GetRequiredService(); + llama.LoadModel(); + + var executor = new InstructExecutor(llama.Model); + var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 }; + + string totalResponse = ""; + foreach (var response in executor.Infer(text, inferenceParams)) + { + Console.Write(response); + totalResponse += response; + } + + return Task.FromResult(totalResponse); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextEmbeddingProvider.cs b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextEmbeddingProvider.cs index fa6e956a..a061df47 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextEmbeddingProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextEmbeddingProvider.cs @@ -1,19 +1,25 @@ using BotSharp.Abstraction.MLTasks; +using LLama; +using LLama.Common; namespace BotSharp.Core.Plugins.LLamaSharp; public class TextEmbeddingProvider : ITextEmbedding { + private readonly IServiceProvider _services; public int Dimension => throw new NotImplementedException(); - private readonly LlamaAiModel _llama; - public TextEmbeddingProvider(LlamaAiModel llama) + public TextEmbeddingProvider(IServiceProvider services) { - _llama = llama; + _services = services; } public float[] GetVector(string text) { - return new float[0]; + var llama = _services.GetRequiredService(); + + var executor = new LLamaEmbedder(new ModelParams(llama.Settings.ModelPath)); + + return executor.GetEmbeddings(text); } } diff --git a/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs b/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs index b81742b2..0d8a8c75 100644 --- a/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs +++ b/src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs @@ -1,5 +1,5 @@ using BotSharp.Abstraction.VectorStorage; -using Tensorflow; +using System.IO; using Tensorflow.NumPy; namespace BotSharp.Core.Plugins.MemVecDb; @@ -20,7 +20,7 @@ public class MemVectorDatabase : IVectorDb return Task.FromResult(_collections.Select(x => x.Key).ToList()); } - public Task> Search(string collectionName, float[] vector, int limit = 10) + public Task> Search(string collectionName, float[] vector, int limit = 10) { var similarities = new float[_vectors[collectionName].Count]; for (int i = 0; i < _vectors[collectionName].Count; i++) @@ -28,20 +28,22 @@ public class MemVectorDatabase : IVectorDb similarities[i] = CalCosineSimilarity(vector, _vectors[collectionName][i].Vector); } - var indice = np.argsort(similarities).ToArray() + var texts = np.argsort(similarities).ToArray() .Reverse() .Take(limit) + .Select(i => _vectors[collectionName][i].Text) .ToList(); - return Task.FromResult(indice); + return Task.FromResult(texts); } - public Task Upsert(string collectionName, int id, float[] vector) + public Task Upsert(string collectionName, int id, float[] vector, string text) { _vectors[collectionName].Add(new VecRecord { Id = id, - Vector = vector + Vector = vector, + Text = text }); return Task.CompletedTask; diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 1a22e11c..3fac532d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -17,12 +17,12 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, int limit = 10) + public Task> Search(string collectionName, float[] vector, int limit = 10) { throw new NotImplementedException(); } - public Task Upsert(string collectionName, int id, float[] vector) + public Task Upsert(string collectionName, int id, float[] vector, string text) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 97a46e28..62feb4f2 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,9 +1,12 @@ +using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.VectorStorage; +using Microsoft.Extensions.DependencyInjection; using QdrantCSharp; using QdrantCSharp.Enums; using QdrantCSharp.Models; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -13,9 +16,13 @@ public class QdrantDb : IVectorDb { private readonly QdrantHttpClient _client; private readonly QdrantSetting _setting; - public QdrantDb(QdrantSetting setting) + private readonly IServiceProvider _services; + + public QdrantDb(QdrantSetting setting, + IServiceProvider services) { _setting = setting; + _services = services; _client = new QdrantHttpClient ( url: _setting.Url, @@ -37,6 +44,11 @@ public class QdrantDb : IVectorDb { // Create a new collection await _client.CreateCollection(collectionName, new VectorParams(size: dim, distance: Distance.COSINE)); + + var agentService = _services.GetRequiredService(); + var agentDataDir = agentService.GetAgentDataDir(collectionName); + var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt"); + File.WriteAllLines(knowledgePath, new string[0]); } // Get collection info @@ -47,18 +59,30 @@ public class QdrantDb : IVectorDb } } - public async Task Upsert(string collectionName, int id, float[] vector) + public async Task Upsert(string collectionName, int id, float[] vector, string text) { // Insert vectors await _client.Upsert(collectionName, points: new List { new PointStruct(id: id, vector: vector) }); + + // Store chunks in local file system + var agentService = _services.GetRequiredService(); + var agentDataDir = agentService.GetAgentDataDir(collectionName); + var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt"); + File.WriteAllLines(knowledgePath, new string[] { text }); } - public async Task> Search(string collectionName, float[] vector, int limit = 10) + public async Task> Search(string collectionName, float[] vector, int limit = 10) { var result = await _client.Search(collectionName, vector, limit); - return result.Result.Select(x => x.Id).ToList(); + + var agentService = _services.GetRequiredService(); + var agentDataDir = agentService.GetAgentDataDir(collectionName); + var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt"); + var texts = File.ReadAllLines(knowledgePath); + + return result.Result.Select(x => texts[x.Id]).ToList(); } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs index faa0e5e5..6d330259 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs @@ -13,6 +13,6 @@ public class QdrantPlugin : IBotSharpPlugin config.Bind("Qdrant", settings); services.AddSingleton(x => settings); - services.AddSingleton(); + services.AddScoped(); } } diff --git a/src/WebStarter/Program.cs b/src/WebStarter/Program.cs index 6ee5606e..695a6121 100644 --- a/src/WebStarter/Program.cs +++ b/src/WebStarter/Program.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Users; using BotSharp.Core; -using BotSharp.Core.Users.Services; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.IdentityModel.Tokens; using System.Text; diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index bc17abf3..3a23b97b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -23,7 +23,7 @@ "ModelPath": "C:\\Users\\haipi\\Downloads\\wizard-vicuna-13B.ggmlv3.q8_0.bin", "InstructionFile": "Prompts\\chat-with-bob.txt", "ChatSampleFile": "Prompts\\chat-samples.txt", - "MaxContextLength": 2048, + "MaxContextLength": 1024, "NumberOfGpuLayer": 10 }, @@ -72,6 +72,7 @@ "VectorDb": "MemVectorDatabase", "TextEmbedding": "fastTextEmbeddingProvider", "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider" + // "TextCompletion": "LLamaSharp.TextCompletionProvider" }, "PluginLoader": {