#81

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -4,6 +4,6 @@ public interface IVectorDb
 {
     Task<List<string>> GetCollections();
     Task CreateCollection(string collectionName, int dim);
-    Task Upsert(string collectionName, int id, float[] vector);
-    Task<List<int>> Search(string collectionName, float[] vector, int limit = 10);
+    Task Upsert(string collectionName, int id, float[] vector, string text);
+    Task<List<string>> Search(string collectionName, float[] vector, int limit = 10);
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -64,7 +64,7 @@
   <ItemGroup>
     <PackageReference Include="Colorful.Console" Version="1.2.15" />
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="6.2.1" />
-    <PackageReference Include="LLamaSharp" Version="0.3.0" />
+    <PackageReference Include="LLamaSharp" Version="0.4.0" />
     <PackageReference Include="LLamaSharp.Backend.Cuda11" Version="0.3.0" />
     <PackageReference Include="PdfPig" Version="0.1.8" />
     <PackageReference Include="TensorFlow.Keras" Version="0.11.0" />

src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Knowledges.Models;
-using System.IO;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.VectorStorage;
 
@@ -9,17 +8,14 @@ public class KnowledgeService : IKnowledgeService
 {
     private readonly IServiceProvider _services;
     private readonly KnowledgeBaseSettings _settings;
-    private readonly IAgentService _agentService;
     private readonly ITextChopper _textChopper;
 
     public KnowledgeService(IServiceProvider services,
         KnowledgeBaseSettings settings,
-        IAgentService agentService,
         ITextChopper textChopper)
     {
         _services = services;
         _settings = settings;
-        _agentService = agentService;
         _textChopper = textChopper;
     }
 
@@ -32,11 +28,6 @@ public class KnowledgeService : IKnowledgeService
             Conjunction = 32
         });
 
-        // Store chunks in local file system
-        var agentDataDir = _agentService.GetAgentDataDir(knowledge.AgentId);
-        var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt");
-        File.WriteAllLines(knowledgePath, lines);
-
         var db = GetVectorDb();
         var textEmbedding = GetTextEmbedding();
 
@@ -44,7 +35,7 @@ public class KnowledgeService : IKnowledgeService
         foreach (var line in lines)
         {
             var vec = textEmbedding.GetVector(line);
-            await db.Upsert(knowledge.AgentId, idStart, vec);
+            await db.Upsert(knowledge.AgentId, idStart, vec, line);
             idStart++;
         }
     }
@@ -54,23 +45,19 @@ public class KnowledgeService : IKnowledgeService
         var textEmbedding = GetTextEmbedding();
         var vector = textEmbedding.GetVector(retrievalModel.Question);
 
-        // Scan local knowledge directory
-        var agentDataDir = _agentService.GetAgentDataDir(retrievalModel.AgentId);
-        var chunks = File.ReadAllLines(Path.Combine(agentDataDir, "knowledge.txt"));
-
         // Vector search
         var result = await GetVectorDb().Search(retrievalModel.AgentId, vector);
 
         // Restore 
         var prompt = "";
-        foreach (var r in result)
+        foreach (var knowledge in result)
         {
-            prompt += chunks[r] + "\n";
+            prompt += knowledge + "\n";
         }
 
         prompt += "\r\n###\r\n";
         prompt += "Answer the user's question based on the content provided above, and your reply should be as concise and organized as possible.\r\n";
-        prompt += $"Question: {retrievalModel.Question}\r\nAnswer: ";
+        prompt += $"\r\nQuestion: {retrievalModel.Question}\r\nAnswer: ";
 
         var completion = await GetTextCompletion().GetCompletion(prompt);
         return completion;

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs

@@ -2,37 +2,16 @@ using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.MLTasks;
 using LLama;
-using System.IO;
+using LLama.Common;
 
 namespace BotSharp.Core.Plugins.LLamaSharp;
 
 public class ChatCompletionProvider : IChatCompletion
 {
-    private IChatModel _model;
-
-
-    public ChatCompletionProvider(LlamaAiModel model)
+    private readonly IServiceProvider _services;
+    public ChatCompletionProvider(IServiceProvider services)
     {
-        model.LoadModel();
-        _model = model.Model;
-        // _model.InitChatPrompt(prompt, "UTF-8");
-        // _model.InitChatAntiprompt(new string[] { "user:" });
-    }
-
-    public async Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
-        Func<string, Task> onChunkReceived)
-    {
-        string totalResponse = "";
-        var content = string.Join("\n ", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim();
-        content += "\n assistant: ";
-        foreach (var response in _model.Chat(content, "", "UTF-8"))
-        {
-            Console.Write(response);
-            totalResponse += response;
-            await onChunkReceived(response);
-        }
-
-        Console.WriteLine();
+        _services = services;
     }
 
     public Task<string> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations)
@@ -40,12 +19,14 @@ public class ChatCompletionProvider : IChatCompletion
         string totalResponse = "";
         var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim();
         content += "\nassistant: ";
-        foreach (var response in _model.Chat(content, agent.Instruction, "UTF-8"))
+
+        var llama = _services.GetRequiredService<LlamaAiModel>();
+        llama.LoadModel();
+        var executor = new StatelessExecutor(llama.Model);
+        var inferenceParams = new InferenceParams() { Temperature = 1.0f, AntiPrompts = new List<string> { "user:" }, MaxTokens = 64 };
+
+        foreach (var response in executor.Infer(agent.Instruction, inferenceParams))
         {
-            if (response == "\n")
-            {
-                break;
-            }
             Console.Write(response);
             totalResponse += response;
         }

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LLamaSharpPlugin.cs

@@ -13,6 +13,7 @@ public class LLamaSharpPlugin : IBotSharpPlugin
 
         services.AddSingleton<LlamaAiModel>();
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
+        services.AddScoped<ITextCompletion, TextCompletionProvider>();
         services.AddScoped<IChatCompletion, ChatCompletionProvider>();
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LlamaAiModel.cs

@@ -1,9 +1,12 @@
 using LLama;
+using LLama.Common;
+
 namespace BotSharp.Core.Plugins.LLamaSharp;
 
 public class LlamaAiModel
 {
     private readonly LlamaSharpSettings _settings;
+    public LlamaSharpSettings Settings => _settings;
 
     LLamaModel _model;
 
@@ -22,11 +25,8 @@ public class LlamaAiModel
             return;
         }
 
-        _model = new LLamaModel(new LLamaParams(model: _settings.ModelPath,
-            n_ctx: _settings.MaxContextLength,
-            interactive: _settings.Interactive,
-            repeat_penalty: _settings.RepeatPenalty,
-            verbose_prompt: _settings.VerbosePrompt,
-            n_gpu_layers: _settings.NumberOfGpuLayer));
+        _model = new LLamaModel(new ModelParams(_settings.ModelPath,
+            contextSize: _settings.MaxContextLength,
+            gpuLayerCount: _settings.NumberOfGpuLayer));
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextCompletionProvider.cs

@@ -0,0 +1,33 @@
+using BotSharp.Abstraction.MLTasks;
+using LLama;
+using LLama.Common;
+
+namespace BotSharp.Core.Plugins.LLamaSharp;
+
+public class TextCompletionProvider : ITextCompletion
+{
+    private readonly IServiceProvider _services;
+
+    public TextCompletionProvider(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public Task<string> GetCompletion(string text)
+    {
+        var llama = _services.GetRequiredService<LlamaAiModel>();
+        llama.LoadModel();
+
+        var executor = new InstructExecutor(llama.Model);
+        var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 };
+
+        string totalResponse = "";
+        foreach (var response in executor.Infer(text, inferenceParams))
+        {
+            Console.Write(response);
+            totalResponse += response;
+        }
+
+        return Task.FromResult(totalResponse);
+    }
+}

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextEmbeddingProvider.cs

@@ -1,19 +1,25 @@
 using BotSharp.Abstraction.MLTasks;
+using LLama;
+using LLama.Common;
 
 namespace BotSharp.Core.Plugins.LLamaSharp;
 
 public class TextEmbeddingProvider : ITextEmbedding
 {
+    private readonly IServiceProvider _services;
     public int Dimension => throw new NotImplementedException();
-    private readonly LlamaAiModel _llama;
 
-    public TextEmbeddingProvider(LlamaAiModel llama)
+    public TextEmbeddingProvider(IServiceProvider services)
     {
-        _llama = llama;
+        _services = services;
     }
 
     public float[] GetVector(string text)
     {
-        return new float[0];
+        var llama = _services.GetRequiredService<LlamaAiModel>();
+
+        var executor = new LLamaEmbedder(new ModelParams(llama.Settings.ModelPath));
+
+        return executor.GetEmbeddings(text);
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/MemVecDb/MemVectorDatabase.cs

@@ -1,5 +1,5 @@
 using BotSharp.Abstraction.VectorStorage;
-using Tensorflow;
+using System.IO;
 using Tensorflow.NumPy;
 
 namespace BotSharp.Core.Plugins.MemVecDb;
@@ -20,7 +20,7 @@ public class MemVectorDatabase : IVectorDb
         return Task.FromResult(_collections.Select(x => x.Key).ToList());
     }
 
-    public Task<List<int>> Search(string collectionName, float[] vector, int limit = 10)
+    public Task<List<string>> Search(string collectionName, float[] vector, int limit = 10)
     {
         var similarities = new float[_vectors[collectionName].Count];
         for (int i = 0; i < _vectors[collectionName].Count; i++)
@@ -28,20 +28,22 @@ public class MemVectorDatabase : IVectorDb
             similarities[i] = CalCosineSimilarity(vector, _vectors[collectionName][i].Vector);
         }
 
-        var indice = np.argsort(similarities).ToArray<int>()
+        var texts = np.argsort(similarities).ToArray<int>()
             .Reverse()
             .Take(limit)
+            .Select(i => _vectors[collectionName][i].Text)
             .ToList();
 
-        return Task.FromResult(indice);
+        return Task.FromResult(texts);
     }
 
-    public Task Upsert(string collectionName, int id, float[] vector)
+    public Task Upsert(string collectionName, int id, float[] vector, string text)
     {
         _vectors[collectionName].Add(new VecRecord
         {
             Id = id,
-            Vector = vector
+            Vector = vector,
+            Text = text
         });
 
         return Task.CompletedTask;

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -17,12 +17,12 @@ public class FaissDb : IVectorDb
         throw new NotImplementedException();
     }
 
-    public Task<List<int>> Search(string collectionName, float[] vector, int limit = 10)
+    public Task<List<string>> Search(string collectionName, float[] vector, int limit = 10)
     {
         throw new NotImplementedException();
     }
 
-    public Task Upsert(string collectionName, int id, float[] vector)
+    public Task Upsert(string collectionName, int id, float[] vector, string text)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -1,9 +1,12 @@
+using BotSharp.Abstraction.Agents;
 using BotSharp.Abstraction.VectorStorage;
+using Microsoft.Extensions.DependencyInjection;
 using QdrantCSharp;
 using QdrantCSharp.Enums;
 using QdrantCSharp.Models;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 
@@ -13,9 +16,13 @@ public class QdrantDb : IVectorDb
 {
     private readonly QdrantHttpClient _client;
     private readonly QdrantSetting _setting;
-    public QdrantDb(QdrantSetting setting)
+    private readonly IServiceProvider _services;
+
+    public QdrantDb(QdrantSetting setting,
+        IServiceProvider services)
     {
         _setting = setting;
+        _services = services;
         _client = new QdrantHttpClient
         (
             url: _setting.Url,
@@ -37,6 +44,11 @@ public class QdrantDb : IVectorDb
         {
             // Create a new collection
             await _client.CreateCollection(collectionName, new VectorParams(size: dim, distance: Distance.COSINE));
+
+            var agentService = _services.GetRequiredService<IAgentService>();
+            var agentDataDir = agentService.GetAgentDataDir(collectionName);
+            var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt");
+            File.WriteAllLines(knowledgePath, new string[0]);
         }
 
         // Get collection info
@@ -47,18 +59,30 @@ public class QdrantDb : IVectorDb
         }
     }
 
-    public async Task Upsert(string collectionName, int id, float[] vector)
+    public async Task Upsert(string collectionName, int id, float[] vector, string text)
     {
         // Insert vectors
         await _client.Upsert(collectionName, points: new List<PointStruct>
         {
             new PointStruct(id: id, vector: vector)
         });
+
+        // Store chunks in local file system
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agentDataDir = agentService.GetAgentDataDir(collectionName);
+        var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt");
+        File.WriteAllLines(knowledgePath, new string[] { text });
     }
 
-    public async Task<List<int>> Search(string collectionName, float[] vector, int limit = 10)
+    public async Task<List<string>> Search(string collectionName, float[] vector, int limit = 10)
     {
         var result = await _client.Search(collectionName, vector, limit);
-        return result.Result.Select(x => x.Id).ToList();
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agentDataDir = agentService.GetAgentDataDir(collectionName);
+        var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt");
+        var texts = File.ReadAllLines(knowledgePath);
+
+        return result.Result.Select(x => texts[x.Id]).ToList();
     }
 }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs

@@ -13,6 +13,6 @@ public class QdrantPlugin : IBotSharpPlugin
         config.Bind("Qdrant", settings);
         services.AddSingleton(x => settings);
         
-        services.AddSingleton<IVectorDb, QdrantDb>();
+        services.AddScoped<IVectorDb, QdrantDb>();
     }
 }

src/WebStarter/Program.cs

@@ -1,6 +1,4 @@
-using BotSharp.Abstraction.Users;
 using BotSharp.Core;
-using BotSharp.Core.Users.Services;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.IdentityModel.Tokens;
 using System.Text;

src/WebStarter/appsettings.json

@@ -23,7 +23,7 @@
     "ModelPath": "C:\\Users\\haipi\\Downloads\\wizard-vicuna-13B.ggmlv3.q8_0.bin",
     "InstructionFile": "Prompts\\chat-with-bob.txt",
     "ChatSampleFile": "Prompts\\chat-samples.txt",
-    "MaxContextLength": 2048,
+    "MaxContextLength": 1024,
     "NumberOfGpuLayer": 10
   },
 
@@ -72,6 +72,7 @@
     "VectorDb": "MemVectorDatabase",
     "TextEmbedding": "fastTextEmbeddingProvider",
     "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider"
+    // "TextCompletion": "LLamaSharp.TextCompletionProvider"
   },
 
   "PluginLoader": {