Enable LLamaSharp (WIP) #81

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -85,7 +85,7 @@ public class ConversationService : IConversationService
     public IChatCompletion GetChatCompletion()
     {
         var completions = _services.GetServices<IChatCompletion>();
-        return completions.FirstOrDefault(x => x.GetType().FullName.Contains(_settings.ChatCompletion));
+        return completions.FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.ChatCompletion));
     }
 
     public Task CleanHistory(string agentId)

src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs

@@ -93,7 +93,7 @@ public class KnowledgeService : IKnowledgeService
     public ITextCompletion GetTextCompletion()
     {
         var textCompletion = _services.GetServices<ITextCompletion>()
-            .FirstOrDefault(x => x.GetType().Name == _settings.TextCompletion);
+            .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextCompletion));
         return textCompletion;
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/ChatCompletionProvider.cs

@@ -8,34 +8,24 @@ namespace BotSharp.Core.Plugins.LLamaSharp;
 
 public class ChatCompletionProvider : IChatCompletion
 {
-    private readonly IChatModel _model;
-    private readonly LlamaSharpSettings _settings;
+    private IChatModel _model;
 
-    public ChatCompletionProvider(LlamaSharpSettings settings)
+
+    public ChatCompletionProvider(LlamaAiModel model)
     {
-        _settings = settings;
-        _model = new LLamaModel(new LLamaParams(model: _settings.ModelPath,
-            n_ctx: _settings.MaxContextLength,
-            interactive: _settings.Interactive,
-            repeat_penalty: _settings.RepeatPenalty,
-            verbose_prompt: _settings.VerbosePrompt,
-            n_gpu_layers: _settings.NumberOfGpuLayer));
-
-        var prompt = GetInstruction();
-        _model.InitChatPrompt(prompt, "UTF-8");
-        _model.InitChatAntiprompt(new string[] { "user:" });
+        model.LoadModel();
+        _model = model.Model;
+        // _model.InitChatPrompt(prompt, "UTF-8");
+        // _model.InitChatAntiprompt(new string[] { "user:" });
     }
 
-    public int Priority => 100;
-
     public async Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
         Func<string, Task> onChunkReceived)
     {
         string totalResponse = "";
-        var prompt = GetInstruction();
         var content = string.Join("\n ", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim();
         content += "\n assistant: ";
-        foreach (var response in _model.Chat(content, prompt, "UTF-8"))
+        foreach (var response in _model.Chat(content, "", "UTF-8"))
         {
             Console.Write(response);
             totalResponse += response;
@@ -48,54 +38,18 @@ public class ChatCompletionProvider : IChatCompletion
     public Task<string> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations)
     {
         string totalResponse = "";
-        var prompt = GetInstruction();
-        var content = string.Join("\n ", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim();
-        content += "\n assistant: ";
-        foreach (var response in _model.Chat(content, prompt, "UTF-8"))
+        var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Text.Replace("user:", "")}")).Trim();
+        content += "\nassistant: ";
+        foreach (var response in _model.Chat(content, agent.Instruction, "UTF-8"))
         {
+            if (response == "\n")
+            {
+                break;
+            }
             Console.Write(response);
             totalResponse += response;
         }
 
-        return Task.FromResult(totalResponse);
-    }
-
-    public List<RoleDialogModel> GetChatSamples()
-    {
-        var samples = new List<RoleDialogModel>();
-        if (!string.IsNullOrEmpty(_settings.ChatSampleFile))
-        {
-            var lines = File.ReadAllLines(_settings.ChatSampleFile);
-            for (int i = 0; i < lines.Length; i++)
-            {
-                var line = lines[i];
-                var role = line.Substring(0, line.IndexOf(' ') - 1);
-                var content = line.Substring(line.IndexOf(' ') + 1);
-
-                samples.Add(new RoleDialogModel
-                {
-                    Role = role,
-                    Text = content
-                });
-            }
-        }
-        return samples;
-    }
-
-    public string GetInstruction()
-    {
-        var instruction = "";
-        if (!string.IsNullOrEmpty(_settings.InstructionFile))
-        {
-            instruction = File.ReadAllText(_settings.InstructionFile);
-        }
-
-        instruction += "\n";
-        foreach (var message in GetChatSamples())
-        {
-            instruction += $"\n{message.Role}: {message.Text}";
-        }
-
-        return instruction;
+        return Task.FromResult(totalResponse.Trim());
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LLamaSharpPlugin.cs

@@ -11,6 +11,8 @@ public class LLamaSharpPlugin : IBotSharpPlugin
         config.Bind("LlamaSharp", llamaSharpSettings);
         services.AddSingleton(x => llamaSharpSettings);
 
+        services.AddSingleton<LlamaAiModel>();
+        services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IChatCompletion, ChatCompletionProvider>();
     }
 }

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/LlamaAiModel.cs

@@ -0,0 +1,32 @@
+using LLama;
+namespace BotSharp.Core.Plugins.LLamaSharp;
+
+public class LlamaAiModel
+{
+    private readonly LlamaSharpSettings _settings;
+
+    LLamaModel _model;
+
+    public LLamaModel Model => _model;
+
+    public LlamaAiModel(LlamaSharpSettings settings)
+    {
+        _settings = settings;
+    }
+
+
+    public void LoadModel()
+    {
+        if (_model != null)
+        {
+            return;
+        }
+
+        _model = new LLamaModel(new LLamaParams(model: _settings.ModelPath,
+            n_ctx: _settings.MaxContextLength,
+            interactive: _settings.Interactive,
+            repeat_penalty: _settings.RepeatPenalty,
+            verbose_prompt: _settings.VerbosePrompt,
+            n_gpu_layers: _settings.NumberOfGpuLayer));
+    }
+}

src/Infrastructure/BotSharp.Core/Plugins/LLamaSharp/TextEmbeddingProvider.cs

@@ -0,0 +1,19 @@
+using BotSharp.Abstraction.MLTasks;
+
+namespace BotSharp.Core.Plugins.LLamaSharp;
+
+public class TextEmbeddingProvider : ITextEmbedding
+{
+    public int Dimension => throw new NotImplementedException();
+    private readonly LlamaAiModel _llama;
+
+    public TextEmbeddingProvider(LlamaAiModel llama)
+    {
+        _llama = llama;
+    }
+
+    public float[] GetVector(string text)
+    {
+        return new float[0];
+    }
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -88,7 +88,7 @@ public class ChatCompletionProvider : IChatCompletion
             }
         }
 
-        return output;
+        return output.Trim();
     }
 
     private ChatCompletionsOptions PrepareOptions(Agent agent, List<RoleDialogModel> conversations)

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs

@@ -41,7 +41,7 @@ public class TextCompletionProvider : ITextCompletion
             completion += t.Text;
         };
 
-        return completion;
+        return completion.Trim();
     }
 
     private OpenAIClient GetOpenAIClient()

src/WebStarter/appsettings.json

@@ -15,6 +15,7 @@
 
   "Conversation": {
     "ChatCompletion": "AzureOpenAI.Providers.ChatCompletionProvider"
+    // "ChatCompletion": "LLamaSharp.ChatCompletionProvider"
   },
 
   "LlamaSharp": {
@@ -85,7 +86,7 @@
     "Plugins": [
       "KnowledgeBasePlugin",
       "MemVecDbPlugin",
-      // "LLamaSharpPlugin",
+      "LLamaSharpPlugin",
       "AzureOpenAiPlugin",
       "MetaAiPlugin",
       "QdrantPlugin",