Chang GetChatCompletions to async.

src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs

@@ -13,7 +13,7 @@ public interface IChatCompletion
     /// <param name="model"></param>
     void SetModelName(string model);
 
-    RoleDialogModel GetChatCompletions(Agent agent,
+    Task<RoleDialogModel> GetChatCompletions(Agent agent,
         List<RoleDialogModel> conversations);
 
     Task<bool> GetChatCompletionsAsync(Agent agent, 

src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs

@@ -71,7 +71,7 @@ public partial class InstructService : IInstructService
         }
         else if (completer is IChatCompletion chatCompleter)
         {
-            var result = chatCompleter.GetChatCompletions(new Agent
+            var result = await chatCompleter.GetChatCompletions(new Agent
             {
                 Id = agentId,
                 Name = agent.Name,

src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs

@@ -46,7 +46,7 @@ public class HFPlanner : IPlaner
                         MessageId = messageId
                     }
                 };
-                response = completion.GetChatCompletions(router, dialogs);
+                response = await completion.GetChatCompletions(router, dialogs);
 
                 inst = response.Content.JsonContent<FunctionCallFromLlm>();
                 break;

src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs

@@ -52,7 +52,7 @@ public class NaivePlanner : IPlaner
                         MessageId = messageId
                     }
                 };
-                var response = completion.GetChatCompletions(router, dialogs);
+                var response = await completion.GetChatCompletions(router, dialogs);
 
                 inst = response.Content.JsonContent<FunctionCallFromLlm>();
                 break;

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs

@@ -263,6 +263,7 @@ namespace BotSharp.Core.Repository
             agent.AllowRouting = inputAgent.AllowRouting;
             agent.Profiles = inputAgent.Profiles;
             agent.RoutingRules = inputAgent.RoutingRules;
+            agent.LlmConfig = inputAgent.LlmConfig;
             agent.UpdatedDateTime = DateTime.UtcNow;
             var json = JsonSerializer.Serialize(agent, _options);
             File.WriteAllText(agentFile, json);

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs

@@ -25,7 +25,7 @@ public partial class RoutingService
             agentConfig: agent.LlmConfig);
 
         var message = dialogs.Last();
-        var response = chatCompletion.GetChatCompletions(agent, dialogs);
+        var response = await chatCompletion.GetChatCompletions(agent, dialogs);
 
         if (response.Role == AgentRole.Function)
         {

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -60,12 +60,13 @@ public class InstructModeController : ControllerBase
             .SetState("model", input.Model);
 
         var textCompletion = CompletionProvider.GetChatCompletion(_services);
-        return textCompletion.GetChatCompletions(new Agent()
+        var message = await textCompletion.GetChatCompletions(new Agent()
         {
             Id = Guid.Empty.ToString(),
         }, new List<RoleDialogModel>
         {
             new RoleDialogModel(AgentRole.User, input.Text)
-        }).Content;
+        });
+        return message.Content;
     }
 }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs

@@ -50,8 +50,8 @@ public class AgentUpdateModel
     [JsonPropertyName("routing_rules")]
 
     public List<RoutingRuleUpdateModel>? RoutingRules { get; set; }
-    [JsonPropertyName("llm_config")]
 
+    [JsonPropertyName("llm_config")]
     public AgentLlmConfig? LlmConfig { get; set; }
 
     public Agent ToAgent()

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -35,14 +35,14 @@ public class ChatCompletionProvider : IChatCompletion
         _services = services;
     }
 
-    public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
+    public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
         var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
 
         // Before chat completion hook
         foreach (var hook in contentHooks)
         {
-            hook.BeforeGenerating(agent, conversations).Wait();
+            await hook.BeforeGenerating(agent, conversations);
         }
 
         var client = ProviderHelper.GetClient(_model, _services);
@@ -78,14 +78,14 @@ public class ChatCompletionProvider : IChatCompletion
         // After chat completion hook
         foreach(var hook in contentHooks)
         {
-            hook.AfterGenerated(responseMessage, new TokenStatsModel
+            await hook.AfterGenerated(responseMessage, new TokenStatsModel
             {
                 Prompt = prompt,
                 Provider = Provider,
                 Model = _model,
                 PromptCount = response.Value.Usage.PromptTokens,
                 CompletionCount = response.Value.Usage.CompletionTokens
-            }).Wait();
+            });
         }
 
         return responseMessage;

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs

@@ -27,7 +27,7 @@ public class ChatCompletionProvider : IChatCompletion
         _logger = logger;
     }
 
-    public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
+    public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
         var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
 
@@ -45,12 +45,12 @@ public class ChatCompletionProvider : IChatCompletion
         {
             // use text completion
             // var response = client.GenerateTextAsync(prompt, null).Result;
-            var response = client.ChatAsync(new PalmChatCompletionRequest
+            var response = await client.ChatAsync(new PalmChatCompletionRequest
             {
                 Context = prompt,
                 Messages = messages,
                 Temperature = 0.1f
-            }).Result;
+            });
 
             var message = response.Candidates.First();
 
@@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion
         }
         else
         {
-            var response = client.ChatAsync(messages, context: prompt, examples: null, options: null).Result;
+            var response = await client.ChatAsync(messages, context: prompt, examples: null, options: null);
 
             var message = response.Candidates.First();
 

src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs

@@ -88,7 +88,7 @@ public class ChatCompletionProvider : IChatCompletion
         _model = model;
     }
 
-    public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
+    public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
         var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
 

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs

@@ -21,13 +21,16 @@ public class ChatCompletionProvider : IChatCompletion
 
     public string Provider => "llama-sharp";
 
-    public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
+    public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
         var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
 
         // Before chat completion hook
-        Task.WaitAll(hooks.Select(hook =>
-            hook.BeforeGenerating(agent, conversations)).ToArray());
+        // Before chat completion hook
+        foreach (var hook in hooks)
+        {
+            await hook.BeforeGenerating(agent, conversations);
+        }
 
         var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
         content += $"\r\n{AgentRole.Assistant}: ";
@@ -40,7 +43,7 @@ public class ChatCompletionProvider : IChatCompletion
         {
             Temperature = 0.1f,
             AntiPrompts = new List<string> { $"{AgentRole.User}:", "[/INST]" },
-            MaxTokens = 64
+            MaxTokens = 128
         };
 
         string totalResponse = "";
@@ -49,16 +52,10 @@ public class ChatCompletionProvider : IChatCompletion
         var instruction = agentService.RenderedInstruction(agent);
         var prompt = instruction + "\r\n" + content;
 
-        var convSetting = _services.GetRequiredService<ConversationSetting>();
-        if (convSetting.ShowVerboseLog)
+        await foreach(var text in Spinner(executor.InferAsync(prompt, inferenceParams)))
         {
-            _logger.LogInformation(prompt);
-        }
-
-        foreach (var response in executor.InferAsync(prompt, inferenceParams).GetAsyncEnumerator().Current)
-        {
-            Console.Write(response);
-            totalResponse += response;
+            Console.Write(text);
+            totalResponse += text;
         }
 
         foreach (var anti in inferenceParams.AntiPrompts)
@@ -72,15 +69,40 @@ public class ChatCompletionProvider : IChatCompletion
         };
 
         // After chat completion hook
-        Task.WaitAll(hooks.Select(hook =>
-            hook.AfterGenerated(msg, new TokenStatsModel
+        foreach (var hook in hooks)
+        {
+            await hook.AfterGenerated(msg, new TokenStatsModel
             {
+                Prompt = prompt,
+                Provider = Provider,
                 Model = _model
-            })).ToArray());
+            });
+        }
 
         return msg;
     }
 
+    public async IAsyncEnumerable<string> Spinner(IAsyncEnumerable<string> source)
+    {
+        var enumerator = source.GetAsyncEnumerator();
+
+        var characters = new[] { '|', '/', '-', '\\' };
+
+        while (true)
+        {
+            var next = enumerator.MoveNextAsync();
+
+            while (!next.IsCompleted)
+            {
+                await Task.Delay(75);
+            }
+
+            if (!next.Result)
+                break;
+            yield return enumerator.Current;
+        }
+    }
+
     public async Task<bool> GetChatCompletionsAsync(Agent agent,
         List<RoleDialogModel> conversations,
         Func<RoleDialogModel, Task> onMessageReceived,

src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs

@@ -42,7 +42,7 @@ namespace BotSharp.Plugin.SemanticKernel
             this._tokenStatistics = tokenStatistics;
         }
         /// <inheritdoc/>
-        public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
+        public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
         {
             var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
 
@@ -69,14 +69,13 @@ namespace BotSharp.Plugin.SemanticKernel
                 }
             }
 
-            var response = completion.GetChatCompletionsAsync(chatHistory)
+            var response = await completion.GetChatCompletionsAsync(chatHistory)
                 .ContinueWith(async t =>
                 {
                     var result = await t;
                     var message = await result.First().GetChatMessageAsync();
                     return message.Content;
-                }).ConfigureAwait(false).GetAwaiter().GetResult()
-                .ConfigureAwait(false).GetAwaiter().GetResult();
+                }).ConfigureAwait(false).GetAwaiter().GetResult();
 
             var msg = new RoleDialogModel(AgentRole.Assistant, response)
             {

src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs

@@ -35,7 +35,7 @@ public partial class WebDriverService
                     MessageId = messageId
                 }
             };
-            var result = chatCompleter.GetChatCompletions(new Agent
+            var result = await chatCompleter.GetChatCompletions(new Agent
             {
                 Id = agent.Id,
                 Name = agent.Name,

src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs

@@ -35,7 +35,7 @@ public partial class WebDriverService
                     MessageId = messageId
                 }
             };
-            var result = chatCompleter.GetChatCompletions(new Agent
+            var result = await chatCompleter.GetChatCompletions(new Agent
             {
                 Id = agent.Id,
                 Name = agent.Name,

src/WebStarter/appsettings.json

@@ -34,6 +34,15 @@
           "CompletionCost": 0.002
         }
       ]
+    },
+    {
+      "Provider": "llama-sharp",
+      "Models": [
+        {
+          "Name": "llama-2-7b-guanaco-qlora.Q2_K.gguf",
+          "Type": "chat"
+        }
+      ]
     }
   ],
 
@@ -70,7 +79,7 @@
     "ModelDir": "C:/Users/haipi/Downloads",
     "DefaultModel": "llama-2-7b-chat.Q8_0.gguf",
     "MaxContextLength": 1024,
-    "NumberOfGpuLayer": 10
+    "NumberOfGpuLayer": 20
   },
 
   "AzureOpenAi": {
@@ -159,7 +168,8 @@
       "BotSharp.Plugin.ChatHub",
       "BotSharp.Plugin.WeChat",
       "BotSharp.Plugin.PizzaBot",
-      "BotSharp.Plugin.WebDriver"
+      "BotSharp.Plugin.WebDriver",
+      "BotSharp.Plugin.LLamaSharp"
     ]
   }
 }