diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 9d8c5b51..d145d175 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -13,7 +13,7 @@ public interface IChatCompletion /// void SetModelName(string model); - RoleDialogModel GetChatCompletions(Agent agent, + Task GetChatCompletions(Agent agent, List conversations); Task GetChatCompletionsAsync(Agent agent, diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs index 05e54f28..ccab8874 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs @@ -71,7 +71,7 @@ public partial class InstructService : IInstructService } else if (completer is IChatCompletion chatCompleter) { - var result = chatCompleter.GetChatCompletions(new Agent + var result = await chatCompleter.GetChatCompletions(new Agent { Id = agentId, Name = agent.Name, diff --git a/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs index a6c91e76..206c22d5 100644 --- a/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs @@ -46,7 +46,7 @@ public class HFPlanner : IPlaner MessageId = messageId } }; - response = completion.GetChatCompletions(router, dialogs); + response = await completion.GetChatCompletions(router, dialogs); inst = response.Content.JsonContent(); break; diff --git a/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs index 81738225..9ad95121 100644 --- a/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs @@ -52,7 +52,7 @@ public class NaivePlanner : IPlaner MessageId = messageId } }; - var response = completion.GetChatCompletions(router, dialogs); + var response = await completion.GetChatCompletions(router, dialogs); inst = response.Content.JsonContent(); break; diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index 8d3302ec..c72711ff 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -263,6 +263,7 @@ namespace BotSharp.Core.Repository agent.AllowRouting = inputAgent.AllowRouting; agent.Profiles = inputAgent.Profiles; agent.RoutingRules = inputAgent.RoutingRules; + agent.LlmConfig = inputAgent.LlmConfig; agent.UpdatedDateTime = DateTime.UtcNow; var json = JsonSerializer.Serialize(agent, _options); File.WriteAllText(agentFile, json); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index b5140e20..78b339fa 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -25,7 +25,7 @@ public partial class RoutingService agentConfig: agent.LlmConfig); var message = dialogs.Last(); - var response = chatCompletion.GetChatCompletions(agent, dialogs); + var response = await chatCompletion.GetChatCompletions(agent, dialogs); if (response.Role == AgentRole.Function) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 750719b8..c8611003 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -60,12 +60,13 @@ public class InstructModeController : ControllerBase .SetState("model", input.Model); var textCompletion = CompletionProvider.GetChatCompletion(_services); - return textCompletion.GetChatCompletions(new Agent() + var message = await textCompletion.GetChatCompletions(new Agent() { Id = Guid.Empty.ToString(), }, new List { new RoleDialogModel(AgentRole.User, input.Text) - }).Content; + }); + return message.Content; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs index 74589b2b..56b6d968 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs @@ -50,8 +50,8 @@ public class AgentUpdateModel [JsonPropertyName("routing_rules")] public List? RoutingRules { get; set; } - [JsonPropertyName("llm_config")] + [JsonPropertyName("llm_config")] public AgentLlmConfig? LlmConfig { get; set; } public Agent ToAgent() diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 49185d15..34cfe762 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -35,14 +35,14 @@ public class ChatCompletionProvider : IChatCompletion _services = services; } - public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + public async Task GetChatCompletions(Agent agent, List conversations) { var contentHooks = _services.GetServices().ToList(); // Before chat completion hook foreach (var hook in contentHooks) { - hook.BeforeGenerating(agent, conversations).Wait(); + await hook.BeforeGenerating(agent, conversations); } var client = ProviderHelper.GetClient(_model, _services); @@ -78,14 +78,14 @@ public class ChatCompletionProvider : IChatCompletion // After chat completion hook foreach(var hook in contentHooks) { - hook.AfterGenerated(responseMessage, new TokenStatsModel + await hook.AfterGenerated(responseMessage, new TokenStatsModel { Prompt = prompt, Provider = Provider, Model = _model, PromptCount = response.Value.Usage.PromptTokens, CompletionCount = response.Value.Usage.CompletionTokens - }).Wait(); + }); } return responseMessage; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index 9e98e9b4..d278b110 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -27,7 +27,7 @@ public class ChatCompletionProvider : IChatCompletion _logger = logger; } - public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + public async Task GetChatCompletions(Agent agent, List conversations) { var hooks = _services.GetServices().ToList(); @@ -45,12 +45,12 @@ public class ChatCompletionProvider : IChatCompletion { // use text completion // var response = client.GenerateTextAsync(prompt, null).Result; - var response = client.ChatAsync(new PalmChatCompletionRequest + var response = await client.ChatAsync(new PalmChatCompletionRequest { Context = prompt, Messages = messages, Temperature = 0.1f - }).Result; + }); var message = response.Candidates.First(); @@ -66,7 +66,7 @@ public class ChatCompletionProvider : IChatCompletion } else { - var response = client.ChatAsync(messages, context: prompt, examples: null, options: null).Result; + var response = await client.ChatAsync(messages, context: prompt, examples: null, options: null); var message = response.Candidates.First(); diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index f620ffb8..2899a97b 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -88,7 +88,7 @@ public class ChatCompletionProvider : IChatCompletion _model = model; } - public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + public async Task GetChatCompletions(Agent agent, List conversations) { var hooks = _services.GetServices().ToList(); diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 09aac6ee..0db444ce 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -21,13 +21,16 @@ public class ChatCompletionProvider : IChatCompletion public string Provider => "llama-sharp"; - public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + public async Task GetChatCompletions(Agent agent, List conversations) { var hooks = _services.GetServices().ToList(); // Before chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.BeforeGenerating(agent, conversations)).ToArray()); + // Before chat completion hook + foreach (var hook in hooks) + { + await hook.BeforeGenerating(agent, conversations); + } var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; @@ -40,7 +43,7 @@ public class ChatCompletionProvider : IChatCompletion { Temperature = 0.1f, AntiPrompts = new List { $"{AgentRole.User}:", "[/INST]" }, - MaxTokens = 64 + MaxTokens = 128 }; string totalResponse = ""; @@ -49,16 +52,10 @@ public class ChatCompletionProvider : IChatCompletion var instruction = agentService.RenderedInstruction(agent); var prompt = instruction + "\r\n" + content; - var convSetting = _services.GetRequiredService(); - if (convSetting.ShowVerboseLog) + await foreach(var text in Spinner(executor.InferAsync(prompt, inferenceParams))) { - _logger.LogInformation(prompt); - } - - foreach (var response in executor.InferAsync(prompt, inferenceParams).GetAsyncEnumerator().Current) - { - Console.Write(response); - totalResponse += response; + Console.Write(text); + totalResponse += text; } foreach (var anti in inferenceParams.AntiPrompts) @@ -72,15 +69,40 @@ public class ChatCompletionProvider : IChatCompletion }; // After chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.AfterGenerated(msg, new TokenStatsModel + foreach (var hook in hooks) + { + await hook.AfterGenerated(msg, new TokenStatsModel { + Prompt = prompt, + Provider = Provider, Model = _model - })).ToArray()); + }); + } return msg; } + public async IAsyncEnumerable Spinner(IAsyncEnumerable source) + { + var enumerator = source.GetAsyncEnumerator(); + + var characters = new[] { '|', '/', '-', '\\' }; + + while (true) + { + var next = enumerator.MoveNextAsync(); + + while (!next.IsCompleted) + { + await Task.Delay(75); + } + + if (!next.Result) + break; + yield return enumerator.Current; + } + } + public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index 673e9e5d..bf7fbc42 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -42,7 +42,7 @@ namespace BotSharp.Plugin.SemanticKernel this._tokenStatistics = tokenStatistics; } /// - public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + public async Task GetChatCompletions(Agent agent, List conversations) { var hooks = _services.GetServices().ToList(); @@ -69,14 +69,13 @@ namespace BotSharp.Plugin.SemanticKernel } } - var response = completion.GetChatCompletionsAsync(chatHistory) + var response = await completion.GetChatCompletionsAsync(chatHistory) .ContinueWith(async t => { var result = await t; var message = await result.First().GetChatMessageAsync(); return message.Content; - }).ConfigureAwait(false).GetAwaiter().GetResult() - .ConfigureAwait(false).GetAwaiter().GetResult(); + }).ConfigureAwait(false).GetAwaiter().GetResult(); var msg = new RoleDialogModel(AgentRole.Assistant, response) { diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs index 2544b265..506def97 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.ExtraData.cs @@ -35,7 +35,7 @@ public partial class WebDriverService MessageId = messageId } }; - var result = chatCompleter.GetChatCompletions(new Agent + var result = await chatCompleter.GetChatCompletions(new Agent { Id = agent.Id, Name = agent.Name, diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs index b0718b5f..690e55cb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs @@ -35,7 +35,7 @@ public partial class WebDriverService MessageId = messageId } }; - var result = chatCompleter.GetChatCompletions(new Agent + var result = await chatCompleter.GetChatCompletions(new Agent { Id = agent.Id, Name = agent.Name, diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c500511a..dd8ae953 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -34,6 +34,15 @@ "CompletionCost": 0.002 } ] + }, + { + "Provider": "llama-sharp", + "Models": [ + { + "Name": "llama-2-7b-guanaco-qlora.Q2_K.gguf", + "Type": "chat" + } + ] } ], @@ -70,7 +79,7 @@ "ModelDir": "C:/Users/haipi/Downloads", "DefaultModel": "llama-2-7b-chat.Q8_0.gguf", "MaxContextLength": 1024, - "NumberOfGpuLayer": 10 + "NumberOfGpuLayer": 20 }, "AzureOpenAi": { @@ -159,7 +168,8 @@ "BotSharp.Plugin.ChatHub", "BotSharp.Plugin.WeChat", "BotSharp.Plugin.PizzaBot", - "BotSharp.Plugin.WebDriver" + "BotSharp.Plugin.WebDriver", + "BotSharp.Plugin.LLamaSharp" ] } }