From fbd3c6f0ca56201f3d27853dd57fe94de5ab17dc Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 11 Oct 2023 19:59:09 -0500 Subject: [PATCH] Fix #167 --- .../Routing/Models/RoutingArgs.cs | 6 ++- .../RoutingService.GetNextInstruction.cs | 46 +++++++++++++++++-- .../BotSharp.Core/Routing/RoutingService.cs | 1 - .../Providers/ChatCompletionProvider.cs | 12 +++-- .../Providers/TextCompletionProvider.cs | 16 +++++-- 5 files changed, 63 insertions(+), 18 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index d3c1bb65..1c9bf32e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -9,10 +9,12 @@ public class RoutingArgs public string Reason { get; set; } = string.Empty; [JsonPropertyName("answer")] - public string Answer { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Answer { get; set; } [JsonPropertyName("agent")] - public string AgentName { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AgentName { get; set; } public override string ToString() { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs index 28d42523..84fe62b0 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs @@ -9,13 +9,23 @@ public partial class RoutingService { public async Task GetNextInstruction(string prompt) { - var responseFormat = _settings.EnableReasoning ? + var responseRouteToAgent = _settings.EnableReasoning ? JsonSerializer.Serialize(new FunctionCallFromLlm()) : JsonSerializer.Serialize(new RoutingArgs { - Function = "route_to_agent" + Function = "route_to_agent", + AgentName = "" }); - var content = $"{prompt} Response must be in JSON format {responseFormat}"; + + var responseResponseToUser = _settings.EnableReasoning ? + JsonSerializer.Serialize(new FunctionCallFromLlm()) : + JsonSerializer.Serialize(new RoutingArgs + { + Function = "response_to_user", + Answer = "" + }); + + var content = $"{prompt}\r\nIf need to route to specific agent, output in JSON {responseRouteToAgent}.\r\nIf you can handle the request directly, output in JSON {responseResponseToUser}."; var state = _services.GetRequiredService(); @@ -29,8 +39,34 @@ public partial class RoutingService model: _settings.Model); content = _routerInstance.Router.Instruction + "\r\n\r\n" + content + "\r\nResponse: "; - var text = await completion.GetCompletion(content); - response = new RoleDialogModel(AgentRole.Assistant, text); + + int retryCount = 0; + + while (retryCount < 3) + { + try + { + var text = await completion.GetCompletion(content); + response = new RoleDialogModel(AgentRole.Assistant, text); + + var pattern = @"\{(?:[^{}]|(?\{)|(?<-open>\}))+(?(open)(?!))\}"; + response.Content = Regex.Match(response.Content, pattern).Value; + args = JsonSerializer.Deserialize(response.Content); + break; + } + catch (Exception ex) + { + _logger.LogError($"{ex.Message}: {response.Content}"); + args.Function = "response_to_user"; + args.Answer = ex.Message; + args.AgentName = _settings.RouterName; + content += "\r\nPlease response in JSON format."; + } + finally + { + retryCount++; + } + } } else { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index ca4f11ea..8be7bfe0 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -77,7 +77,6 @@ public partial class RoutingService : IRoutingService loopCount++; var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request based on the CONVERSATION?"; - prompt += " Or you can handle without asking specific agent."; var inst = await GetNextInstruction(prompt); inst.Question = inst.Question ?? message; diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index f0034a6f..a517a63e 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -23,15 +23,18 @@ public class ChatCompletionProvider : IChatCompletion private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly LlamaSharpSettings _settings; + private readonly ITokenStatistics _tokenStatistics; private string _model; public ChatCompletionProvider(IServiceProvider services, ILogger logger, - LlamaSharpSettings settings) + LlamaSharpSettings settings, + ITokenStatistics tokenStatistics) { _services = services; _logger = logger; _settings = settings; + _tokenStatistics = tokenStatistics; } public string Provider => "llama-sharp"; @@ -41,11 +44,8 @@ public class ChatCompletionProvider : IChatCompletion var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; - var state = _services.GetRequiredService(); - var model = state.GetState("model", _settings.DefaultModel); - var llama = _services.GetRequiredService(); - llama.LoadModel(model); + llama.LoadModel(_model); var executor = llama.GetStatelessExecutor(); var inferenceParams = new InferenceParams() @@ -65,11 +65,13 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(prompt); } + _tokenStatistics.StartTimer(); foreach (var response in executor.Infer(prompt, inferenceParams)) { Console.Write(response); totalResponse += response; } + _tokenStatistics.StopTimer(); foreach (var anti in inferenceParams.AntiPrompts) { diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs index f33b3bca..d3bb2a82 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs @@ -5,6 +5,7 @@ using BotSharp.Plugins.LLamaSharp; using LLama; using LLama.Common; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; @@ -13,34 +14,39 @@ namespace BotSharp.Plugin.LLamaSharp.Providers; public class TextCompletionProvider : ITextCompletion { private readonly IServiceProvider _services; + private readonly ILogger _logger; private readonly LlamaSharpSettings _settings; + private readonly ITokenStatistics _tokenStatistics; private string _model; public string Provider => "llama-sharp"; public TextCompletionProvider(IServiceProvider services, - LlamaSharpSettings settings) + ILogger logger, + LlamaSharpSettings settings, + ITokenStatistics tokenStatistics) { _services = services; + _logger = logger; _settings = settings; + _tokenStatistics = tokenStatistics; } public Task GetCompletion(string text) { - var state = _services.GetRequiredService(); - var model = state.GetState("model", _settings.DefaultModel); - var llama = _services.GetRequiredService(); - llama.LoadModel(model); + llama.LoadModel(_model); var executor = new InstructExecutor(llama.Model.CreateContext(llama.Params)); var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 }; + _tokenStatistics.StartTimer(); string totalResponse = ""; foreach (var response in executor.Infer(text, inferenceParams)) { Console.Write(response); totalResponse += response; } + _tokenStatistics.StopTimer(); return Task.FromResult(totalResponse); }