From 3740a8cc7abcc46a5e3fbdbba78d8722fe25723b Mon Sep 17 00:00:00 2001 From: Haiping Date: Wed, 27 Sep 2023 07:51:15 -0500 Subject: [PATCH] Add GetChatCompletions and clean routing code. --- .../Agents/IAgentRouting.cs | 10 - .../MLTasks/IChatCompletion.cs | 3 + .../Routing/IRouterInstance.cs | 13 ++ .../Routing/IRoutingHandler.cs | 8 +- .../Routing/IRoutingService.cs | 5 +- .../Routing/RoutingHandlerBase.cs | 32 +++ .../Agents/Services/AgentService.GetAgents.cs | 4 +- .../BotSharpServiceCollectionExtensions.cs | 2 +- .../ContinueExecuteTaskRoutingHandler.cs | 3 +- .../Handlers/ConversationEndRoutingHandler.cs | 2 +- .../GetNextInstructionRoutingHandler.cs | 24 -- .../InterruptTaskExecutionRoutingHandler.cs | 2 +- .../Handlers/ResponseToUserRoutingHandler.cs | 2 +- .../RetrieveDataFromAgentRoutingHandler.cs | 11 +- .../Handlers/RouteToAgentRoutingHandler.cs | 4 +- .../Routing/Handlers/RoutingHandlerBase.cs | 215 ------------------ .../Routing/Handlers/TaskEndRoutingHandler.cs | 2 +- .../BotSharp.Core/Routing/RouteToAgentFn.cs | 3 +- .../BotSharp.Core/Routing/Router.cs | 74 ------ .../BotSharp.Core/Routing/RouterInstance.cs | 173 ++++++++++++++ .../RoutingService.GetNextInstruction.cs | 82 +++++++ .../Routing/RoutingService.InvokeAgent.cs | 69 ++++++ .../BotSharp.Core/Routing/RoutingService.cs | 128 +++-------- .../Controllers/AgentController.cs | 4 +- .../Providers/ChatCompletionProvider.cs | 48 ++++ .../Providers/ChatCompletionProvider.cs | 69 ++++-- .../Providers/ChatCompletionProvider.cs | 48 ++++ src/WebStarter/appsettings.json | 2 +- 28 files changed, 569 insertions(+), 473 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/IRouterInstance.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Routing/RoutingHandlerBase.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Routing/Handlers/GetNextInstructionRoutingHandler.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs delete mode 100644 src/Infrastructure/BotSharp.Core/Routing/Router.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/RouterInstance.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs deleted file mode 100644 index 7ab7c2b1..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs +++ /dev/null @@ -1,10 +0,0 @@ -using BotSharp.Abstraction.Routing.Models; - -namespace BotSharp.Abstraction.Agents; - -public interface IAgentRouting -{ - string AgentId { get; } - Task LoadRouter(); - RoutingRule[] GetRulesByName(string name); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 3854e1b0..9d8c5b51 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -13,6 +13,9 @@ public interface IChatCompletion /// void SetModelName(string model); + RoleDialogModel GetChatCompletions(Agent agent, + List conversations); + Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRouterInstance.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRouterInstance.cs new file mode 100644 index 00000000..bc204c98 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRouterInstance.cs @@ -0,0 +1,13 @@ +using BotSharp.Abstraction.Routing.Models; + +namespace BotSharp.Abstraction.Routing; + +public interface IRouterInstance +{ + string AgentId { get; } + Agent Router { get; } + List GetHandlers(); + IRouterInstance Load(); + IRouterInstance WithDialogs(List dialogs); + RoutingRule[] GetRulesByName(string name); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHandler.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHandler.cs index ddb83fec..6c26510c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHandler.cs @@ -13,11 +13,7 @@ public interface IRoutingHandler void SetRouter(Agent router) { } - void SetDialogs(List dialogs) { } + void SetDialogs(List dialogs) { } - Task GetNextInstructionFromReasoner(string prompt) - => throw new NotImplementedException(""); - - Task Handle(FunctionCallFromLlm inst) - => throw new NotImplementedException(""); + Task Handle(IRoutingService routing, FunctionCallFromLlm inst); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index 7ed8843d..8c836645 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -1,9 +1,12 @@ +using BotSharp.Abstraction.Functions.Models; + namespace BotSharp.Abstraction.Routing; public interface IRoutingService { - Agent LoadRouter(); List Dialogs { get; } + Task GetNextInstruction(string prompt); + Task InvokeAgent(string agentId); Task InstructLoop(); Task ExecuteOnce(Agent agent); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/RoutingHandlerBase.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/RoutingHandlerBase.cs new file mode 100644 index 00000000..eceeca3a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/RoutingHandlerBase.cs @@ -0,0 +1,32 @@ +using BotSharp.Abstraction.Routing.Settings; +using Microsoft.Extensions.Logging; + +namespace BotSharp.Abstraction.Routing; + +public abstract class RoutingHandlerBase +{ + protected Agent _router; + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + protected RoutingSettings _settings; + protected List _dialogs; + + public RoutingHandlerBase(IServiceProvider services, + ILogger logger, + RoutingSettings settings) + { + _services = services; + _logger = logger; + _settings = settings; + } + + public void SetRouter(Agent router) + { + _router = router; + } + + public void SetDialogs(List dialogs) + { + _dialogs = dialogs; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index 2eee1169..8daa6109 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -23,10 +23,10 @@ public partial class AgentService public async Task GetAgent(string id) { var settings = _services.GetRequiredService(); - var routingService = _services.GetRequiredService(); + var routerInstance = _services.GetRequiredService(); if (settings.RouterId == id) { - return routingService.LoadRouter(); + return routerInstance.Load().Router; } var profile = _db.GetAgent(id); diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 5b3bad2b..35540c6d 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -52,7 +52,7 @@ public static class BotSharpServiceCollectionExtensions config.Bind("Router", routingSettings); services.AddSingleton((IServiceProvider x) => routingSettings); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); if (myDatabaseSettings.Default == "FileRepository") diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs index 52d8ba88..6673f339 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs @@ -26,9 +26,8 @@ public class ContinueExecuteTaskRoutingHandler : RoutingHandlerBase, IRoutingHan { } - public async Task Handle(FunctionCallFromLlm inst) + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { - var routing = _services.GetRequiredService(); var db = _services.GetRequiredService(); var record = db.Agents.First(x => x.Name.ToLower() == inst.AgentName.ToLower()); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs index 35041d27..d3ad6bca 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs @@ -23,7 +23,7 @@ public class ConversationEndRoutingHandler : RoutingHandlerBase, IRoutingHandler { } - public Task Handle(FunctionCallFromLlm inst) + public Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/GetNextInstructionRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/GetNextInstructionRoutingHandler.cs deleted file mode 100644 index 10eafb22..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/GetNextInstructionRoutingHandler.cs +++ /dev/null @@ -1,24 +0,0 @@ -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Routing.Settings; - -namespace BotSharp.Core.Routing.Handlers; - -public class GetNextInstructionRoutingHandler : RoutingHandlerBase, IRoutingHandler -{ - public string Name => "get_next_instruction"; - - public string Description => ""; - - public bool IsReasoning => false; - - public GetNextInstructionRoutingHandler(IServiceProvider services, ILogger logger, RoutingSettings settings) - : base(services, logger, settings) - { - } - - public async Task Handle(FunctionCallFromLlm inst) - { - throw new NotImplementedException(); - } -} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/InterruptTaskExecutionRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/InterruptTaskExecutionRoutingHandler.cs index 5a9dfd43..096f9d61 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/InterruptTaskExecutionRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/InterruptTaskExecutionRoutingHandler.cs @@ -24,7 +24,7 @@ public class InterruptTaskExecutionRoutingHandler : RoutingHandlerBase, IRouting { } - public async Task Handle(FunctionCallFromLlm inst) + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { var result = new RoleDialogModel(AgentRole.User, inst.Reason) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs index 77368127..d2dd1809 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs @@ -24,7 +24,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler { } - public async Task Handle(FunctionCallFromLlm inst) + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { var result = new RoleDialogModel(AgentRole.Assistant, inst.Answer) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs index 97d9ac41..a3d81fe0 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs @@ -27,17 +27,12 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH { } - public async Task Handle(FunctionCallFromLlm inst) + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { - if (string.IsNullOrEmpty(inst.AgentName)) - { - inst = await GetNextInstructionFromReasoner($"What's the next step? your response must have agent name."); - } - // Retrieve information from specific agent var db = _services.GetRequiredService(); var record = db.Agents.First(x => x.Name.ToLower() == inst.AgentName.ToLower()); - var response = await InvokeAgent(record.Id); + var response = await routing.InvokeAgent(record.Id); inst.Answer = response.Content; @@ -60,7 +55,7 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH _router.Instruction += $"\r\n{AgentRole.Function}: {response.Content}"; // Got the response from agent, then send to reasoner again to make the decision - inst = await GetNextInstructionFromReasoner($"What's the next step based on user's original goal and function result?"); + // inst = await GetNextInstructionFromReasoner($"What's the next step based on user's original goal and function result?"); return null; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 0acd7c3f..549f4ef2 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -27,7 +27,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler { } - public async Task Handle(FunctionCallFromLlm inst) + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { var function = _services.GetServices().FirstOrDefault(x => x.Name == inst.Function); var message = new RoleDialogModel(AgentRole.Function, inst.Question) @@ -41,7 +41,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler var ret = await function.Execute(message); - var result = await InvokeAgent(message.CurrentAgentId); + var result = await routing.InvokeAgent(message.CurrentAgentId); result.ExecutionData = result.ExecutionData ?? message.ExecutionData; return result; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs deleted file mode 100644 index 458557fa..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs +++ /dev/null @@ -1,215 +0,0 @@ -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Routing.Models; -using BotSharp.Abstraction.Routing.Settings; -using BotSharp.Abstraction.Templating; -using System.Drawing; -using System.Text.RegularExpressions; - -namespace BotSharp.Core.Routing.Handlers; - -public abstract class RoutingHandlerBase -{ - protected Agent _router; - protected readonly IServiceProvider _services; - protected readonly ILogger _logger; - protected RoutingSettings _settings; - protected List _dialogs; - public virtual bool RequireAgent => true; - - public RoutingHandlerBase(IServiceProvider services, - ILogger logger, - RoutingSettings settings) - { - _services = services; - _logger = logger; - _settings = settings; - } - - public void SetRouter(Agent router) - { - _router = router; - } - - public void SetDialogs(List dialogs) - { - _dialogs = dialogs; - } - - public async Task GetNextInstructionFromReasoner(string prompt) - { - var responseFormat = _settings.EnableReasoning ? - JsonSerializer.Serialize(new FunctionCallFromLlm()) : - JsonSerializer.Serialize(new RoutingArgs - { - Function = "route_to_agent" - }); - var content = $"{prompt} Response must be in JSON format {responseFormat}"; - - var chatCompletion = CompletionProvider.GetChatCompletion(_services, - provider: _settings.Provider, - model: _settings.Model); - - RoleDialogModel response = null; - await chatCompletion.GetChatCompletionsAsync(_router, new List - { - new RoleDialogModel(AgentRole.User, content) - }, async msg - => response = msg, fn - => Task.CompletedTask); - - var args = new FunctionCallFromLlm(); - try - { -#if DEBUG - Console.WriteLine(response.Content, Color.Gray); -#else - _logger.LogInformation(response.Content); -#endif - var pattern = @"\{(?:[^{}]|(?\{)|(?<-open>\}))+(?(open)(?!))\}"; - response.Content = Regex.Match(response.Content, pattern).Value; - args = JsonSerializer.Deserialize(response.Content); - - // Sometimes it populate malformed Function in Agent name - if (!string.IsNullOrEmpty(args.Function) && args.Function == args.AgentName) - { - args.Function = "route_to_agent"; - _logger.LogWarning($"Captured LLM malformed response"); - } - - // Another case of malformed response - var agentService = _services.GetRequiredService(); - var agents = await agentService.GetAgents(); - if (string.IsNullOrEmpty(args.AgentName) && agents.Select(x => x.Name).Contains(args.Function)) - { - args.AgentName = args.Function; - args.Function = "route_to_agent"; - _logger.LogWarning($"Captured LLM malformed response"); - } - } - catch (Exception ex) - { - _logger.LogError($"{ex.Message}: {response.Content}"); - args.Function = "response_to_user"; - args.Answer = ex.Message; - args.AgentName = _settings.RouterName; - } - - if (args.Arguments != null) - { - SaveStateByArgs(args.Arguments); - } - - args.Function = args.Function.Split('.').Last(); - -#if DEBUG - Console.WriteLine($"*** Next Instruction *** {args}", Color.Green); -#else - _logger.LogInformation($"*** Next Instruction *** {args}"); -#endif - - return args; - } - - public async Task GetResponseFromReasoner() - { - var wholeDialogs = new List - { - new RoleDialogModel(AgentRole.User, $"How to response to user?") - }; - - var chatCompletion = CompletionProvider.GetChatCompletion(_services, - provider: _settings.Provider, - model: _settings.Model); - - RoleDialogModel response = null; - await chatCompletion.GetChatCompletionsAsync(_router, wholeDialogs, async msg - => response = msg, fn - => Task.CompletedTask); - - return response; - } - - const int MAXIMUM_RECURSION_DEPTH = 2; - int CurrentRecursionDepth = 0; - protected async Task InvokeAgent(string agentId) - { - CurrentRecursionDepth++; - if (CurrentRecursionDepth > MAXIMUM_RECURSION_DEPTH) - { - return _dialogs.Last(); - } - - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(agentId); - - var chatCompletion = CompletionProvider.GetChatCompletion(_services); - - RoleDialogModel response = null; - await chatCompletion.GetChatCompletionsAsync(agent, _dialogs, - async msg => - { - response = msg; - }, async fn => - { - // execute function - // Save states - SaveStateByArgs(JsonSerializer.Deserialize(fn.FunctionArgs)); - - var conversationService = _services.GetRequiredService(); - // Call functions - await conversationService.CallFunctions(fn); - - if (string.IsNullOrEmpty(fn.Content)) - { - fn.Content = fn.ExecutionResult; - } - - _dialogs.Add(fn); - - if (!fn.StopCompletion) - { - // Find response template - var templateService = _services.GetRequiredService(); - var quickResponse = await templateService.RenderFunctionResponse(agent.Id, fn); - if (!string.IsNullOrEmpty(quickResponse)) - { - response = new RoleDialogModel(AgentRole.Assistant, quickResponse) - { - CurrentAgentId = agent.Id - }; - } - else - { - response = await InvokeAgent(fn.CurrentAgentId); - } - } - else - { - response = fn; - } - }); - - return response; - } - - protected void SaveStateByArgs(JsonDocument args) - { - if (args == null) - { - return; - } - - var stateService = _services.GetRequiredService(); - if (args.RootElement is JsonElement root) - { - foreach (JsonProperty property in root.EnumerateObject()) - { - if (!string.IsNullOrEmpty(property.Value.ToString())) - { - stateService.SetState(property.Name, property.Value); - } - } - } - } -} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs index 607762bf..a9b42865 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs @@ -23,7 +23,7 @@ public class TaskEndRoutingHandler : RoutingHandlerBase, IRoutingHandler { } - public Task Handle(FunctionCallFromLlm inst) + public Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs index 0c16dc09..12e30261 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Functions; using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; using System.Drawing; @@ -57,7 +58,7 @@ public class RouteToAgentFn : IFunctionCallback private bool HasMissingRequiredField(RoleDialogModel message, out string agentId) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - var router = _services.GetRequiredService(); + var router = _services.GetRequiredService(); var routingRules = router.GetRulesByName(args.AgentName); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Router.cs b/src/Infrastructure/BotSharp.Core/Routing/Router.cs deleted file mode 100644 index dfc0c92f..00000000 --- a/src/Infrastructure/BotSharp.Core/Routing/Router.cs +++ /dev/null @@ -1,74 +0,0 @@ -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Routing.Models; -using BotSharp.Abstraction.Routing.Settings; - -namespace BotSharp.Core.Routing; - -public class Router : IAgentRouting -{ - protected readonly IServiceProvider _services; - protected readonly ILogger _logger; - protected readonly RoutingSettings _settings; - - public virtual string AgentId => _settings.RouterId; - - public Router(IServiceProvider services, - ILogger logger, - RoutingSettings settings) - { - _services = services; - _logger = logger; - _settings = settings; - } - - public virtual async Task LoadRouter() - { - var agentService = _services.GetRequiredService(); - return await agentService.LoadAgent(AgentId); - } - -#if !DEBUG - [MemoryCache(10 * 60)] -#endif - protected RoutingRule[] GetRoutingRecords() - { - var db = _services.GetRequiredService(); - - var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray(); - var records = agents.SelectMany(x => - { - x.RoutingRules.ForEach(r => - { - r.AgentId = x.Id; - r.AgentName = x.Name; - }); - return x.RoutingRules; - }).ToArray(); - - // Filter agents by profile - var state = _services.GetRequiredService(); - var name = state.GetState("channel"); - var specifiedProfile = agents.FirstOrDefault(x => x.Profiles.Contains(name)); - if (specifiedProfile != null) - { - records = records.Where(x => specifiedProfile.Profiles.Contains(name)).ToArray(); - } - - return records; - } - - public RoutingRule[] GetRulesByName(string name) - { - return GetRoutingRecords() - .Where(x => x.AgentName.ToLower() == name.ToLower()) - .ToArray(); - } - - public RoutingRule[] GetRulesByAgentId(string id) - { - return GetRoutingRecords() - .Where(x => x.AgentId == id) - .ToArray(); - } -} diff --git a/src/Infrastructure/BotSharp.Core/Routing/RouterInstance.cs b/src/Infrastructure/BotSharp.Core/Routing/RouterInstance.cs new file mode 100644 index 00000000..3782cf60 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/RouterInstance.cs @@ -0,0 +1,173 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Models; +using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Routing; +using BotSharp.Abstraction.Routing.Models; +using BotSharp.Abstraction.Routing.Settings; + +namespace BotSharp.Core.Routing; + +public class RouterInstance : IRouterInstance +{ + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + protected readonly RoutingSettings _settings; + + private Agent _router; + public Agent Router => _router; + public virtual string AgentId => _router.Id; + + public RouterInstance(IServiceProvider services, + ILogger logger, + RoutingSettings settings) + { + _services = services; + _logger = logger; + _settings = settings; + } + + public IRouterInstance Load() + { + var db = _services.GetRequiredService(); + + _router = new Agent() + { + Id = _settings.RouterId, + Name = _settings.RouterName, + Description = _settings.Description + }; + var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray(); + + // Assemble prompt + var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request: +1. Read the CONVERSATION context. +2. Select a appropriate function from FUNCTIONS. +3. Determine which agent is suitable according to conversation context. +4. Re-think about selected function is from FUNCTIONS to handle the request. +5. Make sure agent is not in args."; + + // Append function + prompt += "\r\n"; + prompt += "\r\nFUNCTIONS"; + GetHandlers().Select((handler, i) => + { + prompt += "\r\n"; + prompt += $"\r\n{i + 1}. {handler.Name}"; + prompt += $"\r\n{handler.Description}"; + + // Append parameters + if (handler.Parameters.Any()) + { + prompt += "\r\nParameters:"; + handler.Parameters.Select((p, i) => + { + prompt += $"\r\n - {p.Name}: {p.Description}"; + return p; + }).ToList(); + } + + return handler; + }).ToList(); + + prompt += "\r\n"; + prompt += "\r\nAGENTS"; + agents.Select(x => new RoutingItem + { + AgentId = x.Id, + Description = x.Description, + Name = x.Name, + RequiredFields = x.RoutingRules.Where(x => x.Required) + .Select(x => new NameDesc(x.Field, x.Description)) + .ToList() + }).Select((agent, i) => + { + prompt += "\r\n"; + prompt += $"\r\n{i + 1}. {agent.Name}"; + prompt += $"\r\n{agent.Description}"; + + // Append parameters + if (agent.RequiredFields.Any()) + { + prompt += $"\r\nRequired:"; + agent.RequiredFields.Select((field, i) => + { + prompt += $"\r\n - {field.Name}: {field.Description}"; + return field; + }).ToList(); + } + return agent; + }).ToList(); + + prompt += "\r\n"; + prompt += "\r\nCONVERSATION"; + _router.Instruction = prompt; + + return this; + } + + public IRouterInstance WithDialogs(List dialogs) + { + foreach (var dialog in dialogs.TakeLast(20)) + { + _router.Instruction += $"\r\n{dialog.Role}: {dialog.Content}"; + } + return this; + } + + public List GetHandlers() + { + return _services.GetServices() + .Where(x => x.IsReasoning == _settings.EnableReasoning) + .Where(x => !string.IsNullOrEmpty(x.Description)) + .Select(x => new RoutingHandlerDef + { + Name = x.Name, + Description = x.Description, + Parameters = x.Parameters + }).ToList(); + } + +#if !DEBUG + [MemoryCache(10 * 60)] +#endif + protected RoutingRule[] GetRoutingRecords() + { + var db = _services.GetRequiredService(); + + var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray(); + var records = agents.SelectMany(x => + { + x.RoutingRules.ForEach(r => + { + r.AgentId = x.Id; + r.AgentName = x.Name; + }); + return x.RoutingRules; + }).ToArray(); + + // Filter agents by profile + var state = _services.GetRequiredService(); + var name = state.GetState("channel"); + var specifiedProfile = agents.FirstOrDefault(x => x.Profiles.Contains(name)); + if (specifiedProfile != null) + { + records = records.Where(x => specifiedProfile.Profiles.Contains(name)).ToArray(); + } + + return records; + } + + public RoutingRule[] GetRulesByName(string name) + { + return GetRoutingRecords() + .Where(x => x.AgentName.ToLower() == name.ToLower()) + .ToArray(); + } + + public RoutingRule[] GetRulesByAgentId(string id) + { + return GetRoutingRecords() + .Where(x => x.AgentId == id) + .ToArray(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs new file mode 100644 index 00000000..9286d85b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs @@ -0,0 +1,82 @@ +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Routing; +using BotSharp.Abstraction.Routing.Models; +using System.Drawing; +using System.Text.RegularExpressions; + +namespace BotSharp.Core.Routing; + +public partial class RoutingService +{ + public async Task GetNextInstruction(string prompt) + { + var responseFormat = _settings.EnableReasoning ? + JsonSerializer.Serialize(new FunctionCallFromLlm()) : + JsonSerializer.Serialize(new RoutingArgs + { + Function = "route_to_agent" + }); + var content = $"{prompt} Response must be in JSON format {responseFormat}"; + + var chatCompletion = CompletionProvider.GetChatCompletion(_services, + provider: _settings.Provider, + model: _settings.Model); + + var response = chatCompletion.GetChatCompletions(_routerInstance.Router, new List + { + new RoleDialogModel(AgentRole.User, content) + }); + + var args = new FunctionCallFromLlm(); + try + { +#if DEBUG + Console.WriteLine(response.Content, Color.Gray); +#else + _logger.LogInformation(response.Content); +#endif + var pattern = @"\{(?:[^{}]|(?\{)|(?<-open>\}))+(?(open)(?!))\}"; + response.Content = Regex.Match(response.Content, pattern).Value; + args = JsonSerializer.Deserialize(response.Content); + + // Sometimes it populate malformed Function in Agent name + if (!string.IsNullOrEmpty(args.Function) && args.Function == args.AgentName) + { + args.Function = "route_to_agent"; + _logger.LogWarning($"Captured LLM malformed response"); + } + + // Another case of malformed response + var agentService = _services.GetRequiredService(); + var agents = await agentService.GetAgents(); + if (string.IsNullOrEmpty(args.AgentName) && agents.Select(x => x.Name).Contains(args.Function)) + { + args.AgentName = args.Function; + args.Function = "route_to_agent"; + _logger.LogWarning($"Captured LLM malformed response"); + } + } + catch (Exception ex) + { + _logger.LogError($"{ex.Message}: {response.Content}"); + args.Function = "response_to_user"; + args.Answer = ex.Message; + args.AgentName = _settings.RouterName; + } + + if (args.Arguments != null) + { + SaveStateByArgs(args.Arguments); + } + + args.Function = args.Function.Split('.').Last(); + +#if DEBUG + Console.WriteLine($"*** Next Instruction *** {args}", Color.Green); +#else + _logger.LogInformation($"*** Next Instruction *** {args}"); +#endif + + return args; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs new file mode 100644 index 00000000..c592fb31 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -0,0 +1,69 @@ +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Routing; + +public partial class RoutingService +{ + const int MAXIMUM_RECURSION_DEPTH = 2; + int CurrentRecursionDepth = 0; + public async Task InvokeAgent(string agentId) + { + CurrentRecursionDepth++; + if (CurrentRecursionDepth > MAXIMUM_RECURSION_DEPTH) + { + return Dialogs.Last(); + } + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(agentId); + + var chatCompletion = CompletionProvider.GetChatCompletion(_services); + + RoleDialogModel response = null; + await chatCompletion.GetChatCompletionsAsync(agent, Dialogs, + async msg => + { + response = msg; + }, async fn => + { + // execute function + // Save states + SaveStateByArgs(JsonSerializer.Deserialize(fn.FunctionArgs)); + + var conversationService = _services.GetRequiredService(); + // Call functions + await conversationService.CallFunctions(fn); + + if (string.IsNullOrEmpty(fn.Content)) + { + fn.Content = fn.ExecutionResult; + } + + Dialogs.Add(fn); + + if (!fn.StopCompletion) + { + // Find response template + var templateService = _services.GetRequiredService(); + var quickResponse = await templateService.RenderFunctionResponse(agent.Id, fn); + if (!string.IsNullOrEmpty(quickResponse)) + { + response = new RoleDialogModel(AgentRole.Assistant, quickResponse) + { + CurrentAgentId = agent.Id + }; + } + else + { + response = await InvokeAgent(fn.CurrentAgentId); + } + } + else + { + response = fn; + } + }); + + return response; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 665425d4..f2c7f330 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -1,16 +1,15 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Settings; + namespace BotSharp.Core.Routing; -public class RoutingService : IRoutingService +public partial class RoutingService : IRoutingService { private readonly IServiceProvider _services; private readonly RoutingSettings _settings; + private readonly IRouterInstance _routerInstance; private readonly ILogger _logger; private List _dialogs; public List Dialogs { @@ -28,11 +27,13 @@ public class RoutingService : IRoutingService public RoutingService(IServiceProvider services, RoutingSettings settings, - ILogger logger) + ILogger logger, + IRouterInstance routerInstance) { _services = services; _settings = settings; _logger = logger; + _routerInstance = routerInstance; } @@ -44,7 +45,7 @@ public class RoutingService : IRoutingService var handler = handlers.FirstOrDefault(x => x.Name == "route_to_agent"); handler.SetDialogs(Dialogs); - var result = await handler.Handle(new FunctionCallFromLlm + var result = await handler.Handle(this, new FunctionCallFromLlm { Function = "route_to_agent", Question = message, @@ -57,24 +58,18 @@ public class RoutingService : IRoutingService public async Task InstructLoop() { - var router = LoadRouter(); + _routerInstance.Load().WithDialogs(Dialogs); + var router = _routerInstance.Router; + var result = new RoleDialogModel(AgentRole.Assistant, "Can you repeat your request again?") { CurrentAgentId = router.Id }; var message = Dialogs.Last().Content; - foreach (var dialog in Dialogs.TakeLast(20)) - { - router.Instruction += $"\r\n{dialog.Role}: {dialog.Content}"; - } var handlers = _services.GetServices(); - var handler = handlers.FirstOrDefault(x => x.Name == "get_next_instruction"); - handler.SetRouter(router); - handler.SetDialogs(Dialogs); - int loopCount = 0; var stop = false; while (!stop && loopCount < 5) @@ -82,20 +77,21 @@ public class RoutingService : IRoutingService loopCount++; var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request?"; - var inst = await handler.GetNextInstructionFromReasoner(prompt); + prompt += " Or you can handle without asking specific agent."; + var inst = await GetNextInstruction(prompt); inst.Question = inst.Question ?? message; - handler = handlers.FirstOrDefault(x => x.Name == inst.Function); + var handler = handlers.FirstOrDefault(x => x.Name == inst.Function); if (handler == null) { handler = handlers.FirstOrDefault(x => x.Name == "get_next_instruction"); - router.Instruction += $"\r\n{AgentRole.System}: the function must be one of {string.Join(",", GetHandlers().Select(x => x.Name))}."; + router.Instruction += $"\r\n{AgentRole.System}: the function must be one of {string.Join(",", _routerInstance.GetHandlers().Select(x => x.Name))}."; continue; } handler.SetRouter(router); handler.SetDialogs(Dialogs); - result = await handler.Handle(inst); + result = await handler.Handle(this, inst); message = result.Content.Replace("\r\n", " "); router.Instruction += $"\r\n{result.Role}: {message}"; @@ -106,95 +102,23 @@ public class RoutingService : IRoutingService return result; } - public Agent LoadRouter() + protected void SaveStateByArgs(JsonDocument args) { - var db = _services.GetRequiredService(); - - var router = new Agent() + if (args == null) { - Id = _settings.RouterId, - Name = _settings.RouterName, - Description = _settings.Description - }; - var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray(); + return; + } - // Assemble prompt - var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request: -1. Read the CONVERSATION context. -2. Select a appropriate function from FUNCTIONS. -3. Determine which agent is suitable according to conversation context. -4. Re-think about selected function is from FUNCTIONS to handle the request. -5. Make sure agent is not in args."; - - // Append function - prompt += "\r\n"; - prompt += "\r\nFUNCTIONS"; - GetHandlers().Select((handler, i) => + var stateService = _services.GetRequiredService(); + if (args.RootElement is JsonElement root) { - prompt += "\r\n"; - prompt += $"\r\n{i + 1}. {handler.Name}"; - prompt += $"\r\n{handler.Description}"; - - // Append parameters - if (handler.Parameters.Any()) + foreach (JsonProperty property in root.EnumerateObject()) { - prompt += "\r\nParameters:"; - handler.Parameters.Select((p, i) => + if (!string.IsNullOrEmpty(property.Value.ToString())) { - prompt += $"\r\n - {p.Name}: {p.Description}"; - return p; - }).ToList(); + stateService.SetState(property.Name, property.Value); + } } - - return handler; - }).ToList(); - - prompt += "\r\n"; - prompt += "\r\nAGENTS"; - agents.Select(x => new RoutingItem - { - AgentId = x.Id, - Description = x.Description, - Name = x.Name, - RequiredFields = x.RoutingRules.Where(x => x.Required) - .Select(x => new NameDesc(x.Field, x.Description)) - .ToList() - }).Select((agent, i) => - { - prompt += "\r\n"; - prompt += $"\r\n{i + 1}. {agent.Name}"; - prompt += $"\r\n{agent.Description}"; - - // Append parameters - if (agent.RequiredFields.Any()) - { - prompt += $"\r\nRequired:"; - agent.RequiredFields.Select((field, i) => - { - prompt += $"\r\n - {field.Name}: {field.Description}"; - return field; - }).ToList(); - } - return agent; - }).ToList(); - - prompt += "\r\n"; - prompt += "\r\nCONVERSATION"; - router.Instruction = prompt; - - return router; - } - - private List GetHandlers() - { - return _services.GetServices() - .Where(x => x.IsReasoning == _settings.EnableReasoning) - .Where(x => !string.IsNullOrEmpty(x.Description)) - .Select(x => new RoutingHandlerDef - { - Name = x.Name, - Description = x.Description, - Parameters = x.Parameters - }).ToList(); + } } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 163b541e..bd6a0c43 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -26,8 +26,8 @@ public class AgentController : ControllerBase, IApiAdapter var agents = await _agentService.GetAgents(); // Add the router as agent - var routing = _services.GetRequiredService(); - agents.Insert(0, routing.LoadRouter()); + var routing = _services.GetRequiredService(); + agents.Insert(0, routing.Load().Router); return agents.Select(x => AgentViewModel.FromAgent(x)).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 6e80cfc5..cffa69f7 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -95,6 +95,54 @@ public class ChatCompletionProvider : IChatCompletion return functions; } + public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + { + var (client, deploymentModel) = GetClient(); + var chatCompletionsOptions = PrepareOptions(agent, conversations); + + var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions); + var choice = response.Value.Choices[0]; + var message = choice.Message; + + _tokenStatistics.AddToken(new TokenStatsModel + { + Model = _model, + PromptCount = response.Value.Usage.PromptTokens, + CompletionCount = response.Value.Usage.CompletionTokens, + PromptCost = 0.0015f, + CompletionCost = 0.002f + }); + + if (choice.FinishReason == CompletionsFinishReason.FunctionCall) + { + _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name} => {message.FunctionCall.Arguments}"); + + var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content) + { + CurrentAgentId = agent.Id, + FunctionName = message.FunctionCall.Name, + FunctionArgs = message.FunctionCall.Arguments + }; + + // Somethings LLM will generate a function name with agent name. + if (!string.IsNullOrEmpty(funcContextIn.FunctionName)) + { + funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last(); + } + + return funcContextIn; + } + else + { + var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) + { + CurrentAgentId = agent.Id + }; + + return msg; + } + } + public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index cefdd632..42e4625b 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -39,29 +39,26 @@ public class ChatCompletionProvider : IChatCompletion var api = _services.GetRequiredService(); - if (_model.Contains('/')) + var space = _model.Split('/')[0]; + var model = _model.Split("/")[1]; + + var response = await api.Post(space, model, new InferenceInput { - var space = _model.Split('/')[0]; - var model = _model.Split("/")[1]; + Inputs = prompt + }); - var response = await api.Post(space, model, new InferenceInput - { - Inputs = prompt - }); + var falcon = JsonSerializer.Deserialize>(response); - var falcon = JsonSerializer.Deserialize>(response); + var message = falcon[0].GeneratedText.Trim(); + _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}"); - var message = falcon[0].GeneratedText.Trim(); - _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}"); + var msg = new RoleDialogModel(AgentRole.Assistant, message) + { + CurrentAgentId = agent.Id + }; - var msg = new RoleDialogModel(AgentRole.Assistant, message) - { - CurrentAgentId = agent.Id - }; - - // Text response received - await onMessageReceived(msg); - } + // Text response received + await onMessageReceived(msg); return true; } @@ -75,4 +72,40 @@ public class ChatCompletionProvider : IChatCompletion { _model = model; } + + public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + { + var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim(); + content += $"\r\n{AgentRole.Assistant}: "; + + var prompt = agent.Instruction + "\r\n" + content; + + var convSetting = _services.GetRequiredService(); + if (convSetting.ShowVerboseLog) + { + _logger.LogInformation(prompt); + } + + var api = _services.GetRequiredService(); + + var space = _model.Split('/')[0]; + var model = _model.Split("/")[1]; + + var response = api.Post(space, model, new InferenceInput + { + Inputs = prompt + }).Result; + + var falcon = JsonSerializer.Deserialize>(response); + + var message = falcon[0].GeneratedText.Trim(); + _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}"); + + var msg = new RoleDialogModel(AgentRole.Assistant, message) + { + CurrentAgentId = agent.Id + }; + + return msg; + } } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index dbdaef4d..f0034a6f 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -36,6 +36,54 @@ public class ChatCompletionProvider : IChatCompletion public string Provider => "llama-sharp"; + public RoleDialogModel GetChatCompletions(Agent agent, List conversations) + { + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); + content += $"\r\n{AgentRole.Assistant}: "; + + var state = _services.GetRequiredService(); + var model = state.GetState("model", _settings.DefaultModel); + + var llama = _services.GetRequiredService(); + llama.LoadModel(model); + var executor = llama.GetStatelessExecutor(); + + var inferenceParams = new InferenceParams() + { + Temperature = 0.1f, + AntiPrompts = new List { $"{AgentRole.User}:", "[/INST]" }, + MaxTokens = 64 + }; + + string totalResponse = ""; + + var prompt = agent.Instruction + "\r\n" + content; + + var convSetting = _services.GetRequiredService(); + if (convSetting.ShowVerboseLog) + { + _logger.LogInformation(prompt); + } + + foreach (var response in executor.Infer(prompt, inferenceParams)) + { + Console.Write(response); + totalResponse += response; + } + + foreach (var anti in inferenceParams.AntiPrompts) + { + totalResponse = totalResponse.Replace(anti, "").Trim(); + } + + var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse) + { + CurrentAgentId = agent.Id + }; + + return msg; + } + public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 79f2fae8..fe4552c6 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -19,7 +19,7 @@ "Description": "Pizza restaurant AI Bot", "EnableReasoning": false, "Provider": "azure-openai", - "Model": "gpt-3.5" + "Model": "gpt-3.5-turbo" }, "Agent": {