From 8531d8ab09b6a1984ceb320d07c075115eedae8d Mon Sep 17 00:00:00 2001 From: Haiping Date: Mon, 16 Oct 2023 15:07:07 -0500 Subject: [PATCH] Add IContentGeneratingHook. --- docs/architecture/hooks.md | 21 ++++- .../BotSharp.Abstraction/Agents/IAgentHook.cs | 4 +- .../Conversations/ConversationHookBase.cs | 25 ++++-- .../Conversations/IConversationHook.cs | 52 ++++++++++-- .../MLTasks/IContentGeneratingHook.cs | 19 +++++ .../ConversationService.SendMessage.cs | 16 ++-- .../Services/ConversationService.cs | 8 ++ .../Handlers/ConversationEndRoutingHandler.cs | 2 +- .../HumanInterventionNeededHandler.cs | 41 ++++++++++ .../Routing/Handlers/TaskEndRoutingHandler.cs | 20 ++++- .../RoutingService.GetNextInstruction.cs | 10 +-- .../AzureOpenAiPlugin.cs | 2 + .../Hooks/TokenStatsConversationHook.cs | 35 ++++++++ .../Providers/ChatCompletionProvider.cs | 79 ++++++++++--------- .../Providers/TextCompletionProvider.cs | 34 ++++---- .../Providers/ChatCompletionProvider.cs | 20 +++-- .../Providers/TextCompletionProvider.cs | 17 +++- .../Providers/ChatCompletionProvider.cs | 26 ++++++ .../Providers/ChatCompletionProvider.cs | 38 ++++----- .../Providers/TextCompletionProvider.cs | 32 ++++---- .../Providers/TextEmbeddingProvider.cs | 6 -- .../BotSharp.Plugin.LLamaSharp/Using.cs | 21 +++++ .../RoutingConversationHook.cs | 4 +- .../{ => templates}/next_step_prompt.liquid | 3 +- .../BotSharp.TestingConsole.csproj | 1 + 25 files changed, 393 insertions(+), 143 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/IContentGeneratingHook.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Hooks/TokenStatsConversationHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.LLamaSharp/Using.cs rename src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/{ => templates}/next_step_prompt.liquid (74%) diff --git a/docs/architecture/hooks.md b/docs/architecture/hooks.md index ac655a5e..c75fb05c 100644 --- a/docs/architecture/hooks.md +++ b/docs/architecture/hooks.md @@ -5,25 +5,40 @@ ## Agent Hook `IAgentHook` ```csharp +// Triggered when agent is loading. bool OnAgentLoading(ref string id); bool OnInstructionLoaded(string template, Dictionary dict); bool OnFunctionsLoaded(List functions); bool OnSamplesLoaded(ref string samples); -Agent OnAgentLoaded(); + +// Triggered when agent is loaded completely. +void OnAgentLoaded(Agent agent); ``` More information about agent hook please go to [Agent Hook](../agent/hook.md). ## Conversation Hook `IConversationHook` ```csharp +// Triggered once for every new conversation. +Task OnConversationInitialized(Conversation conversation); Task OnDialogsLoaded(List dialogs); -Task BeforeCompletion(); +Task OnMessageReceived(RoleDialogModel message); + +// Triggered before LLM calls function. Task OnFunctionExecuting(RoleDialogModel message); + +// Triggered when the function calling completed. Task OnFunctionExecuted(RoleDialogModel message); -Task AfterCompletion(RoleDialogModel message); +Task OnResponseGenerated(RoleDialogModel message); + +// LLM detected the current task is completed. +Task CurrentTaskEnding(RoleDialogModel conversation); // LLM detected the user's intention to end the conversation Task ConversationEnding(RoleDialogModel conversation); + +// LLM can't handle user's request or user requests human being to involve. +Task HumanInterventionNeeded(RoleDialogModel conversation); ``` More information about conversation hook please go to [Conversation Hook](../conversation/hook.md). diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs index ee6234b0..047a0b03 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs @@ -8,13 +8,13 @@ public interface IAgentHook void SetAget(Agent agent); /// - /// Triggered before loading, you can change the returned id to switch agent. + /// Triggered when agent is loading. + /// Return different agent for redirection purpose. /// /// Agent Id /// bool OnAgentLoading(ref string id); - bool OnInstructionLoaded(string template, Dictionary dict); bool OnFunctionsLoaded(List functions); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index 47808628..d7fc039b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -36,7 +36,23 @@ public abstract class ConversationHookBase : IConversationHook return Task.CompletedTask; } - public virtual Task BeforeCompletion(RoleDialogModel message) + public virtual Task OnDialogsLoaded(List dialogs) + { + _dialogs = dialogs; + return Task.CompletedTask; + } + + public virtual Task ConversationEnding(RoleDialogModel message) + { + return Task.CompletedTask; + } + + public virtual Task CurrentTaskEnding(RoleDialogModel conversation) + { + return Task.CompletedTask; + } + + public virtual Task HumanInterventionNeeded(RoleDialogModel conversation) { return Task.CompletedTask; } @@ -51,18 +67,17 @@ public abstract class ConversationHookBase : IConversationHook return Task.CompletedTask; } - public virtual Task AfterCompletion(RoleDialogModel message) + public virtual Task OnMessageReceived(RoleDialogModel message) { return Task.CompletedTask; } - public virtual Task OnDialogsLoaded(List dialogs) + public virtual Task OnResponseGenerated(RoleDialogModel message) { - _dialogs = dialogs; return Task.CompletedTask; } - public virtual Task ConversationEnding(RoleDialogModel message) + public virtual Task OnConversationInitialized(Conversation conversation) { return Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index c754ff16..e6553216 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -4,14 +4,21 @@ public interface IConversationHook { int Priority { get; } Agent Agent { get; } + List Dialogs { get; } IConversationHook SetAgent(Agent agent); Conversation Conversation { get; } IConversationHook SetConversation(Conversation conversation); - List Dialogs { get; } /// - /// Triggered when dialog history is loaded + /// Triggered once for every new conversation. + /// + /// + /// + Task OnConversationInitialized(Conversation conversation); + + /// + /// Triggered when dialog history is loaded. /// /// /// @@ -20,10 +27,43 @@ public interface IConversationHook Task OnStateLoaded(ConversationState state); Task OnStateChanged(string name, string preValue, string currentValue); - Task BeforeCompletion(RoleDialogModel message); - Task OnFunctionExecuting(RoleDialogModel message); - Task OnFunctionExecuted(RoleDialogModel message); - Task AfterCompletion(RoleDialogModel message); + Task OnMessageReceived(RoleDialogModel message); + /// + /// Triggered before LLM calls function. + /// + /// + /// + Task OnFunctionExecuting(RoleDialogModel message); + + /// + /// Triggered when the function calling completed. + /// + /// + /// + Task OnFunctionExecuted(RoleDialogModel message); + + Task OnResponseGenerated(RoleDialogModel message); + + /// + /// LLM detected the current task is completed. + /// It's useful for the situation of multiple tasks in the same conversation. + /// + /// + /// + Task CurrentTaskEnding(RoleDialogModel conversation); + + /// + /// LLM detected the whole conversation is going to be end. + /// + /// + /// Task ConversationEnding(RoleDialogModel conversation); + + /// + /// LLM can't handle user's request or user requests human being to involve. + /// + /// + /// + Task HumanInterventionNeeded(RoleDialogModel conversation); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IContentGeneratingHook.cs new file mode 100644 index 00000000..603031b2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IContentGeneratingHook.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Abstraction.MLTasks; + +/// +/// Model content generating hook, it can be used for logging, metrics and tracing. +/// +public interface IContentGeneratingHook +{ + /// + /// Before content generating. + /// + /// + Task BeforeGenerating(Agent agent, List conversations) => Task.CompletedTask; + + /// + /// After content generated. + /// + /// + Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) => Task.CompletedTask; +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index d4736e7e..da44f034 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -32,7 +32,7 @@ public partial class ConversationService hook.SetAgent(agent) .SetConversation(conversation); - await hook.BeforeCompletion(incoming); + await hook.OnMessageReceived(incoming); // Interrupted by hook if (incoming.StopCompletion) @@ -79,14 +79,6 @@ public partial class ConversationService private async Task HandleAssistantMessage(RoleDialogModel message, Func onMessageReceived) { - var hooks = _services.GetServices().ToList(); - - // After chat completion hook - foreach (var hook in hooks) - { - await hook.AfterCompletion(message); - } - var routingSetting = _services.GetRequiredService(); var agentName = routingSetting.RouterId == message.CurrentAgentId ? "Router" : @@ -101,6 +93,12 @@ public partial class ConversationService _logger.LogInformation(text); #endif + var hooks = _services.GetServices().ToList(); + foreach (var hook in hooks) + { + await hook.OnResponseGenerated(message); + } + await onMessageReceived(message); // Add to dialog history diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index daccc6a9..51e4e24a 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories; namespace BotSharp.Core.Conversations.Services; @@ -64,6 +65,13 @@ public partial class ConversationService : IConversationService record.Title = "New Conversation"; db.CreateNewConversation(record); + + var hooks = _services.GetServices().ToList(); + foreach (var hook in hooks) + { + await hook.OnConversationInitialized(record); + } + return record; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs index bf3ad621..039b90e8 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs @@ -32,7 +32,7 @@ public class ConversationEndRoutingHandler : RoutingHandlerBase, IRoutingHandler foreach (var hook in hooks) { - await hook.OnFunctionExecuting(result); + await hook.ConversationEnding(result); } return result; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs new file mode 100644 index 00000000..bdd6e302 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs @@ -0,0 +1,41 @@ +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Routing; +using BotSharp.Abstraction.Routing.Settings; + +namespace BotSharp.Core.Routing.Handlers; + +public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandler +{ + public string Name => "human_intervention_needed"; + + public string Description => "Reach out to a real human or customer representative."; + + private readonly RoutingSettings _settings; + + public HumanInterventionNeededHandler(IServiceProvider services, ILogger logger, RoutingSettings settings) + : base(services, logger, settings) + { + _settings = settings; + } + + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) + { + var result = new RoleDialogModel(AgentRole.Assistant, inst.Response) + { + CurrentAgentId = _settings.RouterId, + FunctionName = inst.Function, + ExecutionData = inst + }; + + var hooks = _services.GetServices() + .OrderBy(x => x.Priority) + .ToList(); + + foreach (var hook in hooks) + { + await hook.HumanInterventionNeeded(result); + } + + return result; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs index a9b42865..671510dd 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs @@ -23,8 +23,24 @@ public class TaskEndRoutingHandler : RoutingHandlerBase, IRoutingHandler { } - public Task Handle(IRoutingService routing, FunctionCallFromLlm inst) + public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst) { - throw new NotImplementedException(); + var result = new RoleDialogModel(AgentRole.Assistant, inst.Response) + { + CurrentAgentId = _settings.RouterId, + FunctionName = inst.Function, + ExecutionData = inst + }; + + var hooks = _services.GetServices() + .OrderBy(x => x.Priority) + .ToList(); + + foreach (var hook in hooks) + { + await hook.CurrentTaskEnding(result); + } + + return result; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs index 3bb71e94..e44a8860 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Templating; using System.Drawing; @@ -133,10 +134,9 @@ public partial class RoutingService #endif private string GetNextStepPrompt() { - var agentService = _services.GetRequiredService(); - var agentSettings = _services.GetRequiredService(); - var filePath = Path.Combine(agentService.GetAgentDataDir(_routerInstance.AgentId), $"next_step_prompt.{agentSettings.TemplateFormat}"); - var template = File.ReadAllText(filePath); + var db = _services.GetRequiredService(); + // _routerInstance.Router.Templates.First(x => x.Name == "next_step_prompt").Content; + var template = db.GetAgentTemplate(_routerInstance.AgentId, "next_step_prompt"); // If enabled reasoning // JsonSerializer.Serialize(new FunctionCallFromLlm()); @@ -144,7 +144,7 @@ public partial class RoutingService var render = _services.GetRequiredService(); return render.Render(template, new Dictionary { - { "enabled_reasoning", false } + { "enabled_reasoning", _settings.EnableReasoning } }); } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index d5baff1e..e18316c1 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Utilities; +using BotSharp.Plugin.AzureOpenAI.Hooks; using BotSharp.Plugin.AzureOpenAI.Providers; using BotSharp.Plugin.AzureOpenAI.Settings; using Microsoft.Extensions.Configuration; @@ -29,5 +30,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Hooks/TokenStatsConversationHook.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Hooks/TokenStatsConversationHook.cs new file mode 100644 index 00000000..fc2c2827 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Hooks/TokenStatsConversationHook.cs @@ -0,0 +1,35 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.MLTasks; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AzureOpenAI.Hooks; + +/// +/// Token statistics for Azure OpenAI +/// +public class TokenStatsConversationHook : IContentGeneratingHook +{ + private readonly ITokenStatistics _tokenStatistics; + + public TokenStatsConversationHook(ITokenStatistics tokenStatistics) + { + _tokenStatistics = tokenStatistics; + } + + public async Task BeforeGenerating(Agent agent, List conversations) + { + _tokenStatistics.StartTimer(); + } + + public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) + { + _tokenStatistics.StopTimer(); + + tokenStats.PromptCost = 0.0015f; + tokenStats.CompletionCost = 0.002f; + _tokenStatistics.AddToken(tokenStats); + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 33b85a5c..450d5095 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -21,48 +21,45 @@ public class ChatCompletionProvider : IChatCompletion private readonly AzureOpenAiSettings _settings; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly ITokenStatistics _tokenStatistics; + private string _model; public string Provider => "azure-openai"; public ChatCompletionProvider(AzureOpenAiSettings settings, ILogger logger, - IServiceProvider services, - ITokenStatistics tokenStatistics) + IServiceProvider services) { _settings = settings; _logger = logger; _services = services; - _tokenStatistics = tokenStatistics; } public RoleDialogModel GetChatCompletions(Agent agent, List conversations) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(agent, conversations)).ToArray()); + var (client, deploymentModel) = ProviderHelper.GetClient(_model, _settings); var chatCompletionsOptions = PrepareOptions(agent, conversations); - _tokenStatistics.StartTimer(); var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions); - _tokenStatistics.StopTimer(); - var choice = response.Value.Choices[0]; var message = choice.Message; - _tokenStatistics.AddToken(new TokenStatsModel + var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) { - Model = _model, - PromptCount = response.Value.Usage.PromptTokens, - CompletionCount = response.Value.Usage.CompletionTokens, - PromptCost = 0.0015f, - CompletionCost = 0.002f - }); + CurrentAgentId = agent.Id + }; if (choice.FinishReason == CompletionsFinishReason.FunctionCall) { _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name}({message.FunctionCall.Arguments})"); - var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content) + msg = new RoleDialogModel(AgentRole.Function, message.Content) { CurrentAgentId = agent.Id, FunctionName = message.FunctionCall.Name, @@ -70,22 +67,22 @@ public class ChatCompletionProvider : IChatCompletion }; // Somethings LLM will generate a function name with agent name. - if (!string.IsNullOrEmpty(funcContextIn.FunctionName)) + if (!string.IsNullOrEmpty(msg.FunctionName)) { - funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last(); + msg.FunctionName = msg.FunctionName.Split('.').Last(); } - - return funcContextIn; } - else - { - var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) + + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(msg, new TokenStatsModel { - CurrentAgentId = agent.Id - }; + Model = _model, + PromptCount = response.Value.Usage.PromptTokens, + CompletionCount = response.Value.Usage.CompletionTokens + })).ToArray()); - return msg; - } + return msg; } public async Task GetChatCompletionsAsync(Agent agent, @@ -93,6 +90,12 @@ public class ChatCompletionProvider : IChatCompletion Func onMessageReceived, Func onFunctionExecuting) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(agent, conversations)).ToArray()); + var (client, deploymentModel) = ProviderHelper.GetClient(_model, _settings); var chatCompletionsOptions = PrepareOptions(agent, conversations); @@ -100,14 +103,19 @@ public class ChatCompletionProvider : IChatCompletion var choice = response.Value.Choices[0]; var message = choice.Message; - _tokenStatistics.AddToken(new TokenStatsModel + var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) { - Model = _model, - PromptCount = response.Value.Usage.PromptTokens, - CompletionCount = response.Value.Usage.CompletionTokens, - PromptCost = 0.0015f, - CompletionCost = 0.002f - }); + CurrentAgentId = agent.Id + }; + + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(msg, new TokenStatsModel + { + Model = _model, + PromptCount = response.Value.Usage.PromptTokens, + CompletionCount = response.Value.Usage.CompletionTokens + })).ToArray()); if (choice.FinishReason == CompletionsFinishReason.FunctionCall) { @@ -131,11 +139,6 @@ public class ChatCompletionProvider : IChatCompletion } else { - var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) - { - CurrentAgentId= agent.Id - }; - // Text response received await onMessageReceived(msg); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs index 9b74ce6e..98d17b5d 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs @@ -8,6 +8,9 @@ using BotSharp.Abstraction.Conversations; using Microsoft.Extensions.DependencyInjection; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Agents.Enums; +using System.Linq; +using System.Collections.Generic; +using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.AzureOpenAI.Providers; @@ -16,23 +19,26 @@ public class TextCompletionProvider : ITextCompletion private readonly IServiceProvider _services; private readonly AzureOpenAiSettings _settings; private readonly ILogger _logger; - private readonly ITokenStatistics _tokenStatistics; private string _model; public string Provider => "azure-openai"; public TextCompletionProvider(IServiceProvider services, AzureOpenAiSettings settings, - ILogger logger, - ITokenStatistics tokenStatistics) + ILogger logger) { _services = services; _settings = settings; _logger = logger; - _tokenStatistics = tokenStatistics; } public async Task GetCompletion(string text) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(new Agent(), new List { new RoleDialogModel(AgentRole.User, text) })).ToArray()); + var (client, _) = ProviderHelper.GetClient(_model, _settings); var completionsOptions = new CompletionsOptions() @@ -51,20 +57,9 @@ public class TextCompletionProvider : ITextCompletion completionsOptions.Temperature = temperature; completionsOptions.NucleusSamplingFactor = samplingFactor; - _tokenStatistics.StartTimer(); var response = await client.GetCompletionsAsync( deploymentOrModelName: _settings.DeploymentModel.TextCompletionModel, completionsOptions); - _tokenStatistics.StopTimer(); - - _tokenStatistics.AddToken(new TokenStatsModel - { - Model = _model, - PromptCount = response.Value.Usage.PromptTokens, - CompletionCount = response.Value.Usage.CompletionTokens, - PromptCost = 0.0015f, - CompletionCost = 0.002f - }); // OpenAI var completion = ""; @@ -73,7 +68,14 @@ public class TextCompletionProvider : ITextCompletion completion += t.Text; }; - _logger.LogInformation(text); + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel + { + Model = _model, + PromptCount = response.Value.Usage.PromptTokens, + CompletionCount = response.Value.Usage.CompletionTokens + })).ToArray()); return completion.Trim(); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index abe1ca9c..0495799d 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -12,29 +12,30 @@ public class ChatCompletionProvider : IChatCompletion private readonly IServiceProvider _services; private readonly GoogleAiSettings _settings; private readonly ILogger _logger; - private readonly ITokenStatistics _tokenStatistics; private string _model; public ChatCompletionProvider(IServiceProvider services, GoogleAiSettings settings, - ILogger logger, - ITokenStatistics tokenStatistics) + ILogger logger) { _services = services; _settings = settings; _logger = logger; - _tokenStatistics = tokenStatistics; } public RoleDialogModel GetChatCompletions(Agent agent, List conversations) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(agent, conversations)).ToArray()); + var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey); var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI")) .ToList(); - _tokenStatistics.StartTimer(); var response = client.ChatAsync(messages, agent.Instruction, null).Result; - _tokenStatistics.StopTimer(); var message = response.Candidates.First(); var msg = new RoleDialogModel(AgentRole.Assistant, message.Content) @@ -42,6 +43,13 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id }; + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(msg, new TokenStatsModel + { + Model = _model + })).ToArray()); + return msg; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs index fc854726..be73cd73 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; using BotSharp.Plugin.GoogleAI.Settings; using LLMSharp.Google.Palm; @@ -27,16 +28,28 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(new Agent(), new List { new RoleDialogModel(AgentRole.User, text) })).ToArray()); + var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey); _tokenStatistics.StartTimer(); var response = await client.GenerateTextAsync(text, null); _tokenStatistics.StopTimer(); var message = response.Candidates.First(); + var completion = message.Output.Trim(); - _logger.LogInformation(text); + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel + { + Model = _model + })).ToArray()); - return message.Output.Trim(); + return completion; } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 42e4625b..db6daf4f 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -26,6 +26,12 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(agent, conversations)).ToArray()); + var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; @@ -57,6 +63,13 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id }; + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(msg, new TokenStatsModel + { + Model = _model + })).ToArray()); + // Text response received await onMessageReceived(msg); @@ -75,6 +88,12 @@ public class ChatCompletionProvider : IChatCompletion public RoleDialogModel GetChatCompletions(Agent agent, List conversations) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(agent, conversations)).ToArray()); + var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; @@ -106,6 +125,13 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id }; + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(msg, new TokenStatsModel + { + Model = _model + })).ToArray()); + return msg; } } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index a517a63e..46a82dfd 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -1,21 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Conversations; -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Conversations.Settings; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Plugin.LLamaSharp.Settings; -using BotSharp.Plugins.LLamaSharp; -using LLama; -using LLama.Common; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace BotSharp.Plugin.LLamaSharp.Providers; public class ChatCompletionProvider : IChatCompletion @@ -23,24 +5,27 @@ public class ChatCompletionProvider : IChatCompletion private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly LlamaSharpSettings _settings; - private readonly ITokenStatistics _tokenStatistics; private string _model; public ChatCompletionProvider(IServiceProvider services, ILogger logger, - LlamaSharpSettings settings, - ITokenStatistics tokenStatistics) + LlamaSharpSettings settings) { _services = services; _logger = logger; _settings = settings; - _tokenStatistics = tokenStatistics; } public string Provider => "llama-sharp"; public RoleDialogModel GetChatCompletions(Agent agent, List conversations) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(agent, conversations)).ToArray()); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; @@ -65,13 +50,11 @@ public class ChatCompletionProvider : IChatCompletion _logger.LogInformation(prompt); } - _tokenStatistics.StartTimer(); foreach (var response in executor.Infer(prompt, inferenceParams)) { Console.Write(response); totalResponse += response; } - _tokenStatistics.StopTimer(); foreach (var anti in inferenceParams.AntiPrompts) { @@ -83,6 +66,13 @@ public class ChatCompletionProvider : IChatCompletion CurrentAgentId = agent.Id }; + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(msg, new TokenStatsModel + { + Model = _model + })).ToArray()); + return msg; } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs index d3bb2a82..c5a66f6f 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs @@ -1,14 +1,3 @@ -using BotSharp.Abstraction.Conversations; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Plugin.LLamaSharp.Settings; -using BotSharp.Plugins.LLamaSharp; -using LLama; -using LLama.Common; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using System; -using System.Threading.Tasks; - namespace BotSharp.Plugin.LLamaSharp.Providers; public class TextCompletionProvider : ITextCompletion @@ -31,8 +20,14 @@ public class TextCompletionProvider : ITextCompletion _tokenStatistics = tokenStatistics; } - public Task GetCompletion(string text) + public async Task GetCompletion(string text) { + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.BeforeGenerating(new Agent(), new List { new RoleDialogModel(AgentRole.User, text) })).ToArray()); + var llama = _services.GetRequiredService(); llama.LoadModel(_model); @@ -40,15 +35,22 @@ public class TextCompletionProvider : ITextCompletion var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 }; _tokenStatistics.StartTimer(); - string totalResponse = ""; + string completion = ""; foreach (var response in executor.Infer(text, inferenceParams)) { Console.Write(response); - totalResponse += response; + completion += response; } _tokenStatistics.StopTimer(); - return Task.FromResult(totalResponse); + // After chat completion hook + Task.WaitAll(hooks.Select(hook => + hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel + { + Model = _model + })).ToArray()); + + return completion; } public void SetModelName(string model) diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs index d5774735..1a485bf1 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.MLTasks; -using BotSharp.Plugin.LLamaSharp.Settings; -using LLama; -using LLama.Common; -using System; -using System.Collections.Generic; using System.IO; namespace BotSharp.Plugin.LLamaSharp.Providers; diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Using.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Using.cs new file mode 100644 index 00000000..720c10aa --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Using.cs @@ -0,0 +1,21 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Threading.Tasks; +global using System.Linq; +global using System.Text.Json; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.MLTasks; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using System.Text.Json.Serialization; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Plugin.LLamaSharp.Settings; +global using BotSharp.Plugins.LLamaSharp; +global using LLama; +global using LLama.Common; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs index 75aa07f1..368f49bf 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs @@ -24,7 +24,7 @@ public class RoutingConversationHook: ConversationHookBase _services = service; _settings = settings; } - public override async Task BeforeCompletion(RoleDialogModel message) + public override async Task OnMessageReceived(RoleDialogModel message) { var intentClassifier = _services.GetRequiredService(); var vector = intentClassifier.GetTextEmbedding(message.Content); @@ -50,7 +50,7 @@ public class RoutingConversationHook: ConversationHookBase } } - public override async Task AfterCompletion(RoleDialogModel message) + public override async Task OnResponseGenerated(RoleDialogModel message) { var routerSettings = _services.GetRequiredService(); bool saveFlag = message.CurrentAgentId != routerSettings.RouterId; diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/next_step_prompt.liquid b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid similarity index 74% rename from src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/next_step_prompt.liquid rename to src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid index 562672e6..13315c02 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/next_step_prompt.liquid +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid @@ -7,4 +7,5 @@ Response must be in JSON format {"function":"route_to_agent","reason":"the reason why you select this function or agent","response":"content of replying to user","next_action_agent":"agent for next action based on user latest response","user_goal_agent":"agent who can achieve user original goal"} {%- endif %} -If the user has no other tasks need help with, set function as conversation_end with reason and reply user courteously. \ No newline at end of file +If the user has no other tasks need help with, set function as conversation_end with reason and reply user courteously. +If the user wants to reach out to real human being, set function as human_intervention_needed with reason and reply user courteously. \ No newline at end of file diff --git a/tests/BotSharp.TestingConsole/BotSharp.TestingConsole.csproj b/tests/BotSharp.TestingConsole/BotSharp.TestingConsole.csproj index 0bec8ecb..35c76b3d 100644 --- a/tests/BotSharp.TestingConsole/BotSharp.TestingConsole.csproj +++ b/tests/BotSharp.TestingConsole/BotSharp.TestingConsole.csproj @@ -5,6 +5,7 @@ net6.0 enable enable + False