Merge pull request #171 from Deep-Blue-2013/master

Add IContentGeneratingHook.

docs/architecture/hooks.md

@@ -5,25 +5,40 @@
 ## Agent Hook
 `IAgentHook`
 ```csharp
+// Triggered when agent is loading.
 bool OnAgentLoading(ref string id);
 bool OnInstructionLoaded(string template, Dictionary<string, object> dict);
 bool OnFunctionsLoaded(List<FunctionDef> functions);
 bool OnSamplesLoaded(ref string samples);
-Agent OnAgentLoaded();
+
+// Triggered when agent is loaded completely.
+void OnAgentLoaded(Agent agent);
 ```
 More information about agent hook please go to [Agent Hook](../agent/hook.md).
 
 ## Conversation Hook
 `IConversationHook`
 ```csharp
+// Triggered once for every new conversation.
+Task OnConversationInitialized(Conversation conversation);
 Task OnDialogsLoaded(List<RoleDialogModel> dialogs);
-Task BeforeCompletion();
+Task OnMessageReceived(RoleDialogModel message);
+
+// Triggered before LLM calls function.
 Task OnFunctionExecuting(RoleDialogModel message);
+
+// Triggered when the function calling completed.
 Task OnFunctionExecuted(RoleDialogModel message);
-Task AfterCompletion(RoleDialogModel message);
+Task OnResponseGenerated(RoleDialogModel message);
+
+// LLM detected the current task is completed.
+Task CurrentTaskEnding(RoleDialogModel conversation);
 
 // LLM detected the user's intention to end the conversation
 Task ConversationEnding(RoleDialogModel conversation);
+
+// LLM can't handle user's request or user requests human being to involve.
+Task HumanInterventionNeeded(RoleDialogModel conversation);
 ```
 More information about conversation hook please go to [Conversation Hook](../conversation/hook.md).
 

src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs

@@ -8,13 +8,13 @@ public interface IAgentHook
     void SetAget(Agent agent);
 
     /// <summary>
-    /// Triggered before loading, you can change the returned id to switch agent.
+    /// Triggered when agent is loading.
+    /// Return different agent for redirection purpose.
     /// </summary>
     /// <param name="id">Agent Id</param>
     /// <returns></returns>
     bool OnAgentLoading(ref string id);
 
-
     bool OnInstructionLoaded(string template, Dictionary<string, object> dict);
 
     bool OnFunctionsLoaded(List<FunctionDef> functions);

src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs

@@ -36,7 +36,23 @@ public abstract class ConversationHookBase : IConversationHook
         return Task.CompletedTask;
     }
 
-    public virtual Task BeforeCompletion(RoleDialogModel message)
+    public virtual Task OnDialogsLoaded(List<RoleDialogModel> dialogs)
+    {
+        _dialogs = dialogs;
+        return Task.CompletedTask;
+    }
+
+    public virtual Task ConversationEnding(RoleDialogModel message)
+    {
+        return Task.CompletedTask;
+    }
+
+    public virtual Task CurrentTaskEnding(RoleDialogModel conversation)
+    {
+        return Task.CompletedTask;
+    }
+
+    public virtual Task HumanInterventionNeeded(RoleDialogModel conversation)
     {
         return Task.CompletedTask;
     }
@@ -51,18 +67,17 @@ public abstract class ConversationHookBase : IConversationHook
         return Task.CompletedTask;
     }
 
-    public virtual Task AfterCompletion(RoleDialogModel message)
+    public virtual Task OnMessageReceived(RoleDialogModel message)
     {
         return Task.CompletedTask;
     }
 
-    public virtual Task OnDialogsLoaded(List<RoleDialogModel> dialogs)
+    public virtual Task OnResponseGenerated(RoleDialogModel message)
     {
-        _dialogs = dialogs;
         return Task.CompletedTask;
     }
 
-    public virtual Task ConversationEnding(RoleDialogModel message)
+    public virtual Task OnConversationInitialized(Conversation conversation)
     {
         return Task.CompletedTask;
     }

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs

@@ -4,14 +4,21 @@ public interface IConversationHook
 {
     int Priority { get; }
     Agent Agent { get; }
+    List<RoleDialogModel> Dialogs { get; }
     IConversationHook SetAgent(Agent agent);
 
     Conversation Conversation { get; }
     IConversationHook SetConversation(Conversation conversation);
 
-    List<RoleDialogModel> Dialogs { get; }
     /// <summary>
-    /// Triggered when dialog history is loaded
+    /// Triggered once for every new conversation.
+    /// </summary>
+    /// <param name="conversation"></param>
+    /// <returns></returns>
+    Task OnConversationInitialized(Conversation conversation);
+
+    /// <summary>
+    /// Triggered when dialog history is loaded.
     /// </summary>
     /// <param name="dialogs"></param>
     /// <returns></returns>
@@ -20,10 +27,43 @@ public interface IConversationHook
     Task OnStateLoaded(ConversationState state);
     Task OnStateChanged(string name, string preValue, string currentValue);
 
-    Task BeforeCompletion(RoleDialogModel message);
-    Task OnFunctionExecuting(RoleDialogModel message);
-    Task OnFunctionExecuted(RoleDialogModel message);
-    Task AfterCompletion(RoleDialogModel message);
+    Task OnMessageReceived(RoleDialogModel message);
 
+    /// <summary>
+    /// Triggered before LLM calls function.
+    /// </summary>
+    /// <param name="message"></param>
+    /// <returns></returns>
+    Task OnFunctionExecuting(RoleDialogModel message);
+
+    /// <summary>
+    /// Triggered when the function calling completed.
+    /// </summary>
+    /// <param name="message"></param>
+    /// <returns></returns>
+    Task OnFunctionExecuted(RoleDialogModel message);
+
+    Task OnResponseGenerated(RoleDialogModel message);
+
+    /// <summary>
+    /// LLM detected the current task is completed.
+    /// It's useful for the situation of multiple tasks in the same conversation.
+    /// </summary>
+    /// <param name="conversation"></param>
+    /// <returns></returns>
+    Task CurrentTaskEnding(RoleDialogModel conversation);
+
+    /// <summary>
+    /// LLM detected the whole conversation is going to be end.
+    /// </summary>
+    /// <param name="conversation"></param>
+    /// <returns></returns>
     Task ConversationEnding(RoleDialogModel conversation);
+
+    /// <summary>
+    /// LLM can't handle user's request or user requests human being to involve.
+    /// </summary>
+    /// <param name="conversation"></param>
+    /// <returns></returns>
+    Task HumanInterventionNeeded(RoleDialogModel conversation);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IContentGeneratingHook.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Abstraction.MLTasks;
+
+/// <summary>
+/// Model content generating hook, it can be used for logging, metrics and tracing.
+/// </summary>
+public interface IContentGeneratingHook
+{
+    /// <summary>
+    /// Before content generating.
+    /// </summary>
+    /// <returns></returns>
+    Task BeforeGenerating(Agent agent, List<RoleDialogModel> conversations) => Task.CompletedTask;
+
+    /// <summary>
+    /// After content generated.
+    /// </summary>
+    /// <returns></returns>
+    Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) => Task.CompletedTask;
+}

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -32,7 +32,7 @@ public partial class ConversationService
             hook.SetAgent(agent)
                 .SetConversation(conversation);
 
-            await hook.BeforeCompletion(incoming);
+            await hook.OnMessageReceived(incoming);
 
             // Interrupted by hook
             if (incoming.StopCompletion)
@@ -79,14 +79,6 @@ public partial class ConversationService
 
     private async Task HandleAssistantMessage(RoleDialogModel message, Func<RoleDialogModel, Task> onMessageReceived)
     {
-        var hooks = _services.GetServices<IConversationHook>().ToList();
-
-        // After chat completion hook
-        foreach (var hook in hooks)
-        {
-            await hook.AfterCompletion(message);
-        }
-
         var routingSetting = _services.GetRequiredService<RoutingSettings>();
         var agentName = routingSetting.RouterId == message.CurrentAgentId ? 
             "Router" : 
@@ -101,6 +93,12 @@ public partial class ConversationService
         _logger.LogInformation(text);
 #endif
 
+        var hooks = _services.GetServices<IConversationHook>().ToList();
+        foreach (var hook in hooks)
+        {
+            await hook.OnResponseGenerated(message);
+        }
+
         await onMessageReceived(message);
 
         // Add to dialog history

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Repositories;
 
 namespace BotSharp.Core.Conversations.Services;
@@ -64,6 +65,13 @@ public partial class ConversationService : IConversationService
         record.Title = "New Conversation";
 
         db.CreateNewConversation(record);
+
+        var hooks = _services.GetServices<IConversationHook>().ToList();
+        foreach (var hook in hooks)
+        {
+            await hook.OnConversationInitialized(record);
+        }
+
         return record;
     }
 

src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs

@@ -32,7 +32,7 @@ public class ConversationEndRoutingHandler : RoutingHandlerBase, IRoutingHandler
 
         foreach (var hook in hooks)
         {
-            await hook.OnFunctionExecuting(result);
+            await hook.ConversationEnding(result);
         }
 
         return result;

src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs

@@ -0,0 +1,41 @@
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Abstraction.Routing.Settings;
+
+namespace BotSharp.Core.Routing.Handlers;
+
+public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandler
+{
+    public string Name => "human_intervention_needed";
+
+    public string Description => "Reach out to a real human or customer representative.";
+
+    private readonly RoutingSettings _settings;
+
+    public HumanInterventionNeededHandler(IServiceProvider services, ILogger<HumanInterventionNeededHandler> logger, RoutingSettings settings)
+        : base(services, logger, settings)
+    {
+        _settings = settings;
+    }
+
+    public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
+    {
+        var result = new RoleDialogModel(AgentRole.Assistant, inst.Response)
+        {
+            CurrentAgentId = _settings.RouterId,
+            FunctionName = inst.Function,
+            ExecutionData = inst
+        };
+
+        var hooks = _services.GetServices<IConversationHook>()
+            .OrderBy(x => x.Priority)
+            .ToList();
+
+        foreach (var hook in hooks)
+        {
+            await hook.HumanInterventionNeeded(result);
+        }
+
+        return result;
+    }
+}

src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs

@@ -23,8 +23,24 @@ public class TaskEndRoutingHandler : RoutingHandlerBase, IRoutingHandler
     {
     }
 
-    public Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
+    public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
     {
-        throw new NotImplementedException();
+        var result = new RoleDialogModel(AgentRole.Assistant, inst.Response)
+        {
+            CurrentAgentId = _settings.RouterId,
+            FunctionName = inst.Function,
+            ExecutionData = inst
+        };
+
+        var hooks = _services.GetServices<IConversationHook>()
+            .OrderBy(x => x.Priority)
+            .ToList();
+
+        foreach (var hook in hooks)
+        {
+            await hook.CurrentTaskEnding(result);
+        }
+
+        return result;
     }
 }

src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Repositories;
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Templating;
 using System.Drawing;
@@ -133,10 +134,9 @@ public partial class RoutingService
 #endif
     private string GetNextStepPrompt()
     {
-        var agentService = _services.GetRequiredService<IAgentService>();
-        var agentSettings = _services.GetRequiredService<AgentSettings>();
-        var filePath = Path.Combine(agentService.GetAgentDataDir(_routerInstance.AgentId), $"next_step_prompt.{agentSettings.TemplateFormat}");
-        var template = File.ReadAllText(filePath);
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        // _routerInstance.Router.Templates.First(x => x.Name == "next_step_prompt").Content;
+        var template = db.GetAgentTemplate(_routerInstance.AgentId, "next_step_prompt");
 
         // If enabled reasoning
         // JsonSerializer.Serialize(new FunctionCallFromLlm());
@@ -144,7 +144,7 @@ public partial class RoutingService
         var render = _services.GetRequiredService<ITemplateRender>();
         return render.Render(template, new Dictionary<string, object>
         {
-            { "enabled_reasoning", false }
+            { "enabled_reasoning", _settings.EnableReasoning }
         });
     }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Plugins;
 using BotSharp.Abstraction.Utilities;
+using BotSharp.Plugin.AzureOpenAI.Hooks;
 using BotSharp.Plugin.AzureOpenAI.Providers;
 using BotSharp.Plugin.AzureOpenAI.Settings;
 using Microsoft.Extensions.Configuration;
@@ -29,5 +30,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
 
         services.AddScoped<ITextCompletion, TextCompletionProvider>();
         services.AddScoped<IChatCompletion, ChatCompletionProvider>();
+        services.AddScoped<IContentGeneratingHook, TokenStatsConversationHook>();
     }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Hooks/TokenStatsConversationHook.cs

@@ -0,0 +1,35 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Conversations;
+using BotSharp.Abstraction.Conversations.Models;
+using BotSharp.Abstraction.MLTasks;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace BotSharp.Plugin.AzureOpenAI.Hooks;
+
+/// <summary>
+/// Token statistics for Azure OpenAI
+/// </summary>
+public class TokenStatsConversationHook : IContentGeneratingHook
+{
+    private readonly ITokenStatistics _tokenStatistics;
+
+    public TokenStatsConversationHook(ITokenStatistics tokenStatistics)
+    {
+        _tokenStatistics = tokenStatistics;
+    }
+
+    public async Task BeforeGenerating(Agent agent, List<RoleDialogModel> conversations)
+    {
+        _tokenStatistics.StartTimer();
+    }
+
+    public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats)
+    {
+        _tokenStatistics.StopTimer();
+
+        tokenStats.PromptCost = 0.0015f;
+        tokenStats.CompletionCost = 0.002f;
+        _tokenStatistics.AddToken(tokenStats);
+    }
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -21,48 +21,45 @@ public class ChatCompletionProvider : IChatCompletion
     private readonly AzureOpenAiSettings _settings;
     private readonly IServiceProvider _services;
     private readonly ILogger _logger;
-    private readonly ITokenStatistics _tokenStatistics;
+    
     private string _model;
 
     public string Provider => "azure-openai";
 
     public ChatCompletionProvider(AzureOpenAiSettings settings, 
         ILogger<ChatCompletionProvider> logger,
-        IServiceProvider services,
-        ITokenStatistics tokenStatistics)
+        IServiceProvider services)
     {
         _settings = settings;
         _logger = logger;
         _services = services;
-        _tokenStatistics = tokenStatistics;
     }
 
     public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook => 
+            hook.BeforeGenerating(agent, conversations)).ToArray());
+
         var (client, deploymentModel) = ProviderHelper.GetClient(_model, _settings);
         var chatCompletionsOptions = PrepareOptions(agent, conversations);
 
-        _tokenStatistics.StartTimer();
         var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions);
-        _tokenStatistics.StopTimer();
-
         var choice = response.Value.Choices[0];
         var message = choice.Message;
 
-        _tokenStatistics.AddToken(new TokenStatsModel
+        var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
         {
-            Model = _model,
-            PromptCount = response.Value.Usage.PromptTokens,
-            CompletionCount = response.Value.Usage.CompletionTokens,
-            PromptCost = 0.0015f,
-            CompletionCost = 0.002f
-        });
+            CurrentAgentId = agent.Id
+        };
 
         if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
         {
             _logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name}({message.FunctionCall.Arguments})");
 
-            var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content)
+            msg = new RoleDialogModel(AgentRole.Function, message.Content)
             {
                 CurrentAgentId = agent.Id,
                 FunctionName = message.FunctionCall.Name,
@@ -70,22 +67,22 @@ public class ChatCompletionProvider : IChatCompletion
             };
 
             // Somethings LLM will generate a function name with agent name.
-            if (!string.IsNullOrEmpty(funcContextIn.FunctionName))
+            if (!string.IsNullOrEmpty(msg.FunctionName))
             {
-                funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last();
+                msg.FunctionName = msg.FunctionName.Split('.').Last();
             }
-
-            return funcContextIn;
         }
-        else
-        {
-            var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
+
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(msg, new TokenStatsModel
             {
-                CurrentAgentId = agent.Id
-            };
+                Model = _model,
+                PromptCount = response.Value.Usage.PromptTokens,
+                CompletionCount = response.Value.Usage.CompletionTokens
+            })).ToArray());
 
-            return msg;
-        }
+        return msg;
     }
 
     public async Task<bool> GetChatCompletionsAsync(Agent agent, 
@@ -93,6 +90,12 @@ public class ChatCompletionProvider : IChatCompletion
         Func<RoleDialogModel, Task> onMessageReceived,
         Func<RoleDialogModel, Task> onFunctionExecuting)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(agent, conversations)).ToArray());
+
         var (client, deploymentModel) = ProviderHelper.GetClient(_model, _settings);
         var chatCompletionsOptions = PrepareOptions(agent, conversations);
 
@@ -100,14 +103,19 @@ public class ChatCompletionProvider : IChatCompletion
         var choice = response.Value.Choices[0];
         var message = choice.Message;
 
-        _tokenStatistics.AddToken(new TokenStatsModel
+        var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
         {
-            Model = _model,
-            PromptCount = response.Value.Usage.PromptTokens, 
-            CompletionCount = response.Value.Usage.CompletionTokens,
-            PromptCost = 0.0015f,
-            CompletionCost = 0.002f
-        });
+            CurrentAgentId = agent.Id
+        };
+
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(msg, new TokenStatsModel
+            {
+                Model = _model,
+                PromptCount = response.Value.Usage.PromptTokens,
+                CompletionCount = response.Value.Usage.CompletionTokens
+            })).ToArray());
 
         if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
         {
@@ -131,11 +139,6 @@ public class ChatCompletionProvider : IChatCompletion
         }
         else
         {
-            var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
-            {
-                CurrentAgentId= agent.Id
-            };
-
             // Text response received
             await onMessageReceived(msg);
         }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs

@@ -8,6 +8,9 @@ using BotSharp.Abstraction.Conversations;
 using Microsoft.Extensions.DependencyInjection;
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Agents.Enums;
+using System.Linq;
+using System.Collections.Generic;
+using BotSharp.Abstraction.Agents.Models;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
@@ -16,23 +19,26 @@ public class TextCompletionProvider : ITextCompletion
     private readonly IServiceProvider _services;
     private readonly AzureOpenAiSettings _settings;
     private readonly ILogger _logger;
-    private readonly ITokenStatistics _tokenStatistics;
     private string _model;
     public string Provider => "azure-openai";
 
     public TextCompletionProvider(IServiceProvider services,
         AzureOpenAiSettings settings, 
-        ILogger<TextCompletionProvider> logger,
-        ITokenStatistics tokenStatistics)
+        ILogger<TextCompletionProvider> logger)
     {
         _services = services;
         _settings = settings;
         _logger = logger;
-        _tokenStatistics = tokenStatistics;
     }
 
     public async Task<string> GetCompletion(string text)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(new Agent(), new List<RoleDialogModel> { new RoleDialogModel(AgentRole.User, text) })).ToArray());
+
         var (client, _) = ProviderHelper.GetClient(_model, _settings);
 
         var completionsOptions = new CompletionsOptions()
@@ -51,20 +57,9 @@ public class TextCompletionProvider : ITextCompletion
         completionsOptions.Temperature = temperature;
         completionsOptions.NucleusSamplingFactor = samplingFactor;
 
-        _tokenStatistics.StartTimer();
         var response = await client.GetCompletionsAsync(
             deploymentOrModelName: _settings.DeploymentModel.TextCompletionModel,
             completionsOptions);
-        _tokenStatistics.StopTimer();
-
-        _tokenStatistics.AddToken(new TokenStatsModel
-        {
-            Model = _model,
-            PromptCount = response.Value.Usage.PromptTokens,
-            CompletionCount = response.Value.Usage.CompletionTokens,
-            PromptCost = 0.0015f,
-            CompletionCost = 0.002f
-        });
 
         // OpenAI
         var completion = "";
@@ -73,7 +68,14 @@ public class TextCompletionProvider : ITextCompletion
             completion += t.Text;
         };
 
-        _logger.LogInformation(text);
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel
+            {
+                Model = _model,
+                PromptCount = response.Value.Usage.PromptTokens,
+                CompletionCount = response.Value.Usage.CompletionTokens
+            })).ToArray());
 
         return completion.Trim();
     }

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs

@@ -12,29 +12,30 @@ public class ChatCompletionProvider : IChatCompletion
     private readonly IServiceProvider _services;
     private readonly GoogleAiSettings _settings;
     private readonly ILogger _logger;
-    private readonly ITokenStatistics _tokenStatistics;
     private string _model;
 
     public ChatCompletionProvider(IServiceProvider services, 
         GoogleAiSettings settings,
-        ILogger<ChatCompletionProvider> logger,
-        ITokenStatistics tokenStatistics)
+        ILogger<ChatCompletionProvider> logger)
     {
         _services = services;
         _settings = settings;
         _logger = logger;
-        _tokenStatistics = tokenStatistics;
     }
 
     public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(agent, conversations)).ToArray());
+
         var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey);
         var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI"))
             .ToList();
 
-        _tokenStatistics.StartTimer();
         var response = client.ChatAsync(messages, agent.Instruction, null).Result;
-        _tokenStatistics.StopTimer();
 
         var message = response.Candidates.First();
         var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
@@ -42,6 +43,13 @@ public class ChatCompletionProvider : IChatCompletion
             CurrentAgentId = agent.Id
         };
 
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(msg, new TokenStatsModel
+            {
+                Model = _model
+            })).ToArray());
+
         return msg;
     }
 

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.Conversations;
 using BotSharp.Plugin.GoogleAI.Settings;
 using LLMSharp.Google.Palm;
@@ -27,16 +28,28 @@ public class TextCompletionProvider : ITextCompletion
 
     public async Task<string> GetCompletion(string text)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(new Agent(), new List<RoleDialogModel> { new RoleDialogModel(AgentRole.User, text) })).ToArray());
+
         var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey);
         _tokenStatistics.StartTimer();
         var response = await client.GenerateTextAsync(text, null);
         _tokenStatistics.StopTimer();
 
         var message = response.Candidates.First();
+        var completion = message.Output.Trim();
 
-        _logger.LogInformation(text);
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel
+            {
+                Model = _model
+            })).ToArray());
 
-        return message.Output.Trim();
+        return completion;
     }
 
     public void SetModelName(string model)

src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs

@@ -26,6 +26,12 @@ public class ChatCompletionProvider : IChatCompletion
 
     public async Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(agent, conversations)).ToArray());
+
         var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim();
         content += $"\r\n{AgentRole.Assistant}: ";
 
@@ -57,6 +63,13 @@ public class ChatCompletionProvider : IChatCompletion
             CurrentAgentId = agent.Id
         };
 
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(msg, new TokenStatsModel
+            {
+                Model = _model
+            })).ToArray());
+
         // Text response received
         await onMessageReceived(msg);
 
@@ -75,6 +88,12 @@ public class ChatCompletionProvider : IChatCompletion
 
     public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(agent, conversations)).ToArray());
+
         var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim();
         content += $"\r\n{AgentRole.Assistant}: ";
 
@@ -106,6 +125,13 @@ public class ChatCompletionProvider : IChatCompletion
             CurrentAgentId = agent.Id
         };
 
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(msg, new TokenStatsModel
+            {
+                Model = _model
+            })).ToArray());
+
         return msg;
     }
 }

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs

@@ -1,21 +1,3 @@
-using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.Agents.Models;
-using BotSharp.Abstraction.Conversations;
-using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Conversations.Settings;
-using BotSharp.Abstraction.MLTasks;
-using BotSharp.Plugin.LLamaSharp.Settings;
-using BotSharp.Plugins.LLamaSharp;
-using LLama;
-using LLama.Common;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace BotSharp.Plugin.LLamaSharp.Providers;
 
 public class ChatCompletionProvider : IChatCompletion
@@ -23,24 +5,27 @@ public class ChatCompletionProvider : IChatCompletion
     private readonly IServiceProvider _services;
     private readonly ILogger _logger;
     private readonly LlamaSharpSettings _settings;
-    private readonly ITokenStatistics _tokenStatistics;
     private string _model;
 
     public ChatCompletionProvider(IServiceProvider services,
         ILogger<ChatCompletionProvider> logger,
-        LlamaSharpSettings settings,
-        ITokenStatistics tokenStatistics)
+        LlamaSharpSettings settings)
     {
         _services = services;
         _logger = logger;
         _settings = settings;
-        _tokenStatistics = tokenStatistics;
     }
 
     public string Provider => "llama-sharp";
 
     public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(agent, conversations)).ToArray());
+
         var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
         content += $"\r\n{AgentRole.Assistant}: ";
 
@@ -65,13 +50,11 @@ public class ChatCompletionProvider : IChatCompletion
             _logger.LogInformation(prompt);
         }
 
-        _tokenStatistics.StartTimer();
         foreach (var response in executor.Infer(prompt, inferenceParams))
         {
             Console.Write(response);
             totalResponse += response;
         }
-        _tokenStatistics.StopTimer();
 
         foreach (var anti in inferenceParams.AntiPrompts)
         {
@@ -83,6 +66,13 @@ public class ChatCompletionProvider : IChatCompletion
             CurrentAgentId = agent.Id
         };
 
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(msg, new TokenStatsModel
+            {
+                Model = _model
+            })).ToArray());
+
         return msg;
     }
 

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs

@@ -1,14 +1,3 @@
-using BotSharp.Abstraction.Conversations;
-using BotSharp.Abstraction.MLTasks;
-using BotSharp.Plugin.LLamaSharp.Settings;
-using BotSharp.Plugins.LLamaSharp;
-using LLama;
-using LLama.Common;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Threading.Tasks;
-
 namespace BotSharp.Plugin.LLamaSharp.Providers;
 
 public class TextCompletionProvider : ITextCompletion
@@ -31,8 +20,14 @@ public class TextCompletionProvider : ITextCompletion
         _tokenStatistics = tokenStatistics;
     }
 
-    public Task<string> GetCompletion(string text)
+    public async Task<string> GetCompletion(string text)
     {
+        var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        // Before chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.BeforeGenerating(new Agent(), new List<RoleDialogModel> { new RoleDialogModel(AgentRole.User, text) })).ToArray());
+
         var llama = _services.GetRequiredService<LlamaAiModel>();
         llama.LoadModel(_model);
 
@@ -40,15 +35,22 @@ public class TextCompletionProvider : ITextCompletion
         var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 };
 
         _tokenStatistics.StartTimer();
-        string totalResponse = "";
+        string completion = "";
         foreach (var response in executor.Infer(text, inferenceParams))
         {
             Console.Write(response);
-            totalResponse += response;
+            completion += response;
         }
         _tokenStatistics.StopTimer();
 
-        return Task.FromResult(totalResponse);
+        // After chat completion hook
+        Task.WaitAll(hooks.Select(hook =>
+            hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel
+            {
+                Model = _model
+            })).ToArray());
+
+        return completion;
     }
 
     public void SetModelName(string model)

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs

@@ -1,9 +1,3 @@
-using BotSharp.Abstraction.MLTasks;
-using BotSharp.Plugin.LLamaSharp.Settings;
-using LLama;
-using LLama.Common;
-using System;
-using System.Collections.Generic;
 using System.IO;
 
 namespace BotSharp.Plugin.LLamaSharp.Providers;

src/Plugins/BotSharp.Plugin.LLamaSharp/Using.cs

@@ -0,0 +1,21 @@
+global using System;
+global using System.Collections.Generic;
+global using System.Text;
+global using System.Threading.Tasks;
+global using System.Linq;
+global using System.Text.Json;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.MLTasks;
+global using Microsoft.Extensions.Configuration;
+global using Microsoft.Extensions.DependencyInjection;
+global using Microsoft.Extensions.Logging;
+global using System.Text.Json.Serialization;
+global using BotSharp.Abstraction.Utilities;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Settings;
+global using BotSharp.Plugin.LLamaSharp.Settings;
+global using BotSharp.Plugins.LLamaSharp;
+global using LLama;
+global using LLama.Common;

src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs

@@ -24,7 +24,7 @@ public class RoutingConversationHook: ConversationHookBase
         _services = service;
         _settings = settings;
     }
-    public override async Task BeforeCompletion(RoleDialogModel message)
+    public override async Task OnMessageReceived(RoleDialogModel message)
     {
         var intentClassifier = _services.GetRequiredService<IntentClassifier>();
         var vector = intentClassifier.GetTextEmbedding(message.Content);
@@ -50,7 +50,7 @@ public class RoutingConversationHook: ConversationHookBase
         }
     }
 
-    public override async Task AfterCompletion(RoleDialogModel message)
+    public override async Task OnResponseGenerated(RoleDialogModel message)
     {
         var routerSettings = _services.GetRequiredService<RoutingSettings>();
         bool saveFlag = message.CurrentAgentId != routerSettings.RouterId;

src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/next_step_prompt.liquid

@@ -7,4 +7,5 @@ Response must be in JSON format
 {"function":"route_to_agent","reason":"the reason why you select this function or agent","response":"content of replying to user","next_action_agent":"agent for next action based on user latest response","user_goal_agent":"agent who can achieve user original goal"}
 {%- endif %}
 
-If the user has no other tasks need help with, set function as conversation_end with reason and reply user courteously.
+If the user has no other tasks need help with, set function as conversation_end with reason and reply user courteously.
+If the user wants to reach out to real human being, set function as human_intervention_needed with reason and reply user courteously.

tests/BotSharp.TestingConsole/BotSharp.TestingConsole.csproj

@@ -5,6 +5,7 @@
     <TargetFramework>net6.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <GeneratePackageOnBuild>False</GeneratePackageOnBuild>
   </PropertyGroup>
 
   <ItemGroup>