SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into Production

Browse source
This commit is contained in:
易磊 2025-05-20 09:25:54 +08:00 committed by GitHub
parent 23c02eb9ed 92542d0df7
commit 73a4076cd0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
111 changed files with 1546 additions and 1188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
BotSharp.sln
View file

@ -131,8 +131,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "s
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer", "tests\BotSharp.PizzaBot.MCPServer\BotSharp.PizzaBot.MCPServer.csproj", "{8D2AD45F-836A-516F-DE6A-71443CEBB18A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}"
@ -553,14 +551,6 @@ Global
{8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|Any CPU.Build.0 = Release|Any CPU
{8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|x64.ActiveCfg = Release|Any CPU
{8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|x64.Build.0 = Release|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|Any CPU.Build.0 = Debug|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|x64.ActiveCfg = Debug|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|x64.Build.0 = Debug|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.ActiveCfg = Release|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU
{534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU
@ -672,7 +662,6 @@ Global
{AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
{781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
{B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
{970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}

6
Directory.Packages.props
View file

@ -60,7 +60,7 @@
<PackageVersion Include="LLamaSharp" Version="0.21.0" />
<PackageVersion Include="FaissMask" Version="0.4.2" />
<PackageVersion Include="FastText.NetWrapper" Version="1.3.1" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.4.3-preview.1.25230.7" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.5.0" />
<PackageVersion Include="System.Text.Encodings.Web" Version="8.0.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.1.0" />
<PackageVersion Include="Docnet.Core" Version="2.7.0-alpha.1" />
@ -114,8 +114,8 @@
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageVersion Include="Shouldly" Version="4.3.0" />
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.8" />
<PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.8" />
<PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.11" />
<PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.11" />
</ItemGroup>
<ItemGroup>
<PackageVersion Include="BotSharp.Core" Version="$(BotSharpVersion)" />

15
IFunctionExecutor.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Routing
{
public interface IFunctionExecutor
{
public Task<bool> Execute(RoleDialogModel message);
public Task<string> GetIndication(RoleDialogModel message);
}
}

7
src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs
View file

@ -1,13 +1,10 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Abstraction.Agents;
public interface IAgentHook
public interface IAgentHook : IHookBase
{
/// <summary>
/// Agent Id
/// </summary>
string SelfId { get; }
Agent Agent { get; }
void SetAgent(Agent agent);

20
src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs
View file

@ -1,20 +0,0 @@
namespace BotSharp.Abstraction.Conversations;
public class ConversationHookProvider
{
public IEnumerable<IConversationHook> Hooks { get; }
private readonly Lazy<IEnumerable<IConversationHook>> _hooksOrderByPriority;
public IEnumerable<IConversationHook> HooksOrderByPriority
=> _hooksOrderByPriority.Value;
public ConversationHookProvider(IEnumerable<IConversationHook> conversationHooks)
{
Hooks = conversationHooks;
_hooksOrderByPriority = new Lazy<IEnumerable<IConversationHook>>(() =>
{
return conversationHooks.OrderBy(hook => hook.Priority).ToArray();
});
}
}

4
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Abstraction.Conversations;
public interface IConversationHook
public interface IConversationHook : IHookBase
{
int Priority { get; }
Agent Agent { get; }

13
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
View file

@ -47,15 +47,19 @@ public class DialogElement
public string Content { get; set; } = default!;
[JsonPropertyName("secondary_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? SecondaryContent { get; set; }
[JsonPropertyName("rich_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? RichContent { get; set; }
[JsonPropertyName("secondary_rich_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? SecondaryRichContent { get; set; }
[JsonPropertyName("payload")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Payload { get; set; }
public DialogElement()
@ -95,8 +99,17 @@ public class DialogMetaData
public string MessageType { get; set; } = default!;
[JsonPropertyName("function_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionName { get; set; }
[JsonPropertyName("function_args")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionArgs { get; set; }
[JsonPropertyName("tool_call_id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ToolCallId { get; set; }
[JsonPropertyName("sender_id")]
public string? SenderId { get; set; }

4
src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Abstraction.Crontab;
public interface ICrontabHook
public interface ICrontabHook : IHookBase
{
string[]? Triggers
=> null;

19
src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs Normal file
View file

@ -0,0 +1,19 @@
using BotSharp.Abstraction.Conversations;
using Microsoft.Extensions.DependencyInjection;
namespace BotSharp.Abstraction.Hooks;
public static class HookProvider
{
public static List<T> GetHooks<T>(this IServiceProvider services, string agentId) where T : IHookBase
{
var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
return hooks.ToList();
}
public static List<T> GetHooksOrderByPriority<T>(this IServiceProvider services, string agentId) where T: IConversationHook
{
var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
return hooks.OrderBy(p => p.Priority).ToList();
}
}

10
src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Hooks;
public interface IHookBase
{
/// <summary>
/// Agent Id
/// </summary>
string SelfId => string.Empty;
bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId;
}

4
src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs
View file

@ -1,10 +1,10 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Instructs.Models;
namespace BotSharp.Abstraction.Instructs;
public interface IInstructHook
public interface IInstructHook : IHookBase
{
string SelfId { get; }
Task BeforeCompletion(Agent agent, RoleDialogModel message);
Task AfterCompletion(Agent agent, InstructResult result);
Task OnResponseGenerated(InstructResponseModel response);

3
src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs
View file

@ -1,11 +1,12 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Abstraction.Loggers;
/// <summary>
/// Model content generating hook, it can be used for logging, metrics and tracing.
/// </summary>
public interface IContentGeneratingHook
public interface IContentGeneratingHook : IHookBase
{
/// <summary>
/// Before content generating.

2
src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs
View file

@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services;
public interface IMcpService
{
IEnumerable<McpServerOptionModel> GetServerConfigs() => [];
Task<IEnumerable<McpServerOptionModel>> GetServerConfigsAsync() => Task.FromResult<IEnumerable<McpServerOptionModel>>([]);
}

19
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -10,14 +10,15 @@ public interface IRealTimeCompletion
Task Connect(
RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onInterruptionDetected);
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected);
Task AppenAudioBuffer(string message);
Task AppenAudioBuffer(ArraySegment<byte> data, int length);
@ -29,6 +30,4 @@ public interface IRealTimeCompletion
Task RemoveConversationItem(string itemId);
Task TriggerModelInference(string? instructions = null);
Task CancelModelResponse();
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
}

4
src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Abstraction.Planning;
public interface IPlanningHook
public interface IPlanningHook : IHookBase
{
Task<string> GetSummaryAdditionalRequirements(string planner, RoleDialogModel message)
=> Task.FromResult(string.Empty);

3
src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs
View file

@ -1,8 +1,9 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Abstraction.Realtime;
public interface IRealtimeHook
public interface IRealtimeHook : IHookBase
{
Task OnModelReady(Agent agent, IRealTimeCompletion completer);
string[] OnModelTranscriptPrompt(Agent agent);

7
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -12,7 +12,12 @@ public class RealtimeModelSettings
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public int ModelResponseTimeout { get; set; } = 30;
public int ModelResponseTimeoutSeconds { get; set; } = 30;
/// <summary>
/// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done"
/// </summary>
public string? ModelResponseTimeoutEndEvent { get; set; }
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

7
src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Abstraction.Routing.Executor;
public interface IFunctionExecutor
{
public Task<bool> ExecuteAsync(RoleDialogModel message);
public Task<string> GetIndicatorAsync(RoleDialogModel message);
}

3
src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs
View file

@ -1,8 +1,9 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Abstraction.Routing;
public interface IRoutingHook
public interface IRoutingHook : IHookBase
{
/// <summary>
/// Routing instruction is received from Router

2
src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs
View file

@ -125,6 +125,6 @@ public class CrontabService : ICrontabService, ITaskFeeder
await hook.OnCronTriggered(item);
await hook.OnTaskExecuted(item);
}
});
}, item.AgentId);
}
}

18
src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj
View file

@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ModelContextProtocol" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
</Project>

67
src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs
View file

@ -1,67 +0,0 @@
using BotSharp.Core.MCP.Functions;
using BotSharp.Core.MCP.Hooks;
using BotSharp.Core.MCP.Managers;
using BotSharp.Core.MCP.Services;
using BotSharp.Core.MCP.Settings;
using Microsoft.Extensions.Configuration;
using ModelContextProtocol.Client;
namespace BotSharp.Core.MCP;
public static class BotSharpMcpExtensions
{
/// <summary>
/// Add mcp
/// </summary>
/// <param name="services"></param>
/// <param name="config"></param>
/// <returns></returns>
public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config)
{
services.AddScoped<IMcpService, McpService>();
var settings = config.GetSection("MCP").Get<McpSettings>();
services.AddScoped(provider => settings);
if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty())
{
var clientManager = new McpClientManager(settings);
services.AddSingleton(clientManager);
foreach (var server in settings.McpServerConfigs)
{
RegisterFunctionCall(services, server, clientManager)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
}
// Register hooks
services.AddScoped<IAgentHook, McpToolAgentHook>();
}
return services;
}
private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfigModel server, McpClientManager clientManager)
{
try
{
var client = await clientManager.GetMcpClientAsync(server.Id);
var tools = await client.ListToolsAsync();
foreach (var tool in tools)
{
services.AddScoped(provider => tool);
services.AddScoped<IFunctionCallback>(provider =>
{
var funcTool = new McpToolAdapter(provider, server.Name, tool, clientManager);
return funcTool;
});
}
}
catch (Exception ex)
{
Console.WriteLine($"Error when registering {server?.Name} MCP tools. {ex.Message}\r\n{ex.InnerException}");
}
}
}

54
src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs
View file

@ -1,54 +0,0 @@
using BotSharp.Core.MCP.Settings;
using ModelContextProtocol.Client;
using ModelContextProtocol.Protocol.Transport;
namespace BotSharp.Core.MCP.Managers;
public class McpClientManager : IDisposable
{
private readonly McpSettings _mcpSettings;
public McpClientManager(McpSettings mcpSettings)
{
_mcpSettings = mcpSettings;
}
public async Task<IMcpClient> GetMcpClientAsync(string serverId)
{
var config = _mcpSettings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault();
IClientTransport transport;
if (config.SseConfig != null)
{
transport = new SseClientTransport(new SseClientTransportOptions
{
Name = config.Name,
Endpoint = new Uri(config.SseConfig.EndPoint),
AdditionalHeaders = config.SseConfig.AdditionalHeaders,
ConnectionTimeout = config.SseConfig.ConnectionTimeout
});
}
else if (config.StdioConfig != null)
{
transport = new StdioClientTransport(new StdioClientTransportOptions
{
Name = config.Name,
Command = config.StdioConfig.Command,
Arguments = config.StdioConfig.Arguments,
EnvironmentVariables = config.StdioConfig.EnvironmentVariables,
ShutdownTimeout = config.StdioConfig.ShutdownTimeout
});
}
else
{
throw new ArgumentNullException("Invalid MCP server configuration!");
}
return await McpClientFactory.CreateAsync(transport, _mcpSettings.McpClientOptions);
}
public void Dispose()
{
}
}

18
src/Infrastructure/BotSharp.Core.MCP/Using.cs
View file

@ -1,18 +0,0 @@
global using System;
global using System.Collections.Generic;
global using System.Linq;
global using System.Threading.Tasks;
global using Microsoft.Extensions.DependencyInjection;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Agents.Settings;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.MCP.Models;
global using BotSharp.Abstraction.MCP.Services;

3
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -72,7 +72,8 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
{
return;
}
else if (message.StopCompletion)
if (message.StopCompletion)
{
await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
}

22
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Options;
using BotSharp.Core.Infrastructures;
@ -23,7 +24,6 @@ public class RealtimeHub : IRealtimeHub
public async Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null)
{
var hookProvider = _services.GetService<ConversationHookProvider>();
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(_conn.ConversationId, []);
var conversation = await convService.GetConversation(_conn.ConversationId);
@ -49,7 +49,8 @@ public class RealtimeHub : IRealtimeHub
// Not TriggerModelInference, waiting for user utter.
var instruction = await _completer.UpdateSession(_conn, isInit: true);
var data = _conn.OnModelReady();
await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModelReady(agent, _completer));
await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnModelReady(agent, _completer),
agent.Id);
await (init?.Invoke(data) ?? Task.CompletedTask);
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
@ -76,8 +77,8 @@ public class RealtimeHub : IRealtimeHub
{
var data = _conn.OnModelAudioResponseDone();
await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
},
onAudioTranscriptDone: async transcript =>
},
onModelAudioTranscriptDone: async transcript =>
{
},
@ -92,10 +93,13 @@ public class RealtimeHub : IRealtimeHub
if (message.FunctionName == "route_to_agent")
{
var instruction = JsonSerializer.Deserialize<FunctionCallFromLlm>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message));
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message),
agent.Id);
}
await routing.InvokeFunction(message.FunctionName, message);
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
}
else
{
@ -103,7 +107,8 @@ public class RealtimeHub : IRealtimeHub
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
foreach (var hook in hookProvider?.HooksOrderByPriority ?? [])
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(_conn.CurrentAgentId);
foreach (var hook in hooks)
{
hook.SetAgent(agent)
.SetConversation(conversation);
@ -117,14 +122,15 @@ public class RealtimeHub : IRealtimeHub
{
},
onInputAudioTranscriptionCompleted: async message =>
onInputAudioTranscriptionDone: async message =>
{
// append input audio transcript to conversation
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
routing.Context.SetMessageId(_conn.ConversationId, message.MessageId);
foreach (var hook in hookProvider?.HooksOrderByPriority ?? [])
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(_conn.CurrentAgentId);
foreach (var hook in hooks)
{
hook.SetAgent(agent)
.SetConversation(conversation);

39
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Routing.Models;
using System.Collections.Concurrent;
@ -10,29 +11,12 @@ public partial class AgentService
// [SharpCache(10, perInstanceCache: true)]
public async Task<Agent> LoadAgent(string id, bool loadUtility = true)
{
if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString())
{
return null;
}
if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) return null;
var hooks = _services.GetServices<IAgentHook>();
// Before agent is loaded.
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != id)
{
continue;
}
hook.OnAgentLoading(ref id);
}
HookEmitter.Emit<IAgentHook>(_services, hook => hook.OnAgentLoading(ref id), id);
var agent = await GetAgent(id);
if (agent == null)
{
return null;
}
if (agent == null) return null;
await InheritAgent(agent);
OverrideInstructionByChannel(agent);
@ -43,13 +27,7 @@ public partial class AgentService
PopulateState(agent.TemplateDict);
// After agent is loaded
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != id)
{
continue;
}
HookEmitter.Emit<IAgentHook>(_services, hook => {
hook.SetAgent(agent);
if (!string.IsNullOrEmpty(agent.Instruction))
@ -72,13 +50,14 @@ public partial class AgentService
hook.OnAgentUtilityLoaded(agent);
}
if(!agent.McpTools.IsNullOrEmpty())
if (!agent.McpTools.IsNullOrEmpty())
{
hook.OnAgentMcpToolLoaded(agent);
}
hook.OnAgentLoaded(agent);
}
}, id);
_logger.LogInformation($"Loaded agent {agent}.");

5
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs
View file

@ -132,9 +132,8 @@ public partial class AgentService
agent.TemplateDict[TemplateRenderConstant.RENDER_AGENT] = agent;
var content = render.Render(template, agent.TemplateDict);
HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
await hook.OnRenderingTemplate(agent, templateName, content)
).Wait();
HookEmitter.Emit<IContentGeneratingHook>(_services, async hook => await hook.OnRenderingTemplate(agent, templateName, content),
agent.Id).Wait();
return content;
}

8
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -96,6 +96,8 @@
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
<None Remove="data\plugins\config.json" />
<None Remove="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json" />
</ItemGroup>
<ItemGroup>
@ -204,6 +206,11 @@
<Content Include="data\plugins\config.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
@ -221,6 +228,7 @@
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Nanoid" />
<PackageReference Include="Rougamo.Fody" />
<PackageReference Include="ModelContextProtocol" />
<PackageReference Include="System.ClientModel" />
</ItemGroup>

15
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent;
@ -29,7 +30,6 @@ public partial class ConversationService
var dialogs = conv.GetDialogHistory();
var statistics = _services.GetRequiredService<ITokenStatistics>();
var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
RoleDialogModel response = message;
bool stopCompletion = false;
@ -44,7 +44,8 @@ public partial class ConversationService
message.Payload = replyMessage.Payload;
}
foreach (var hook in hookProvider.HooksOrderByPriority)
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(message.CurrentAgentId);
foreach (var hook in hooks)
{
hook.SetAgent(agent)
.SetConversation(conversation);
@ -158,16 +159,14 @@ public partial class ConversationService
// Emit conversation ending hook
if (response.Instruction.ConversationEnd)
{
await HookEmitter.Emit<IConversationHook>(_services, async hook =>
await hook.OnConversationEnding(response)
);
await HookEmitter.Emit<IConversationHook>(_services, async hook => await hook.OnConversationEnding(response),
response.CurrentAgentId);
response.FunctionName = "conversation_end";
}
}
await HookEmitter.Emit<IConversationHook>(_services, async hook =>
await hook.OnResponseGenerated(response)
);
await HookEmitter.Emit<IConversationHook>(_services, async hook => await hook.OnResponseGenerated(response),
response.CurrentAgentId);
await onResponseReceived(response);

5
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Infrastructures.Enums;
namespace BotSharp.Core.Conversations.Services;
@ -31,9 +32,7 @@ public partial class ConversationService
states.CleanStates(excludedStates);
}
var hooks = _services
.GetRequiredService<ConversationHookProvider>()
.HooksOrderByPriority;
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(routingCtx.GetCurrentAgentId());
// Before executing functions
foreach (var hook in hooks)

3
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Models;
namespace BotSharp.Core.Conversations.Services;
@ -116,7 +117,7 @@ public partial class ConversationService : IConversationService
db.CreateNewConversation(record);
var hooks = _services.GetServices<IConversationHook>();
var hooks = _services.GetHooks<IConversationHook>(record.AgentId);
foreach (var hook in hooks)
{

26
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -15,6 +15,7 @@
******************************************************************************/
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.SideCar;
@ -28,6 +29,7 @@ public class ConversationStateService : IConversationStateService
private readonly ILogger _logger;
private readonly IServiceProvider _services;
private readonly IBotSharpRepository _db;
private readonly IRoutingContext _routingContext;
private readonly IConversationSideCar? _sidecar;
private string _conversationId;
/// <summary>
@ -42,10 +44,12 @@ public class ConversationStateService : IConversationStateService
public ConversationStateService(
IServiceProvider services,
IBotSharpRepository db,
IRoutingContext routingContext,
ILogger<ConversationStateService> logger)
{
_services = services;
_db = db;
_routingContext = routingContext;
_logger = logger;
_curStates = new ConversationState();
_historyStates = new ConversationState();
@ -87,7 +91,6 @@ public class ConversationStateService : IConversationStateService
}
_logger.LogDebug($"[STATE] {name} = {value}");
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var isNoChange = ContainsState(name)
&& preValue == currentValue
@ -98,7 +101,7 @@ public class ConversationStateService : IConversationStateService
&& prevLeafNode?.Active == curActive
&& pair?.Readonly == readOnly;
var hooks = _services.GetServices<IConversationHook>();
var hooks = _services.GetHooks<IConversationHook>(_routingContext.GetCurrentAgentId());
if (!ContainsState(name) || preValue != currentValue || prevLeafNode?.ActiveRounds != curActiveRounds)
{
foreach (var hook in hooks)
@ -106,7 +109,7 @@ public class ConversationStateService : IConversationStateService
hook.OnStateChanged(new StateChangeModel
{
ConversationId = _conversationId,
MessageId = routingCtx.MessageId,
MessageId = _routingContext.MessageId,
Name = name,
BeforeValue = preValue,
BeforeActiveRounds = prevLeafNode?.ActiveRounds,
@ -129,7 +132,7 @@ public class ConversationStateService : IConversationStateService
var newValue = new StateValue
{
Data = currentValue,
MessageId = routingCtx.MessageId,
MessageId = _routingContext.MessageId,
Active = curActive,
ActiveRounds = curActiveRounds,
DataType = valueType,
@ -171,8 +174,7 @@ public class ConversationStateService : IConversationStateService
return endNodes;
}
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var curMsgId = routingCtx.MessageId;
var curMsgId = _routingContext.MessageId;
var dialogs = _db.GetConversationDialogs(conversationId);
var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User)
.GroupBy(x => x.MetaData?.MessageId)
@ -225,7 +227,7 @@ public class ConversationStateService : IConversationStateService
}
_logger.LogInformation($"Loaded conversation states: {conversationId}");
var hooks = _services.GetServices<IConversationHook>();
var hooks = _services.GetHooks<IConversationHook>(_routingContext.GetCurrentAgentId());
foreach (var hook in hooks)
{
hook.OnStateLoaded(_curStates).Wait();
@ -277,7 +279,6 @@ public class ConversationStateService : IConversationStateService
{
if (!ContainsState(name)) return false;
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var value = _curStates[name];
var leafNode = value?.Values?.LastOrDefault();
if (value == null || !value.Versioning || leafNode == null) return false;
@ -285,7 +286,7 @@ public class ConversationStateService : IConversationStateService
_curStates[name].Values.Add(new StateValue
{
Data = leafNode.Data,
MessageId = routingCtx.MessageId,
MessageId = _routingContext.MessageId,
Active = false,
ActiveRounds = leafNode.ActiveRounds,
DataType = leafNode.DataType,
@ -293,13 +294,13 @@ public class ConversationStateService : IConversationStateService
UpdateTime = DateTime.UtcNow
});
var hooks = _services.GetServices<IConversationHook>();
var hooks = _services.GetHooks<IConversationHook>(_routingContext.GetCurrentAgentId());
foreach (var hook in hooks)
{
hook.OnStateChanged(new StateChangeModel
{
ConversationId = _conversationId,
MessageId = routingCtx.MessageId,
MessageId = _routingContext.MessageId,
Name = name,
BeforeValue = leafNode.Data,
BeforeActiveRounds = leafNode.ActiveRounds,
@ -316,8 +317,7 @@ public class ConversationStateService : IConversationStateService
public void CleanStates(params string[] excludedStates)
{
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var curMsgId = routingCtx.MessageId;
var curMsgId = _routingContext.MessageId;
var utcNow = DateTime.UtcNow;
foreach (var key in _curStates.Keys)

7
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file

@ -40,6 +40,8 @@ public class ConversationStorage : IConversationStorage
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
FunctionName = dialog.FunctionName,
FunctionArgs = dialog.FunctionArgs,
ToolCallId = dialog.ToolCallId,
CreatedTime = dialog.CreatedAt
};
@ -109,7 +111,6 @@ public class ConversationStorage : IConversationStorage
var currentAgentId = meta.AgentId;
var messageId = meta.MessageId;
var messageType = meta.MessageType;
var function = meta.FunctionName;
var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
var createdAt = meta.CreatedTime;
var richContent = !string.IsNullOrEmpty(dialog.RichContent) ?
@ -124,7 +125,9 @@ public class ConversationStorage : IConversationStorage
MessageType = messageType,
CreatedAt = createdAt,
SenderId = senderId,
FunctionName = function,
FunctionName = meta.FunctionName,
FunctionArgs = meta.FunctionArgs,
ToolCallId = meta.ToolCallId,
RichContent = richContent,
SecondaryContent = secondaryContent,
SecondaryRichContent = secondaryRichContent,

23
src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs Normal file
View file

@ -0,0 +1,23 @@
using BotSharp.Abstraction.Functions;
namespace BotSharp.Core.Demo.Functions;
public class GetWeatherFn : IFunctionCallback
{
private readonly IServiceProvider _services;
public GetWeatherFn(IServiceProvider services)
{
_services = services;
}
public string Name => "get_weather";
public string Indication => "Querying weather";
public async Task<bool> Execute(RoleDialogModel message)
{
message.Content = $"It is a sunny day!";
//message.StopCompletion = true;
return true;
}
}

61
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using System.IO;
using BotSharp.Abstraction.Infrastructures;
namespace BotSharp.Core.Files.Services;
@ -24,14 +25,7 @@ public partial class FileInstructService
}
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
@ -41,8 +35,7 @@ public partial class FileInstructService
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
}), innerAgentId);
return message.Content;
}
@ -59,14 +52,7 @@ public partial class FileInstructService
Instruction = instruction
}, new RoleDialogModel(AgentRole.User, instruction ?? text));
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
@ -76,8 +62,7 @@ public partial class FileInstructService
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
}), innerAgentId);
return message;
}
@ -104,14 +89,7 @@ public partial class FileInstructService
stream.Close();
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
@ -119,8 +97,7 @@ public partial class FileInstructService
Model = completion.Model,
UserMessage = string.Empty,
CompletionText = message.Content
});
}
}), innerAgentId);
return message;
}
@ -149,14 +126,7 @@ public partial class FileInstructService
stream.Close();
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
@ -166,8 +136,7 @@ public partial class FileInstructService
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
}), innerAgentId);
return message;
}
@ -205,14 +174,7 @@ public partial class FileInstructService
imageStream.Close();
maskStream.Close();
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
@ -222,8 +184,7 @@ public partial class FileInstructService
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
}), innerAgentId);
return message;
}

13
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Files.Converters;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Infrastructures;
namespace BotSharp.Core.Files.Services;
@ -42,14 +43,7 @@ public partial class FileInstructService
}
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = innerAgentId,
@ -59,8 +53,7 @@ public partial class FileInstructService
UserMessage = text,
SystemInstruction = instruction,
CompletionText = message.Content
});
}
}), innerAgentId);
return message.Content;
}

9
src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs
View file

@ -1,14 +1,15 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Infrastructures;
namespace BotSharp.Core.Infrastructures;
public static class HookEmitter
{
public static HookEmittedResult Emit<T>(IServiceProvider services, Action<T> action, HookEmitOption<T>? option = null)
public static HookEmittedResult Emit<T>(IServiceProvider services, Action<T> action, string agentId, HookEmitOption<T>? option = null) where T : IHookBase
{
var logger = services.GetRequiredService<ILogger<T>>();
var result = new HookEmittedResult();
var hooks = services.GetServices<T>();
var hooks = services.GetHooks<T>(agentId);
option = option ?? new();
foreach (var hook in hooks)
@ -35,11 +36,11 @@ public static class HookEmitter
return result;
}
public static async Task<HookEmittedResult> Emit<T>(IServiceProvider services, Func<T, Task> action, HookEmitOption<T>? option = null)
public static async Task<HookEmittedResult> Emit<T>(IServiceProvider services, Func<T, Task> action, string agentId, HookEmitOption<T>? option = null) where T : IHookBase
{
var logger = services.GetRequiredService<ILogger<T>>();
var result = new HookEmittedResult();
var hooks = services.GetServices<T>();
var hooks = services.GetHooks<T>(agentId);
option = option ?? new();
foreach (var hook in hooks)

4
src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Realtime.Models.Session;
using System.Buffers;
using System.ClientModel;
using System.Net.WebSockets;
@ -44,6 +43,9 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResul
if (receivedResult.CloseStatus.HasValue)
{
#if DEBUG
Console.WriteLine($"Websocket close: {receivedResult.CloseStatus} {receivedResult.CloseStatusDescription}");
#endif
Current = null;
return false;
}

13
src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
@ -60,14 +61,7 @@ public class ExecuteTemplateFn : IFunctionCallback
new(AgentRole.User, text)
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agent.Id)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = agent.Id,
@ -76,8 +70,7 @@ public class ExecuteTemplateFn : IFunctionCallback
Model = completion.Model,
UserMessage = text,
CompletionText = response.Content
});
}
}), agent.Id);
return response.Content;
}

13
src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.MLTasks;
@ -23,14 +24,9 @@ public partial class InstructService
}
// Trigger before completion hooks
var hooks = _services.GetServices<IInstructHook>();
var hooks = _services.GetHooks<IInstructHook>(agentId);
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
{
continue;
}
await hook.BeforeCompletion(agent, message);
// Interrupted by hook
@ -99,11 +95,6 @@ public partial class InstructService
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
{
continue;
}
await hook.AfterCompletion(agent, response);
await hook.OnResponseGenerated(new InstructResponseModel
{

31
src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs Normal file
View file

@ -0,0 +1,31 @@
using BotSharp.Core.MCP.Hooks;
using BotSharp.Core.MCP.Managers;
using BotSharp.Core.MCP.Services;
using BotSharp.Core.MCP.Settings;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Core.MCP;
public static class BotSharpMcpExtensions
{
/// <summary>
/// Add mcp
/// </summary>
/// <param name="services"></param>
/// <param name="config"></param>
/// <returns></returns>
public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config)
{
var settings = config.GetSection("MCP").Get<McpSettings>();
services.AddScoped(provider => settings);
services.AddScoped<IMcpService, McpService>();
if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty())
{
services.AddScoped<McpClientManager>();
services.AddScoped<IAgentHook, McpToolAgentHook>();
}
return services;
}
}

23
src/Infrastructure/BotSharp.Core.MCP/Helpers/AiFunctionHelper.cs → src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs
View file

@ -1,19 +1,28 @@
using System.Text.Json;
using ModelContextProtocol.Client;
namespace BotSharp.Core.MCP.Helpers;
internal static class AiFunctionHelper
{
public static FunctionDef MapToFunctionDef(McpClientTool tool)
public static FunctionDef? MapToFunctionDef(McpClientTool tool)
{
if (tool == null)
{
throw new ArgumentNullException(nameof(tool));
return null;
}
var properties = tool.JsonSchema.GetProperty("properties");
var required = tool.JsonSchema.GetProperty("required");
var properties = "{}";
var required = "[]";
if (tool.JsonSchema.TryGetProperty("properties", out var p))
{
properties = p.GetRawText();
}
if (tool.JsonSchema.TryGetProperty("required", out var r))
{
required = r.GetRawText();
}
var funDef = new FunctionDef
{
@ -23,8 +32,8 @@ internal static class AiFunctionHelper
Parameters = new FunctionParametersDef
{
Type = "object",
Properties = JsonDocument.Parse(properties.GetRawText()),
Required = JsonSerializer.Deserialize<List<string>>(required.GetRawText())
Properties = JsonDocument.Parse(properties),
Required = JsonSerializer.Deserialize<List<string>>(required) ?? []
}
};

22
src/Infrastructure/BotSharp.Core.MCP/Hooks/MCPToolAgentHook.cs → src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs
View file

@ -41,18 +41,26 @@ public class McpToolAgentHook : AgentHookBase
return functionDefs;
}
var mcpClientManager = _services.GetRequiredService<McpClientManager>();
var mcps = agent.McpTools.Where(x => !x.Disabled);
var mcpClientManager = _services.GetService<McpClientManager>();
if (mcpClientManager == null)
{
return functionDefs;
}
var mcps = agent.McpTools?.Where(x => !x.Disabled) ?? [];
foreach (var item in mcps)
{
var mcpClient = await mcpClientManager.GetMcpClientAsync(item.ServerId);
if (mcpClient != null)
if (mcpClient == null) continue;
var tools = await mcpClient.ListToolsAsync();
var toolNames = item.Functions.Select(x => x.Name).ToList();
var targetTools = tools.Where(x => toolNames.Contains(x.Name, StringComparer.OrdinalIgnoreCase));
foreach (var tool in targetTools)
{
var tools = await mcpClient.ListToolsAsync();
var toolnames = item.Functions.Select(x => x.Name).ToList();
foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase)))
var funDef = AiFunctionHelper.MapToFunctionDef(tool);
if (funDef != null)
{
var funDef = AiFunctionHelper.MapToFunctionDef(tool);
functionDefs.Add(funDef);
}
}

72
src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs Normal file
View file

@ -0,0 +1,72 @@
using BotSharp.Core.MCP.Settings;
using ModelContextProtocol.Client;
using ModelContextProtocol.Protocol.Transport;
namespace BotSharp.Core.MCP.Managers;
public class McpClientManager : IDisposable
{
private readonly IServiceProvider _services;
private readonly ILogger<McpClientManager> _logger;
public McpClientManager(
IServiceProvider services,
ILogger<McpClientManager> logger)
{
_services = services;
_logger = logger;
}
public async Task<IMcpClient?> GetMcpClientAsync(string serverId)
{
try
{
var settings = _services.GetRequiredService<McpSettings>();
var config = settings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault();
if (config == null)
{
return null;
}
IClientTransport? transport = null;
if (config.SseConfig != null)
{
transport = new SseClientTransport(new SseClientTransportOptions
{
Name = config.Name,
Endpoint = new Uri(config.SseConfig.EndPoint),
AdditionalHeaders = config.SseConfig.AdditionalHeaders,
ConnectionTimeout = config.SseConfig.ConnectionTimeout
});
}
else if (config.StdioConfig != null)
{
transport = new StdioClientTransport(new StdioClientTransportOptions
{
Name = config.Name,
Command = config.StdioConfig.Command,
Arguments = config.StdioConfig.Arguments,
EnvironmentVariables = config.StdioConfig.EnvironmentVariables,
ShutdownTimeout = config.StdioConfig.ShutdownTimeout
});
}
if (transport == null)
{
return null;
}
return await McpClientFactory.CreateAsync(transport, settings.McpClientOptions);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when loading mcp client {serverId}");
return null;
}
}
public void Dispose()
{
}
}

16
src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs → src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.MCP.Managers;
using BotSharp.Core.MCP.Settings;
using Microsoft.Extensions.Logging;
using ModelContextProtocol.Client;
namespace BotSharp.Core.MCP.Services;
@ -16,23 +17,26 @@ public class McpService : IMcpService
_logger = logger;
}
public IEnumerable<McpServerOptionModel> GetServerConfigs()
public async Task<IEnumerable<McpServerOptionModel>> GetServerConfigsAsync()
{
var clientManager = _services.GetService<McpClientManager>();
if (clientManager == null) return [];
var options = new List<McpServerOptionModel>();
var settings = _services.GetRequiredService<McpSettings>();
var configs = settings?.McpServerConfigs ?? [];
foreach (var config in configs)
{
var tools = _services.GetServices<IFunctionCallback>()
.Where(x => x.Provider == config.Name)
.Select(x => x.Name);
var client = await clientManager.GetMcpClientAsync(config.Id);
if (client == null) continue;
var tools = await client.ListToolsAsync();
options.Add(new McpServerOptionModel
{
Id = config.Id,
Name = config.Name,
Tools = tools
Tools = tools.Select(x => x.Name)
});
}

1
src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs → src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs
View file

@ -7,5 +7,4 @@ public class McpSettings
public bool Enabled { get; set; } = true;
public McpClientOptions McpClientOptions { get; set; }
public List<McpServerConfigModel> McpServerConfigs { get; set; } = [];
}

37
src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs Normal file
View file

@ -0,0 +1,37 @@
using BotSharp.Abstraction.Routing.Executor;
using BotSharp.Abstraction.Templating;
namespace BotSharp.Core.Routing.Executor;
public class DummyFunctionExecutor: IFunctionExecutor
{
private readonly IServiceProvider _services;
private readonly FunctionDef _functionDef;
public DummyFunctionExecutor(IServiceProvider services, FunctionDef functionDef)
{
_services = services;
_functionDef = functionDef;
}
public async Task<bool> ExecuteAsync(RoleDialogModel message)
{
var render = _services.GetRequiredService<ITemplateRender>();
var state = _services.GetRequiredService<IConversationStateService>();
var dict = new Dictionary<string, object>();
foreach (var item in state.GetStates())
{
dict[item.Key] = item.Value;
}
var text = render.Render(_functionDef.Output!, dict);
message.Content = text;
return true;
}
public async Task<string> GetIndicatorAsync(RoleDialogModel message)
{
return "Running";
}
}

24
src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs Normal file
View file

@ -0,0 +1,24 @@
using BotSharp.Abstraction.Routing.Executor;
using BotSharp.Abstraction.Functions;
namespace BotSharp.Core.Routing.Executor;
public class FunctionCallbackExecutor : IFunctionExecutor
{
private readonly IFunctionCallback _functionCallback;
public FunctionCallbackExecutor(IFunctionCallback functionCallback)
{
_functionCallback = functionCallback;
}
public async Task<bool> ExecuteAsync(RoleDialogModel message)
{
return await _functionCallback.Execute(message);
}
public async Task<string> GetIndicatorAsync(RoleDialogModel message)
{
return await _functionCallback.GetIndication(message);
}
}

31
src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs Normal file
View file

@ -0,0 +1,31 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Routing.Executor;
namespace BotSharp.Core.Routing.Executor;
internal class FunctionExecutorFactory
{
public static IFunctionExecutor? Create(IServiceProvider services, string functionName, Agent agent)
{
var functionCall = services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == functionName);
if (functionCall != null)
{
return new FunctionCallbackExecutor(functionCall);
}
var functions = (agent?.Functions ?? []).Concat(agent?.SecondaryFunctions ?? []);
var funcDef = functions.FirstOrDefault(x => x.Name == functionName);
if (!string.IsNullOrWhiteSpace(funcDef?.Output))
{
return new DummyFunctionExecutor(services, funcDef);
}
var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funcDef?.Name))?.FirstOrDefault()?.ServerId;
if (!string.IsNullOrWhiteSpace(mcpServerId))
{
return new McpToolExecutor(services, mcpServerId, functionName);
}
return null;
}
}

48
src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs → src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs
View file

@ -1,46 +1,34 @@
using System.Text.Json;
using BotSharp.Abstraction.Routing.Executor;
using BotSharp.Core.MCP.Managers;
using ModelContextProtocol.Client;
namespace BotSharp.Core.MCP.Functions;
namespace BotSharp.Core.Routing.Executor;
public class McpToolAdapter : IFunctionCallback
public class McpToolExecutor: IFunctionExecutor
{
private readonly string _provider;
private readonly McpClientTool _tool;
private readonly McpClientManager _clientManager;
private readonly IServiceProvider _services;
private readonly string _mcpServerId;
private readonly string _functionName;
public McpToolAdapter(
IServiceProvider services,
string serverName,
McpClientTool tool,
McpClientManager client)
{
_services = services ?? throw new ArgumentNullException(nameof(services));
_tool = tool ?? throw new ArgumentNullException(nameof(tool));
_clientManager = client ?? throw new ArgumentNullException(nameof(client));
_provider = serverName;
public McpToolExecutor(IServiceProvider services, string mcpServerId, string functionName)
{
_services = services;
_mcpServerId = mcpServerId;
_functionName = functionName;
}
public string Provider => _provider;
public string Name => _tool.Name;
public async Task<bool> Execute(RoleDialogModel message)
public async Task<bool> ExecuteAsync(RoleDialogModel message)
{
try
{
// Convert arguments to dictionary format expected by mcpdotnet
Dictionary<string, object> argDict = JsonToDictionary(message.FunctionArgs);
var currentAgentId = message.CurrentAgentId;
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(currentAgentId);
var serverId = agent.McpTools.Where(t => t.Functions.Any(f => f.Name == Name)).FirstOrDefault().ServerId;
var client = await _clientManager.GetMcpClientAsync(serverId);
var clientManager = _services.GetRequiredService<McpClientManager>();
var client = await clientManager.GetMcpClientAsync(_mcpServerId);
// Call the tool through mcpdotnet
var result = await client.CallToolAsync(_tool.Name, !argDict.IsNullOrEmpty() ? argDict : []);
var result = await client.CallToolAsync(_functionName, !argDict.IsNullOrEmpty() ? argDict : []);
// Extract the text content from the result
var json = string.Join("\n", result.Content.Where(c => c.Type == "text").Select(c => c.Text));
@ -51,11 +39,17 @@ public class McpToolAdapter : IFunctionCallback
}
catch (Exception ex)
{
message.Content = $"Error when calling tool {Name} of MCP server {Provider}. {ex.Message}";
message.Content = $"Error when calling tool {_functionName} of MCP server {_mcpServerId}. {ex.Message}";
return false;
}
}
public async Task<string> GetIndicatorAsync(RoleDialogModel message)
{
return message.Indication ?? string.Empty;
}
private static Dictionary<string, object> JsonToDictionary(string? json)
{
if (string.IsNullOrEmpty(json))

5
src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Core.Routing.Functions;
@ -15,9 +16,7 @@ public class HumanInterventionNeededFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var hooks = _services
.GetRequiredService<ConversationHookProvider>()
.HooksOrderByPriority;
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(message.CurrentAgentId);
foreach (var hook in hooks)
{

5
src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs
View file

@ -25,9 +25,8 @@ public class InstructExecutor : IExecutor
{
inst.OriginalAgent = goalAgent;
// Emit hook
await HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnRoutingInstructionRevised(inst, message)
);
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionRevised(inst, message),
message.CurrentAgentId);
}
message.FunctionArgs = JsonSerializer.Serialize(inst);

20
src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs
View file

@ -99,9 +99,8 @@ public class RoutingContext : IRoutingContext
var preAgentId = _stack.Count == 0 ? agentId : _stack.Peek();
_stack.Push(agentId);
HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason)
).Wait();
HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason),
agentId).Wait();
UpdateLazyRoutingAgent(updateLazyRouting);
}
@ -120,9 +119,8 @@ public class RoutingContext : IRoutingContext
var agentId = _stack.Pop();
var currentAgentId = GetCurrentAgentId();
HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnAgentDequeued(agentId, currentAgentId, reason: reason)
).Wait();
HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnAgentDequeued(agentId, currentAgentId, reason: reason),
agentId).Wait();
if (string.IsNullOrEmpty(currentAgentId))
{
@ -203,9 +201,8 @@ public class RoutingContext : IRoutingContext
_stack.Pop();
_stack.Push(agentId);
HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason)
).Wait();
HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason),
agentId).Wait();
}
UpdateLazyRoutingAgent(updateLazyRouting);
@ -220,9 +217,8 @@ public class RoutingContext : IRoutingContext
var agentId = GetCurrentAgentId();
_stack.Clear();
HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnAgentQueueEmptied(agentId, reason: reason)
).Wait();
HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnAgentQueueEmptied(agentId, reason: reason),
agentId).Wait();
}
public void SetMessageId(string conversationId, string messageId)

5
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs
View file

@ -50,9 +50,8 @@ public partial class RoutingService
int loopCount = 1;
while (true)
{
await HookEmitter.Emit<IRoutingHook>(_services, async hook =>
await hook.OnRoutingInstructionReceived(inst, message)
);
await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRoutingInstructionReceived(inst, message),
agent.Id);
// Save states
states.SaveStateByArgs(inst.Arguments);

93
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Templating;
using BotSharp.Abstraction.Hooks;
using BotSharp.Core.Routing.Executor;
namespace BotSharp.Core.Routing;
@ -7,50 +7,32 @@ public partial class RoutingService
{
public async Task<bool> InvokeFunction(string name, RoleDialogModel message)
{
var function = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == name);
var currentAgentId = message.CurrentAgentId;
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(currentAgentId);
var isFillDummyContent = false;
var dummyFuncResponse = string.Empty;
if (function == null)
var funcExecutor = FunctionExecutorFactory.Create(_services, name, agent);
if (funcExecutor == null)
{
dummyFuncResponse = await GetDummyFunctionOutput(name, message);
isFillDummyContent = !string.IsNullOrEmpty(dummyFuncResponse);
if (!isFillDummyContent)
{
message.StopCompletion = true;
message.Content = $"Can't find function implementation of {name}.";
_logger.LogError(message.Content);
return false;
}
message.StopCompletion = true;
message.Content = $"Can't find function implementation of {name}.";
_logger.LogError(message.Content);
return false;
}
// Clone message
var clonedMessage = RoleDialogModel.From(message);
clonedMessage.FunctionName = name;
var hooks = _services
.GetRequiredService<ConversationHookProvider>()
.HooksOrderByPriority;
var progressService = _services.GetService<IConversationProgressService>();
// Before executing functions
if (!isFillDummyContent)
{
clonedMessage.Indication = await function.GetIndication(message);
}
else
{
clonedMessage.Indication = "Running";
}
clonedMessage.Indication = await funcExecutor.GetIndicatorAsync(message);
if (progressService?.OnFunctionExecuting != null)
{
await progressService.OnFunctionExecuting(clonedMessage);
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId);
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(clonedMessage.CurrentAgentId);
foreach (var hook in hooks)
{
hook.SetAgent(agent);
@ -61,19 +43,7 @@ public partial class RoutingService
try
{
if (clonedMessage.Handled)
{
clonedMessage.Content = clonedMessage.Content;
}
else if (!isFillDummyContent)
{
result = await function.Execute(clonedMessage);
}
else
{
clonedMessage.Content = dummyFuncResponse;
result = true;
}
result = await funcExecutor.ExecuteAsync(clonedMessage);
// After functions have been executed
foreach (var hook in hooks)
@ -111,41 +81,6 @@ public partial class RoutingService
message.Content = JsonSerializer.Serialize(message.Data);
}
// Save to Storage as well
/*if (!message.StopCompletion && message.FunctionName != "route_to_agent")
{
var storage = _services.GetRequiredService<IConversationStorage>();
storage.Append(Context.ConversationId, message);
}*/
return result;
}
private async Task<string?> GetDummyFunctionOutput(string functionName, RoleDialogModel message)
{
if (string.IsNullOrEmpty(message.CurrentAgentId))
{
return null;
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(message.CurrentAgentId);
var found = agent?.Functions?.FirstOrDefault(x => x.Name == functionName);
if (string.IsNullOrWhiteSpace(found?.Output))
{
return null;
}
var render = _services.GetRequiredService<ITemplateRender>();
var state = _services.GetRequiredService<IConversationStateService>();
var dict = new Dictionary<string, object>();
foreach (var item in state.GetStates())
{
dict[item.Key] = item.Value;
}
var text = render.Render(found.Output, dict);
return text;
}
}

4
src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs
View file

@ -55,7 +55,7 @@ public class BotSharpRealtimeSession : IDisposable
};
}
public async Task SendEvent(string message)
public async Task SendEventAsync(string message)
{
if (_websocket.State == WebSocketState.Open)
{
@ -64,7 +64,7 @@ public class BotSharpRealtimeSession : IDisposable
}
}
public async Task Disconnect()
public async Task DisconnectAsync()
{
if (_websocket.State == WebSocketState.Open)
{

14
src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs
View file

@ -22,14 +22,17 @@ public class LlmRealtimeSession : IDisposable
_sessionOptions = sessionOptions;
}
public async Task ConnectAsync(Uri uri, Dictionary<string, string> headers, CancellationToken cancellationToken = default)
public async Task ConnectAsync(Uri uri, Dictionary<string, string>? headers = null, CancellationToken cancellationToken = default)
{
_webSocket?.Dispose();
_webSocket = new ClientWebSocket();
foreach (var header in headers)
if (!headers.IsNullOrEmpty())
{
_webSocket.Options.SetRequestHeader(header.Key, header.Value);
foreach (var header in headers)
{
_webSocket.Options.SetRequestHeader(header.Key, header.Value);
}
}
await _webSocket.ConnectAsync(uri, cancellationToken);
@ -68,7 +71,7 @@ public class LlmRealtimeSession : IDisposable
};
}
public async Task SendEventToModel(object message)
public async Task SendEventToModelAsync(object message)
{
if (_webSocket.State != WebSocketState.Open)
{
@ -93,7 +96,7 @@ public class LlmRealtimeSession : IDisposable
}
}
public async Task Disconnect()
public async Task DisconnectAsync()
{
if (_webSocket.State == WebSocketState.Open)
{
@ -103,6 +106,7 @@ public class LlmRealtimeSession : IDisposable
public void Dispose()
{
_clientEventSemaphore?.Dispose();
_webSocket?.Dispose();
}
}

1
src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Statistics.Settings;
namespace BotSharp.Core.Statistics.Services;

74
src/Infrastructure/BotSharp.Core/Using.cs
View file

@ -1,48 +1,44 @@
global using System;
global using System.Collections.Generic;
global using System.Text;
global using System.Threading.Tasks;
global using System.Linq;
global using System.Text.Json;
global using System.Net.Mime;
global using System.Net.Http;
global using System.Threading;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
global using EntityFrameworkCore.BootKit;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Crontab.Models;
global using BotSharp.Abstraction.Users;
global using BotSharp.Abstraction.Roles;
global using BotSharp.Abstraction.Roles.Models;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Agents.Settings;
global using BotSharp.Abstraction.Conversations.Settings;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Agents.Settings;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Conversations.Settings;
global using BotSharp.Abstraction.Crontab.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Infrastructures.Events;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.Abstraction.Loggers.Services;
global using BotSharp.Abstraction.MCP.Models;
global using BotSharp.Abstraction.MCP.Services;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Repositories.Filters;
global using BotSharp.Abstraction.Translation;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Translation.Attributes;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.Abstraction.Roles;
global using BotSharp.Abstraction.Roles.Models;
global using BotSharp.Abstraction.Routing;
global using BotSharp.Abstraction.SideCar.Attributes;
global using BotSharp.Abstraction.Statistics.Models;
global using BotSharp.Abstraction.Statistics.Enums;
global using BotSharp.Abstraction.Statistics.Models;
global using BotSharp.Abstraction.Statistics.Services;
global using BotSharp.Abstraction.Translation;
global using BotSharp.Abstraction.Translation.Attributes;
global using BotSharp.Abstraction.Users;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Loggers.Services;
global using BotSharp.Abstraction.Infrastructures.Events;
global using BotSharp.Abstraction.Templating.Constants;
global using BotSharp.Abstraction.Realtime.Models.Session;
global using BotSharp.Core.Agents.Services;
global using BotSharp.Core.Conversations.Services;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Core.Loggers.Services;
global using BotSharp.Core.Repository;
global using BotSharp.Core.Routing;
global using BotSharp.Core.Agents.Services;
@ -51,4 +47,16 @@ global using BotSharp.Core.Infrastructures;
global using BotSharp.Core.Infrastructures.Websocket;
global using BotSharp.Core.Users.Services;
global using BotSharp.Core.Statistics.Services;
global using BotSharp.Core.Loggers.Services;
global using BotSharp.Core.Users.Services;
global using EntityFrameworkCore.BootKit;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
global using System;
global using System.Collections.Generic;
global using System.Linq;
global using System.Net.Http;
global using System.Net.Mime;
global using System.Text;
global using System.Text.Json;
global using System.Threading;
global using System.Threading.Tasks;

14
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json Normal file
View file

@ -0,0 +1,14 @@
{
"name": "get_weather",
"description": "Get weather information for user.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "The city where the user wants to get weather information."
}
},
"required": [ "city" ]
}
}

62
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -143,55 +143,29 @@ public class ConversationController : ControllerBase
{
var service = _services.GetRequiredService<IConversationService>();
var userService = _services.GetRequiredService<IUserService>();
var settings = _services.GetRequiredService<PluginSettings>();
var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
if (user == null)
{
return null;
}
var filter = new ConversationFilter
{
Id = conversationId,
UserId = !isAdmin ? user.Id : null,
UserId = !isAdmin ? user?.Id : null,
IsLoadLatestStates = isLoadStates
};
var conversations = await service.GetConversations(filter);
if (conversations.Items.IsNullOrEmpty())
{
return null;
}
var conv = !conversations.Items.IsNullOrEmpty()
? ConversationViewModel.FromSession(conversations.Items.First())
: new();
var result = ConversationViewModel.FromSession(conversations.Items.First());
var state = _services.GetRequiredService<IConversationStateService>();
user = await userService.GetUser(result.User.Id);
result.User = UserViewModel.FromUser(user);
user = !string.IsNullOrEmpty(conv?.User?.Id)
? await userService.GetUser(conv.User.Id)
: null;
return result;
}
[HttpPost("/conversation/summary")]
public async Task<string> GetConversationSummary([FromBody] ConversationSummaryModel input)
{
var service = _services.GetRequiredService<IConversationService>();
return await service.GetConversationSummary(input.ConversationIds);
}
[HttpGet("/conversation/{conversationId}/user")]
public async Task<UserViewModel> GetConversationUser([FromRoute] string conversationId)
{
var service = _services.GetRequiredService<IConversationService>();
var conversations = await service.GetConversations(new ConversationFilter
{
Id = conversationId
});
var userService = _services.GetRequiredService<IUserService>();
var conversation = conversations?.Items?.FirstOrDefault();
var userId = conversation == null ? _user.Id : conversation.UserId;
var user = await userService.GetUser(userId);
if (user == null)
{
return new UserViewModel
user = new User
{
Id = _user.Id,
UserName = _user.UserName,
@ -202,7 +176,16 @@ public class ConversationController : ControllerBase
};
}
return UserViewModel.FromUser(user);
conv.User = UserViewModel.FromUser(user);
conv.IsRealtimeEnabled = settings?.Assemblies?.Contains("BotSharp.Core.Realtime") ?? false;
return conv;
}
[HttpPost("/conversation/summary")]
public async Task<string> GetConversationSummary([FromBody] ConversationSummaryModel input)
{
var service = _services.GetRequiredService<IConversationService>();
return await service.GetConversationSummary(input.ConversationIds);
}
[HttpPut("/conversation/{conversationId}/update-title")]
@ -345,7 +328,8 @@ public class ConversationController : ControllerBase
};
await HookEmitter.Emit<IConversationHook>(_services, async hook =>
await hook.OnNotificationGenerated(inputMsg)
await hook.OnNotificationGenerated(inputMsg),
routing.Context.GetCurrentAgentId()
);
return response;

27
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
@ -58,14 +59,7 @@ public class InstructModeController : ControllerBase
var textCompletion = CompletionProvider.GetTextCompletion(_services);
var response = await textCompletion.GetCompletion(input.Text, agentId, Guid.NewGuid().ToString());
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = agentId,
@ -74,8 +68,8 @@ public class InstructModeController : ControllerBase
TemplateName = input.Template,
UserMessage = input.Text,
CompletionText = response
});
}
}), agentId);
return response;
}
@ -103,14 +97,7 @@ public class InstructModeController : ControllerBase
}
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId)
{
continue;
}
await HookEmitter.Emit<IInstructHook>(_services, async hook =>
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = agentId,
@ -120,8 +107,8 @@ public class InstructModeController : ControllerBase
UserMessage = input.Text,
SystemInstruction = message.RenderedInstruction,
CompletionText = message.Content
});
}
}), agentId);
return message.Content;
}
#endregion

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs
View file

@ -13,9 +13,9 @@ public class McpController : ControllerBase
}
[HttpGet("/mcp/server-configs")]
public IEnumerable<McpServerOptionModel> GetMcpServerConfigs()
public async Task<IEnumerable<McpServerOptionModel>> GetMcpServerConfigs()
{
var mcp = _services.GetRequiredService<IMcpService>();
return mcp.GetServerConfigs();
return await mcp.GetServerConfigsAsync();
}
}

4
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs
View file

@ -1,9 +1,13 @@
using BotSharp.Abstraction.Conversations.Dtos;
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Conversations;
public class ConversationViewModel : ConversationDto
{
[JsonPropertyName("is_realtime_enabled")]
public bool IsRealtimeEnabled { get; set; }
public static ConversationViewModel FromSession(Conversation sess)
{
return new ConversationViewModel

3
src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
View file

@ -1,5 +1,6 @@
using Anthropic.SDK.Common;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.MLTasks.Settings;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
@ -29,7 +30,7 @@ public class ChatCompletionProvider : IChatCompletion
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in contentHooks)

9
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,5 +1,6 @@
using Azure;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Hooks;
using OpenAI.Chat;
using System.ClientModel;
@ -29,7 +30,7 @@ public class ChatCompletionProvider : IChatCompletion
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in contentHooks)
@ -128,7 +129,7 @@ public class ChatCompletionProvider : IChatCompletion
Func<RoleDialogModel, Task> onMessageReceived,
Func<RoleDialogModel, Task> onFunctionExecuting)
{
var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
var hooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in hooks)
@ -304,10 +305,10 @@ public class ChatCompletionProvider : IChatCompletion
{
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
}));
messages.Add(new ToolChatMessage(message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content));
}
else if (message.Role == AgentRole.User)
{

3
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.MLTasks.Settings;
using System.Net.Http;
using System.Net.Mime;
@ -36,7 +37,7 @@ public class TextCompletionProvider : ITextCompletion
public async Task<string> GetCompletion(string text, string agentId, string messageId)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agentId);
// Before chat completion hook
var agent = new Agent()

1
src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs
View file

@ -24,7 +24,6 @@ public class ChatHubPlugin : IBotSharpPlugin
services.AddScoped<IConversationHook, ChatHubConversationHook>();
services.AddScoped<IConversationHook, StreamingLogHook>();
services.AddScoped<IConversationHook, WelcomeHook>();
services.AddScoped<ConversationHookProvider>();
services.AddScoped<IRoutingHook, StreamingLogHook>();
services.AddScoped<IContentGeneratingHook, StreamingLogHook>();
services.AddScoped<ICrontabHook, ChatHubCrontabHook>();

5
src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs
View file

@ -94,8 +94,7 @@ public class ChatStreamMiddleware
}
}
await _session.Disconnect();
await _session.DisconnectAsync();
_session.Dispose();
}
@ -105,7 +104,7 @@ public class ChatStreamMiddleware
{
if (_session != null)
{
await _session.SendEvent(data);
await _session.SendEventAsync(data);
}
});
}

3
src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.SignalR;
@ -26,7 +27,6 @@ public class SignalRHub : Hub
{
_logger.LogInformation($"SignalR Hub: {_user.FirstName} {_user.LastName} ({Context.User.Identity.Name}) connected in {Context.ConnectionId}");
var hooks = _services.GetServices<IConversationHook>();
var convService = _services.GetRequiredService<IConversationService>();
_context.HttpContext.Request.Query.TryGetValue("conversationId", out var conversationId);
@ -38,6 +38,7 @@ public class SignalRHub : Hub
var conv = await convService.GetConversation(conversationId);
if (conv != null)
{
var hooks = _services.GetHooks<IConversationHook>(conv.AgentId);
foreach (var hook in hooks)
{
// Check if user connected with agent is the first time.

5
src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging;
using OpenAI.Chat;
using BotSharp.Abstraction.Files;
using BotSharp.Plugin.DeepSeek.Providers;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Plugin.DeepSeekAI.Providers.Chat;
@ -25,7 +26,7 @@ public class ChatCompletionProvider : IChatCompletion
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in contentHooks)
@ -95,7 +96,7 @@ public class ChatCompletionProvider : IChatCompletion
public async Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
{
var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
var hooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in hooks)

3
src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using Microsoft.Extensions.Logging;
using OpenAI.Chat;
@ -22,7 +23,7 @@ public class TextCompletionProvider : ITextCompletion
public async Task<string> GetCompletion(string text, string agentId, string messageId)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agentId);
var state = _services.GetRequiredService<IConversationStateService>();
// Before chat completion hook

2
src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

18
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs Normal file
View file

@ -0,0 +1,18 @@
using GenerativeAI.Types;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeClientPayload
{
[JsonPropertyName("setup")]
public RealtimeGenerateContentSetup? Setup { get; set; }
[JsonPropertyName("clientContent")]
public BidiGenerateContentClientContent? ClientContent { get; set; }
[JsonPropertyName("realtimeInput")]
public BidiGenerateContentRealtimeInput? RealtimeInput { get; set; }
[JsonPropertyName("toolResponse")]
public BidiGenerateContentToolResponse? ToolResponse { get; set; }
}

35
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs Normal file
View file

@ -0,0 +1,35 @@
using GenerativeAI.Types;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeGenerateContentSetup
{
[JsonPropertyName("model")]
public string? Model { get; set; }
[JsonPropertyName("generationConfig")]
public GenerationConfig? GenerationConfig { get; set; }
[JsonPropertyName("systemInstruction")]
public Content? SystemInstruction { get; set; }
[JsonPropertyName("tools")]
public Tool[]? Tools { get; set; }
[JsonPropertyName("inputAudioTranscription")]
public AudioTranscriptionConfig? InputAudioTranscription { get; set; }
[JsonPropertyName("outputAudioTranscription")]
public AudioTranscriptionConfig? OutputAudioTranscription { get; set; }
[JsonPropertyName("sessionResumption")]
public SessionResumptionConfig? SessionResumption { get; set; }
}
internal class AudioTranscriptionConfig { }
internal class SessionResumptionConfig
{
[JsonPropertyName("handle")]
public string? Handle { get; set; }
}

106
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs Normal file
View file

@ -0,0 +1,106 @@
using GenerativeAI.Types;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeServerResponse
{
[JsonPropertyName("setupComplete")]
public RealtimeGenerateContentSetupComplete? SetupComplete { get; set; }
[JsonPropertyName("serverContent")]
public RealtimeGenerateContentServerContent? ServerContent { get; set; }
[JsonPropertyName("usageMetadata")]
public RealtimeUsageMetaData? UsageMetaData { get; set; }
[JsonPropertyName("toolCall")]
public RealtimeToolCall? ToolCall { get; set; }
[JsonPropertyName("sessionResumptionUpdate")]
public RealtimeSessionResumptionUpdate? SessionResumptionUpdate { get; set; }
}
internal class RealtimeGenerateContentSetupComplete { }
internal class RealtimeGenerateContentServerContent
{
[JsonPropertyName("turnComplete")]
public bool? TurnComplete { get; set; }
[JsonPropertyName("generationComplete")]
public bool? GenerationComplete { get; set; }
[JsonPropertyName("interrupted")]
public bool? Interrupted { get; set; }
[JsonPropertyName("modelTurn")]
public Content? ModelTurn { get; set; }
[JsonPropertyName("inputTranscription")]
public RealtimeGenerateContentTranscription? InputTranscription { get; set; }
[JsonPropertyName("outputTranscription")]
public RealtimeGenerateContentTranscription? OutputTranscription { get; set; }
}
internal class RealtimeUsageMetaData
{
[JsonPropertyName("promptTokenCount")]
public int? PromptTokenCount { get; set; }
[JsonPropertyName("responseTokenCount")]
public int? ResponseTokenCount { get; set; }
[JsonPropertyName("totalTokenCount")]
public int? TotalTokenCount { get; set; }
[JsonPropertyName("promptTokensDetails")]
public List<RealtimeTokenDetail>? PromptTokensDetails { get; set; }
[JsonPropertyName("responseTokensDetails")]
public List<RealtimeTokenDetail>? ResponseTokensDetails { get; set; }
}
internal class RealtimeTokenDetail
{
[JsonPropertyName("modality")]
public string? Modality { get; set; }
[JsonPropertyName("tokenCount")]
public int? TokenCount { get; set; }
}
internal class RealtimeGenerateContentTranscription
{
[JsonPropertyName("text")]
public string? Text { get; set; }
}
internal class RealtimeToolCall
{
[JsonPropertyName("functionCalls")]
public List<RealtimeFunctionCall>? FunctionCalls { get; set; }
}
internal class RealtimeFunctionCall
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("name")]
public string Name { get; set; }
[JsonPropertyName("args")]
public JsonNode? Args { get; set; }
}
internal class RealtimeSessionResumptionUpdate
{
[JsonPropertyName("newHandle")]
public string? NewHandle { get; set; }
[JsonPropertyName("resumable")]
public bool? Resumable { get; set; }
}

53
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs Normal file
View file

@ -0,0 +1,53 @@
using System.IO;
namespace BotSharp.Plugin.GoogleAI.Models.Realtime;
internal class RealtimeTranscriptionResponse : IDisposable
{
public RealtimeTranscriptionResponse()
{
}
private MemoryStream _contentStream = new();
public Stream? ContentStream
{
get
{
return _contentStream != null ? _contentStream : new MemoryStream();
}
}
public void Collect(string text)
{
var binary = BinaryData.FromString(text);
var bytes = binary.ToArray();
_contentStream.Position = _contentStream.Length;
_contentStream.Write(bytes, 0, bytes.Length);
_contentStream.Position = 0;
}
public string GetText()
{
if (_contentStream.Length == 0)
{
return string.Empty;
}
var bytes = _contentStream.ToArray();
var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
return text;
}
public void Clear()
{
_contentStream.Position = 0;
_contentStream.SetLength(0);
}
public void Dispose()
{
_contentStream?.Dispose();
}
}

7
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -2,6 +2,7 @@ using System.Text.Json.Nodes;
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
@ -33,7 +34,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in contentHooks)
@ -91,7 +92,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
public async Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
{
var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
var hooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in hooks)
@ -270,6 +271,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
FunctionCall = new FunctionCall
{
Id = message.ToolCallId,
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
}
@ -281,6 +283,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
FunctionResponse = new FunctionResponse
{
Id = message.ToolCallId,
Name = message.FunctionName,
Response = new JsonObject()
{

8
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
View file

@ -1,13 +1,11 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using LLMSharp.Google.Palm;
using LLMSharp.Google.Palm.DiscussService;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;
[Obsolete]
public class PalmChatCompletionProvider : IChatCompletion
{
private readonly IServiceProvider _services;
@ -29,7 +27,7 @@ public class PalmChatCompletionProvider : IChatCompletion
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in contentHooks)

553
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,8 +1,11 @@
using System.Threading;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Realtime.Models.Session;
using BotSharp.Core.Session;
using BotSharp.Plugin.GoogleAI.Models.Realtime;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
using GenerativeAI.Types.Converters;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
@ -12,14 +15,38 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public string Model => _model;
private string _model = GoogleAIModels.Gemini2FlashExp;
private MultiModalLiveClient _client;
private GenerativeModel _chatClient;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private List<string> renderedInstructions = [];
private LlmRealtimeSession _session;
private readonly GoogleAiSettings _settings;
private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000";
private readonly JsonSerializerOptions _jsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() },
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement
};
private RealtimeTranscriptionResponse _inputStream = new();
private RealtimeTranscriptionResponse _outputStream = new();
private RealtimeHubConnection _conn;
private Func<Task> _onModelReady;
private Func<string, string, Task> _onModelAudioDeltaReceived;
private Func<Task> _onModelAudioResponseDone;
private Func<string, Task> _onModelAudioTranscriptDone;
private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
private Func<string, Task> _onConversationItemCreated;
private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
private Func<Task> _onInterruptionDetected;
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings settings,
@ -35,26 +62,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_model = model;
}
private RealtimeHubConnection _conn;
private Action _onModelReady;
private Action<string, string> _onModelAudioDeltaReceived;
private Action _onModelAudioResponseDone;
private Action<string> _onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> _onModelResponseDone;
private Action<string> _onConversationItemCreated;
private Action<RoleDialogModel> _onInputAudioTranscriptionCompleted;
private Action _onUserInterrupted;
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
public async Task Connect(
RealtimeHubConnection conn,
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
_conn = conn;
_onModelReady = onModelReady;
@ -63,206 +80,224 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_onModelAudioTranscriptDone = onModelAudioTranscriptDone;
_onModelResponseDone = onModelResponseDone;
_onConversationItemCreated = onConversationItemCreated;
_onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
_onUserInterrupted = onUserInterrupted;
_onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
_onInterruptionDetected = onInterruptionDetected;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
var modelSettings = settingsService.GetSetting(Provider, _model);
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
_chatClient = client.CreateGenerativeModel(_model);
_client = _chatClient.CreateMultiModalLiveClient(
config: new GenerationConfig
{
ResponseModalities = [Modality.AUDIO],
},
systemInstruction: "You are a helpful assistant.",
logger: _logger);
Reset();
_inputStream = new();
_outputStream = new();
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
{
JsonOptions = _jsonOptions
});
await AttachEvents(_client);
await _client.ConnectAsync(false);
var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta");
await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None);
await onModelReady();
_ = ReceiveMessage();
}
private async Task ReceiveMessage()
{
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
try
{
var response = JsonSerializer.Deserialize<RealtimeServerResponse>(receivedText, _jsonOptions);
if (response == null)
{
continue;
}
if (response.SetupComplete != null)
{
_logger.LogInformation($"Session setup completed.");
}
else if (response.SessionResumptionUpdate != null)
{
_logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");
}
else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
{
var functionCall = response.ToolCall.FunctionCalls!.First();
_logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty}).");
if (functionCall != null)
{
var messages = OnFunctionCall(_conn, functionCall);
await _onModelResponseDone(messages);
}
}
else if (response.ServerContent != null)
{
if (response.ServerContent.InputTranscription?.Text != null)
{
_inputStream.Collect(response.ServerContent.InputTranscription.Text);
}
if (response.ServerContent.OutputTranscription?.Text != null)
{
_outputStream.Collect(response.ServerContent.OutputTranscription.Text);
}
if (response.ServerContent.ModelTurn != null)
{
_logger.LogInformation($"Model audio delta received.");
// Handle input transcription
var inputTranscription = _inputStream.GetText();
if (!string.IsNullOrEmpty(inputTranscription))
{
var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription);
await _onInputAudioTranscriptionDone(message);
}
_inputStream.Clear();
var parts = response.ServerContent.ModelTurn.Parts;
if (!parts.IsNullOrEmpty())
{
foreach (var part in parts)
{
if (!string.IsNullOrEmpty(part.InlineData?.Data))
{
await _onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
}
}
}
}
else if (response.ServerContent.GenerationComplete == true)
{
_logger.LogInformation($"Model generation completed.");
}
else if (response.ServerContent.TurnComplete == true)
{
_logger.LogInformation($"Model turn completed.");
// Handle output transcription
var outputTranscription = _outputStream.GetText();
if (!string.IsNullOrEmpty(outputTranscription))
{
var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData);
await _onModelResponseDone(messages);
}
_inputStream.Clear();
_outputStream.Clear();
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error when deserializing server response. {ex.Message}");
break;
}
}
_inputStream.Dispose();
_outputStream.Dispose();
_session.Dispose();
}
public async Task Disconnect()
{
if (_client != null)
await _client.DisconnectAsync();
if (_session != null)
{
_inputStream?.Dispose();
_outputStream?.Dispose();
await _session.DisconnectAsync();
_session.Dispose();
}
}
public async Task AppenAudioBuffer(string message)
{
await _client.SendAudioAsync(Convert.FromBase64String(message));
await SendEventToModel(new RealtimeClientPayload
{
RealtimeInput = new()
{
MediaChunks = [new() { Data = message, MimeType = DEFAULT_MIME_TYPE }]
}
});
}
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{
var buffer = data.AsSpan(0, length).ToArray();
await _client.SendAudioAsync(buffer,"audio/pcm;rate=16000");
await SendEventToModel(new RealtimeClientPayload
{
RealtimeInput = new()
{
MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = DEFAULT_MIME_TYPE }]
}
});
}
public async Task TriggerModelInference(string? instructions = null)
{
await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
if (string.IsNullOrWhiteSpace(instructions)) return;
var content = new Content(instructions, AgentRole.User);
await SendEventToModel(new RealtimeClientPayload
{
TurnComplete = true,
ClientContent = new()
{
Turns = [content],
TurnComplete = true
}
});
}
public async Task CancelModelResponse()
{
}
public async Task RemoveConversationItem(string itemId)
{
}
private Task AttachEvents(MultiModalLiveClient client)
{
client.Connected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client connected.");
_onModelReady();
};
client.Disconnected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client disconnected.");
};
client.MessageReceived += async (sender, e) =>
{
_logger.LogInformation("User message received.");
if (e.Payload.SetupComplete != null)
{
_onConversationItemCreated(_client.ConnectionId.ToString());
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(_conn, e.Payload.ServerContent);
_onModelResponseDone(responseDone);
}
}
};
client.AudioChunkReceived += (sender, e) =>
{
_onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
};
client.TextChunkReceived += (sender, e) =>
{
_onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
client.GenerationInterrupted += (sender, e) =>
{
_logger.LogInformation("Audio generation interrupted.");
_onUserInterrupted();
};
client.AudioReceiveCompleted += (sender, e) =>
{
_logger.LogInformation("Audio receive completed.");
_onModelAudioResponseDone();
};
client.ErrorOccurred += (sender, e) =>
{
var ex = e.GetException();
_logger.LogError(ex, "Error occurred in Google Realtime Client");
};
return Task.CompletedTask;
}
private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
BidiGenerateContentServerContent serverContent)
{
var outputs = new List<RoleDialogModel>();
var parts = serverContent.ModelTurn?.Parts;
if (parts != null)
{
foreach (var part in parts)
{
var call = part.FunctionCall;
if (call != null)
{
var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
else
{
var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = call.Name,
FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
ToolCallId = call.Id ?? String.Empty,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
});
}
return outputs;
}
public async Task SendEventToModel(object message)
{
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
if (_session == null) return;
await _session.SendEventToModelAsync(message);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
if (!isInit)
{
return string.Empty;
}
var agentService = _services.GetRequiredService<IAgentService>();
var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var (prompt, request) = PrepareOptions(agent, []);
var (prompt, request) = PrepareOptions(_chatClient, agent, new List<RoleDialogModel>());
var config = request.GenerationConfig;
var config = request.GenerationConfig ?? new();
//Output Modality can either be text or audio
if (config != null)
{
config.ResponseModalities = new List<Modality>([Modality.AUDIO]);
config.ResponseModalities = [Modality.AUDIO];
config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f);
config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens;
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
}
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id);
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
@ -270,98 +305,158 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
Name = x.Name ?? string.Empty,
Description = x.Description ?? string.Empty,
Parameters = x.Parameters != null
? JsonSerializer.Deserialize<FunctionParametersDef>(JsonSerializer.Serialize(x.Parameters))
: null
};
fn.Parameters = x.Parameters != null
? JsonSerializer.Deserialize<FunctionParametersDef>(JsonSerializer.Serialize(x.Parameters))
: null;
return fn;
}).ToArray();
await HookEmitter.Emit<IContentGeneratingHook>(_services,
async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); });
async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }, agent.Id);
if (_settings.Gemini.UseGoogleSearch)
{
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools ??= [];
request.Tools.Add(new Tool()
{
GoogleSearch = new GoogleSearchTool()
});
}
// if(request.Tools.Count == 0)
// request.Tools = null;
// config.MaxOutputTokens = null;
await _client.SendSetupAsync(new BidiGenerateContentSetup()
var payload = new RealtimeClientPayload
{
GenerationConfig = config,
Model = Model.ToModelId(),
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
});
Setup = new RealtimeGenerateContentSetup()
{
GenerationConfig = config,
Model = Model.ToModelId(),
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null
}
};
_logger.LogInformation($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}");
await SendEventToModel(payload);
return prompt;
}
public async Task InsertConversationItem(RoleDialogModel message)
{
if (_client == null)
throw new Exception("Client is not initialized");
if (message.Role == AgentRole.Function)
{
var function = new FunctionResponse()
{
Id = message.ToolCallId,
Name = message.FunctionName ?? string.Empty,
Response = JsonNode.Parse(message.Content ?? "{}")
Response = new JsonObject()
{
["result"] = message.Content ?? string.Empty
}
};
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
await SendEventToModel(new RealtimeClientPayload
{
FunctionResponses = [function]
ToolResponse = new()
{
FunctionResponses = [function]
}
});
}
else if (message.Role == AgentRole.Assistant)
{
await SendEventToModel(new RealtimeClientPayload
{
ClientContent = new()
{
Turns = [new Content(message.Content, AgentRole.Model)],
TurnComplete = true
}
});
}
else if (message.Role == AgentRole.User)
{
await _client.SentTextAsync(message.Content);
await SendEventToModel(new RealtimeClientPayload
{
ClientContent = new()
{
Turns = [new Content(message.Content, AgentRole.User)],
TurnComplete = true
}
});
}
else
{
throw new NotImplementedException("");
throw new NotImplementedException($"Unrecognized role {message.Role}.");
}
}
public Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
#region Private methods
private List<RoleDialogModel> OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall)
{
throw new NotImplementedException("");
var outputs = new List<RoleDialogModel>
{
new(AgentRole.Assistant, string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = functionCall.Name,
FunctionArgs = functionCall.Args?.ToJsonString(_jsonOptions),
ToolCallId = functionCall.Id,
MessageType = MessageTypeName.FunctionCall
}
};
return outputs;
}
public Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
private async Task<List<RoleDialogModel>> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? usage)
{
return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
var outputs = new List<RoleDialogModel>
{
new(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = Guid.NewGuid().ToString(),
MessageType = MessageTypeName.Plain
}
};
if (usage != null)
{
var contentHooks = _services.GetHooks<IContentGeneratingHook>(conn.CurrentAgentId);
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
AudioInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0,
TextOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0,
AudioOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0
});
}
}
return outputs;
}
private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
private (string, GenerateContentRequest) PrepareOptions(Agent agent,
List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var googleSettings = _settings;
renderedInstructions = [];
// Add settings
aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
{
AutoCallFunction = false
};
// Assembly messages
var contents = new List<Content>();
var tools = new List<Tool>();
@ -417,6 +512,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
FunctionCall = new FunctionCall
{
Id = message.ToolCallId,
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
}
@ -428,6 +524,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
FunctionResponse = new FunctionResponse
{
Id = message.ToolCallId,
Name = message.FunctionName ?? string.Empty,
Response = new JsonObject()
{
@ -458,6 +555,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
{
SystemInstruction = !systemPrompts.IsNullOrEmpty()
@ -495,4 +593,29 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
return prompt;
}
private RoleDialogModel OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text)
{
return new RoleDialogModel(AgentRole.User, text)
{
CurrentAgentId = conn.CurrentAgentId
};
}
private Uri BuildWebsocketUri(string apiKey, string version = "v1beta")
{
return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}");
}
private void Reset()
{
_inputStream?.Clear();
_outputStream?.Clear();
_inputStream?.Dispose();
_outputStream?.Dispose();
_session?.Dispose();
}
#endregion
}

3
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
@ -32,7 +33,7 @@ public class GeminiTextCompletionProvider : ITextCompletion
public async Task<string> GetCompletion(string text, string agentId, string messageId)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agentId);
// Before completion hook
var agent = new Agent()

7
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
View file

@ -1,9 +1,8 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;
[Obsolete]
public class PalmTextCompletionProvider : ITextCompletion
{
private readonly IServiceProvider _services;
@ -27,7 +26,7 @@ public class PalmTextCompletionProvider : ITextCompletion
public async Task<string> GetCompletion(string text, string agentId, string messageId)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agentId);
// Before completion hook
var agent = new Agent() { Id = agentId };

7
src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
View file

@ -16,14 +16,15 @@ global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Abstraction.Realtime;
global using BotSharp.Abstraction.Realtime.Models;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.GoogleAi.Providers.Chat;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Enums;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Loggers;
global using BotSharp.Abstraction.Loggers;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Plugin.GoogleAi.Providers.Chat;

6
src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs
View file

@ -47,6 +47,8 @@ public class DialogMetaDataMongoElement
public string MessageId { get; set; } = default!;
public string MessageType { get; set; } = default!;
public string? FunctionName { get; set; }
public string? FunctionArgs { get; set; }
public string? ToolCallId { get; set; }
public string? SenderId { get; set; }
public DateTime CreateTime { get; set; }
@ -59,6 +61,8 @@ public class DialogMetaDataMongoElement
MessageId = meta.MessageId,
MessageType = meta.MessageType,
FunctionName = meta.FunctionName,
FunctionArgs = meta.FunctionArgs,
ToolCallId = meta.ToolCallId,
SenderId = meta.SenderId,
CreatedTime = meta.CreateTime,
};
@ -73,6 +77,8 @@ public class DialogMetaDataMongoElement
MessageId = meta.MessageId,
MessageType = meta.MessageType,
FunctionName = meta.FunctionName,
FunctionArgs = meta.FunctionArgs,
ToolCallId = meta.ToolCallId,
SenderId = meta.SenderId,
CreateTime = meta.CreatedTime,
};

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories.Filters;
using MongoDB.Driver;
using System.Text.Json;
namespace BotSharp.Plugin.MongoStorage.Repository;

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -76,7 +76,7 @@ public class RealtimeSessionTurnDetection
public string Type { get; set; } = "semantic_vad";
[JsonPropertyName("eagerness")]
public string eagerness { get;set; } = "auto";
public string Eagerness { get;set; } = "auto";
}
public class InputAudioTranscription

10
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,4 +1,4 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Hooks;
using OpenAI.Chat;
namespace BotSharp.Plugin.OpenAI.Providers.Chat;
@ -32,7 +32,7 @@ public class ChatCompletionProvider : IChatCompletion
public async Task<RoleDialogModel> GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in contentHooks)
@ -105,7 +105,7 @@ public class ChatCompletionProvider : IChatCompletion
Func<RoleDialogModel, Task> onMessageReceived,
Func<RoleDialogModel, Task> onFunctionExecuting)
{
var hooks = _services.GetServices<IContentGeneratingHook>().ToList();
var hooks = _services.GetHooks<IContentGeneratingHook>(agent.Id);
// Before chat completion hook
foreach (var hook in hooks)
@ -272,10 +272,10 @@ public class ChatCompletionProvider : IChatCompletion
{
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
}));
messages.Add(new ToolChatMessage(message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content));
}
else if (message.Role == AgentRole.User)
{

493
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using OpenAI.Chat;
@ -30,26 +31,22 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task Connect(
RealtimeHubConnection conn,
Action onModelReady,
Action<string,string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onInterruptionDetected)
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
_model = realtimeSettings.Model;
var settings = settingsService.GetSetting(Provider, _model);
if (_session != null)
{
_session.Dispose();
}
_session?.Dispose();
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
{
JsonOptions = _botsharpOptions.JsonSerializerOptions
@ -65,6 +62,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
cancellationToken: CancellationToken.None);
_ = ReceiveMessage(
realtimeSettings,
conn,
onModelReady,
onModelAudioDeltaReceived,
@ -72,15 +70,151 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionCompleted,
onInputAudioTranscriptionDone,
onInterruptionDetected);
}
private async Task ReceiveMessage(
RealtimeModelSettings realtimeSettings,
RealtimeHubConnection conn,
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
DateTime? startTime = null;
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
if (realtimeSettings?.ModelResponseTimeoutSeconds > 0
&& !string.IsNullOrWhiteSpace(realtimeSettings?.ModelResponseTimeoutEndEvent)
&& startTime.HasValue
&& (DateTime.UtcNow - startTime.Value).TotalSeconds >= realtimeSettings.ModelResponseTimeoutSeconds
&& response.Type != realtimeSettings.ModelResponseTimeoutEndEvent)
{
startTime = null;
await TriggerModelInference("Responsd to user immediately");
continue;
}
if (response.Type == "error")
{
_logger.LogError($"{response.Type}: {receivedText}");
var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
if (error?.Body.Type == "server_error")
{
break;
}
}
else if (response.Type == "session.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
await onModelReady();
}
else if (response.Type == "session.updated")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
else if (response.Type == "response.audio_transcript.delta")
{
_logger.LogDebug($"{response.Type}: {receivedText}");
}
else if (response.Type == "response.audio_transcript.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
await onModelAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
if (audio?.Delta != null)
{
_logger.LogDebug($"{response.Type}: {receivedText}");
await onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
}
}
else if (response.Type == "response.audio.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
await onModelAudioResponseDone();
}
else if (response.Type == "response.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseDone>(receivedText).Body;
if (data.Status != "completed")
{
if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
await onInterruptionDetected();
await TriggerModelInference("Response user concisely");
}
}
else
{
var messages = await OnResponsedDone(conn, receivedText);
await onModelResponseDone(messages);
}
}
else if (response.Type == "conversation.item.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
if (data?.Item?.Role == "user")
{
startTime = DateTime.UtcNow;
}
await onConversationItemCreated(receivedText);
}
else if (response.Type == "conversation.item.input_audio_transcription.completed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
if (!string.IsNullOrEmpty(message.Content))
{
await onInputAudioTranscriptionDone(message);
}
}
else if (response.Type == "input_audio_buffer.speech_started")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
// Handle user interuption
await onInterruptionDetected();
}
else if (response.Type == "input_audio_buffer.speech_stopped")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
else if (response.Type == "input_audio_buffer.committed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
}
_session.Dispose();
}
public async Task Disconnect()
{
if (_session != null)
{
await _session.Disconnect();
await _session.DisconnectAsync();
_session.Dispose();
}
}
@ -142,136 +276,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
});
}
private async Task ReceiveMessage(
RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
Action onInterruptionDetected)
{
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
if (string.IsNullOrEmpty(receivedText))
{
continue;
}
var response = JsonSerializer.Deserialize<ServerEventResponse>(receivedText);
if (response.Type == "error")
{
_logger.LogError($"{response.Type}: {receivedText}");
var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
if (error?.Body.Type == "server_error")
{
break;
}
}
else if (response.Type == "session.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelReady();
}
else if (response.Type == "session.updated")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
else if (response.Type == "response.audio_transcript.delta")
{
_logger.LogDebug($"{response.Type}: {receivedText}");
}
else if (response.Type == "response.audio_transcript.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
onModelAudioTranscriptDone(data.Transcript);
}
else if (response.Type == "response.audio.delta")
{
var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
if (audio?.Delta != null)
{
_logger.LogDebug($"{response.Type}: {receivedText}");
onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
}
}
else if (response.Type == "response.audio.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
onModelAudioResponseDone();
}
else if (response.Type == "response.done")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ResponseDone>(receivedText).Body;
if (data.Status != "completed")
{
if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
onInterruptionDetected();
await TriggerModelInference("Response user concisely");
}
}
else
{
var messages = await OnResponsedDone(conn, receivedText);
onModelResponseDone(messages);
}
}
else if (response.Type == "conversation.item.created")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
onConversationItemCreated(receivedText);
}
else if (response.Type == "conversation.item.input_audio_transcription.completed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
if (!string.IsNullOrEmpty(message.Content))
{
onUserAudioTranscriptionCompleted(message);
}
}
else if (response.Type == "input_audio_buffer.speech_started")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
// Handle user interuption
onInterruptionDetected();
}
else if (response.Type == "input_audio_buffer.speech_stopped")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
else if (response.Type == "input_audio_buffer.committed")
{
_logger.LogInformation($"{response.Type}: {receivedText}");
}
}
_session.Dispose();
}
public async Task SendEventToModel(object message)
{
if (_session == null) return;
await _session.SendEventToModel(message);
await _session.SendEventToModelAsync(message);
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var (prompt, messages, options) = PrepareOptions(agent, []);
@ -319,7 +336,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
if (realtimeModelSettings.InputAudioTranscribe)
{
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id);
sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
{
@ -332,7 +349,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
{
await hook.OnSessionUpdated(agent, instruction, functions, isInit);
});
}, agent.Id);
await SendEventToModel(sessionUpdate);
await Task.Delay(300);
@ -402,11 +419,105 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
}
else
{
throw new NotImplementedException("");
throw new NotImplementedException($"Unrecognized role {message.Role}.");
}
}
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
public void SetModelName(string model)
{
_model = model;
}
#region Private methods
private async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
{
var outputs = new List<RoleDialogModel>();
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
/*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
await TriggerModelInference("Response user concisely");
}*/
return [];
}
var prompts = new List<string>();
var inputTokenDetails = data.Usage?.InputTokenDetails;
var outputTokenDetails = data.Usage?.OutputTokenDetails;
foreach (var output in data.Outputs)
{
if (output.Type == "function_call")
{
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = output.Name,
FunctionArgs = output.Arguments,
ToolCallId = output.CallId,
MessageId = output.Id,
MessageType = MessageTypeName.FunctionCall
});
prompts.Add($"{output.Name}({output.Arguments})");
}
else if (output.Type == "message")
{
var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty;
outputs.Add(new RoleDialogModel(output.Role, content)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
prompts.Add(content);
}
}
// After chat completion hook
var text = string.Join("\r\n", prompts);
var contentHooks = _services.GetHooks<IContentGeneratingHook>(conn.CurrentAgentId);
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
return outputs;
}
private async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
};
}
private (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -588,103 +699,5 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
return prompt;
}
public void SetModelName(string model)
{
_model = model;
}
public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
{
var outputs = new List<RoleDialogModel>();
var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
if (data.Status != "completed")
{
_logger.LogError(data.StatusDetails.ToString());
/*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
{
await TriggerModelInference("Response user concisely");
}*/
return [];
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var prompts = new List<string>();
var inputTokenDetails = data.Usage?.InputTokenDetails;
var outputTokenDetails = data.Usage?.OutputTokenDetails;
foreach (var output in data.Outputs)
{
if (output.Type == "function_call")
{
outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = output.Name,
FunctionArgs = output.Arguments,
ToolCallId = output.CallId,
MessageId = output.Id,
MessageType = MessageTypeName.FunctionCall
});
prompts.Add($"{output.Name}({output.Arguments})");
}
else if (output.Type == "message")
{
var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty;
outputs.Add(new RoleDialogModel(output.Role, content)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = output.Id,
MessageType = MessageTypeName.Plain
});
prompts.Add(content);
}
}
var text = string.Join("\r\n", prompts);
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text)
{
CurrentAgentId = conn.CurrentAgentId
},
new TokenStatsModel
{
Provider = Provider,
Model = _model,
Prompt = text,
TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0,
AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0,
TextOutputTokens = outputTokenDetails?.TextTokens ?? 0,
AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0
});
}
return outputs;
}
public async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
{
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
return new RoleDialogModel(AgentRole.User, data.Transcript)
{
CurrentAgentId = conn.CurrentAgentId
};
}
public async Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
{
var item = response.JsonContent<ConversationItemCreated>().Item;
var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript);
return message;
}
#endregion
}

3
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Hooks;
using System.Net.Http;
using System.Net.Mime;
@ -25,7 +26,7 @@ public class TextCompletionProvider : ITextCompletion
public async Task<string> GetCompletion(string text, string agentId, string messageId)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var contentHooks = _services.GetHooks<IContentGeneratingHook>(agentId);
// Before chat completion hook
var agent = new Agent()

2
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs
View file

@ -90,7 +90,7 @@ public class SqlGenerationFn : IFunctionCallback
{
var requirement = await x.GetSummaryAdditionalRequirements(nameof(SqlGenerationPlanner), message);
additionalRequirements.Add(requirement);
});
}, message.CurrentAgentId);
var globalKnowledges = new List<string>();
foreach (var hook in knowledgeHooks)

3
src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs
View file

@ -30,7 +30,8 @@ public class SqlReviewFn : IFunctionCallback
if (args != null && !args.IsSqlTemplate && args.ContainsSqlStatements)
{
await HookEmitter.Emit<IPlanningHook>(_services, async hook =>
await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql")
await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql"),
message.CurrentAgentId
);
}
return true;

5
src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs
View file

@ -68,7 +68,8 @@ public class SummaryPlanFn : IFunctionCallback
message.Content = summary.Content;
await HookEmitter.Emit<IPlanningHook>(_services, async hook =>
await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message)
await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message),
message.CurrentAgentId
);
return true;
@ -88,7 +89,7 @@ public class SummaryPlanFn : IFunctionCallback
{
var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner), message);
additionalRequirements.Add(requirement);
});
}, message.CurrentAgentId);
var globalKnowledges = new List<string>();
foreach (var hook in knowledgeHooks)

2
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
View file

@ -30,7 +30,7 @@ public class SqlDriverPlanningHook : IPlanningHook
await HookEmitter.Emit<ISqlDriverHook>(_services, async (hook) =>
{
await hook.SqlGenerated(msg);
});
}, msg.CurrentAgentId);
var settings = _services.GetRequiredService<SqlDriverSetting>();
if (!settings.ExecuteSqlSelectAutonomous)

4
src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs
View file

@ -1,6 +1,8 @@
using BotSharp.Abstraction.Hooks;
namespace BotSharp.Plugin.SqlDriver.Interfaces;
public interface ISqlDriverHook
public interface ISqlDriverHook : IHookBase
{
// Get database type
string GetDatabaseType(RoleDialogModel message);

9
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
@ -55,7 +56,7 @@ public class TwilioInboundController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
}, request.AgentId);
var (agent, conversationId) = await InitConversation(request);
request.ConversationId = conversationId.Id;
@ -65,9 +66,9 @@ public class TwilioInboundController : TwilioController
if (twilio.MachineDetected(request))
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async hook => await hook.OnVoicemailStarting(request));
async hook => await hook.OnVoicemailStarting(request), request.AgentId);
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
@ -118,7 +119,7 @@ public class TwilioInboundController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
});
}, request.AgentId);
return TwiML(response);
}

4
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
@ -32,9 +33,8 @@ public class TwilioOutboundController : TwilioController
if (twilio.MachineDetected(request))
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
async hook => await hook.OnVoicemailStarting(request));
async hook => await hook.OnVoicemailStarting(request), request.AgentId);
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));

3
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
@ -36,7 +37,7 @@ public class TwilioRecordController : TwilioController
convService.SaveStates();
// recording completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnRecordingCompleted(request));
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnRecordingCompleted(request), request.AgentId);
}
else
{

3
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs
View file

@ -49,7 +49,8 @@ public class TwilioTranscribeController : TwilioController
// transcription completed
transcript.Language = request.LanguageCode;
await HookEmitter.Emit<IRealtimeHook>(_services, async x => await x.OnTranscribeCompleted(message, transcript));
await HookEmitter.Emit<IRealtimeHook>(_services, async x => await x.OnTranscribeCompleted(message, transcript),
request.AgentId);
}
}

36
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -65,7 +65,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
});
}, request.AgentId);
var twilio = _services.GetRequiredService<TwilioService>();
if (string.IsNullOrWhiteSpace(request.Intent))
@ -98,7 +98,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
});
}, request.AgentId);
return TwiML(response);
}
@ -151,7 +151,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnReceivedUserMessage(request);
});
}, request.AgentId);
}
else
{
@ -161,7 +161,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentHangUp(request);
});
}, request.AgentId);
response = twilio.HangUp(string.Empty);
}
@ -185,7 +185,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingUserResponse(request, instruction);
});
}, request.AgentId);
response = twilio.ReturnInstructions(instruction);
}
@ -223,7 +223,7 @@ public class TwilioVoiceController : TwilioController
{
request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!";
await hook.OnAgentHangUp(request);
});
}, request.AgentId);
response = twilio.HangUp($"twilio/error.mp3");
}
@ -238,7 +238,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentTransferring(request, _settings);
});
}, request.AgentId);
response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}");
}
@ -249,7 +249,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentHangUp(request);
});
}, request.AgentId);
}
else
{
@ -274,7 +274,7 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentResponsing(request, instruction);
});
}, request.AgentId);
response = twilio.ReturnInstructions(instruction);
}
@ -343,41 +343,35 @@ public class TwilioVoiceController : TwilioController
{
var twilio = _services.GetRequiredService<TwilioService>();
// Define the options with the predicate
var emitOptions = new HookEmitOption<ITwilioCallStatusHook>
{
ShouldExecute = hook => hook.IsMatch(request)
};
switch (request.CallStatus)
{
case "completed":
if (twilio.MachineDetected(request))
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnVoicemailLeft(request), emitOptions);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnVoicemailLeft(request), request.AgentId);
}
else
{
// phone call completed
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnUserDisconnected(request), emitOptions);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnUserDisconnected(request), request.AgentId);
}
break;
case "busy":
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallBusyStatus(request), emitOptions);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallBusyStatus(request), request.AgentId);
break;
case "no-answer":
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallNoAnswerStatus(request), emitOptions);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallNoAnswerStatus(request), request.AgentId);
break;
case "canceled":
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallCanceledStatus(request), emitOptions);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallCanceledStatus(request), request.AgentId);
break;
case "failed":
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallFailedStatus(request), emitOptions);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, hook => hook.OnCallFailedStatus(request), request.AgentId);
break;
default:
_logger.LogError($"Unknown call status: {request.CallStatus}, {request.CallSid}");

Some files were not shown because too many files have changed in this diff Show more