SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #997 from hchen2020/master

Browse source 
Clean code for GeminiLive
This commit is contained in:
Haiping 2025-04-05 17:18:42 -05:00 committed by GitHub
parent 18c6598af3 34a8234b1a
commit 03ab110c2b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 454 additions and 574 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs
View file

@ -22,7 +22,6 @@ public interface IRealTimeCompletion
Task SendEventToModel(object message);
Task Disconnect();
Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
Task InsertConversationItem(RoleDialogModel message);
Task RemoveConversationItem(string itemId);

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -14,7 +14,6 @@ public class RealtimeHubConnection
public string CurrentAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public string Data { get; set; } = string.Empty;
public string Model { get; set; } = null!;
public Func<string, object> OnModelMessageReceived { get; set; } = null!;
public Func<object> OnModelAudioResponseDone { get; set; } = null!;
public Func<object> OnModelUserInterrupted { get; set; } = null!;

6
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -31,9 +31,6 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
return;
}
// Clear cache to force to rebuild the agent instruction
Utilities.ClearCache();
var routing = _services.GetRequiredService<IRoutingService>();
message.Role = AgentRole.Function;
@ -60,6 +57,9 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
}
else
{
// Clear cache to force to rebuild the agent instruction
Utilities.ClearCache();
// Update session for changed states
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);

12
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.MLTasks.Settings;
using BotSharp.Abstraction.Options;
using BotSharp.Core.Infrastructures;
@ -77,17 +76,6 @@ public class RealtimeHub : IRealtimeHub
var agent = await agentService.LoadAgent(conversation.AgentId);
_conn.CurrentAgentId = agent.Id;
// Set model
var model = "gpt-4o-mini-realtime";
if (agent.Profiles.Contains("realtime"))
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel("openai", "gpt-4o", modelType: LlmModelType.Realtime).Name;
}
_completer.SetModelName(model);
_conn.Model = model;
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.Push(agent.Id);

17
src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs
View file

@ -1,6 +1,4 @@
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
namespace BotSharp.OpenAPI.Controllers;
@ -15,21 +13,6 @@ public class RealtimeController : ControllerBase
_services = services;
}
/// <summary>
/// Create an ephemeral API token for use in client-side applications with the Realtime API.
/// </summary>
/// <returns></returns>
[HttpGet("/agent/{agentId}/realtime/session")]
public async Task<RealtimeSession> CreateSession(string agentId)
{
var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4o");
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);
return await completion.CreateSession(agent, []);
}
[HttpPost("/agent/{agentId}/function/{functionName}/execute")]
public async Task<string> ExecuteFunction(string agentId, string functionName, [FromBody] JsonDocument args)
{

1
src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.GoogleAi.Providers.Chat;
using BotSharp.Plugin.GoogleAI.Providers.Embedding;
using BotSharp.Plugin.GoogleAi.Providers.Realtime;
using BotSharp.Plugin.GoogleAi.Providers.Text;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -6,7 +6,6 @@ using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Types;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
View file

@ -5,7 +5,6 @@ using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using LLMSharp.Google.Palm;
using LLMSharp.Google.Palm.DiscussService;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Plugin.GoogleAi.Providers;
using GenerativeAI;
using GenerativeAI.Types;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAI.Providers.Embedding;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs
View file

@ -1,5 +1,4 @@
using LLMSharp.Google.Palm;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers;

855
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,509 +1,484 @@
using System.Net.WebSockets;
using System.Text.Json.Nodes;
using System.Threading;
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.GoogleAi.Providers.Chat;
using BotSharp.Abstraction.MLTasks.Settings;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
public class GoogleRealTimeProvider : IRealTimeCompletion
{
public class GoogleRealTimeProvider : IRealTimeCompletion
public string Provider => "google-ai";
private string _model = GoogleAIModels.Gemini2FlashExp;
public string Model => _model;
private MultiModalLiveClient _client;
private GenerativeModel _chatClient;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private List<string> renderedInstructions = [];
private readonly GoogleAiSettings _settings;
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings settings,
ILogger<GoogleRealTimeProvider> logger)
{
public string Provider => "google-ai";
private string _model = GoogleAIModels.Gemini2FlashExp;
public string Model { get; }
private MultiModalLiveClient? _client;
private readonly IServiceProvider _services;
private readonly ILogger<GeminiChatCompletionProvider> _logger;
private List<string> renderedInstructions = [];
_settings = settings;
_services = services;
_logger = logger;
}
private readonly GoogleAiSettings _settings;
public void SetModelName(string model)
{
_model = model;
}
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings googleSettings,
ILogger<GeminiChatCompletionProvider> logger)
{
_settings = googleSettings;
_services = services;
_logger = logger;
}
private Action onModelReady;
Action<string, string> onModelAudioDeltaReceived;
private Action onModelAudioResponseDone;
Action<string> onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> onModelResponseDone;
Action<string> onConversationItemCreated;
private Action<RoleDialogModel> onInputAudioTranscriptionCompleted;
Action onUserInterrupted;
RealtimeHubConnection conn;
public void SetModelName(string model)
{
_model = model;
}
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
this.conn = conn;
this.onModelReady = onModelReady;
this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
this.onModelAudioResponseDone = onModelAudioResponseDone;
this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
this.onModelResponseDone = onModelResponseDone;
this.onConversationItemCreated = onConversationItemCreated;
this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
this.onUserInterrupted = onUserInterrupted;
private Action onModelReady;
Action<string, string> onModelAudioDeltaReceived;
private Action onModelAudioResponseDone;
Action<string> onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> onModelResponseDone;
Action<string> onConversationItemCreated;
private Action<RoleDialogModel> onInputAudioTranscriptionCompleted;
Action onUserInterrupted;
RealtimeHubConnection conn;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
_model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name;
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
this.conn = conn;
this.onModelReady = onModelReady;
this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
this.onModelAudioResponseDone = onModelAudioResponseDone;
this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
this.onModelResponseDone = onModelResponseDone;
this.onConversationItemCreated = onConversationItemCreated;
this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
this.onUserInterrupted = onUserInterrupted;
}
public async Task Disconnect()
{
if (_client != null)
await _client.DisconnectAsync();
}
public async Task AppenAudioBuffer(string message)
{
await _client.SendAudioAsync(Convert.FromBase64String(message));
}
public async Task TriggerModelInference(string? instructions = null)
{
await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
_chatClient = client.CreateGenerativeModel(_model);
_client = _chatClient.CreateMultiModalLiveClient(
config: new GenerationConfig
{
TurnComplete = true,
ResponseModalities = [Modality.AUDIO],
},
systemInstruction: "You are a helpful assistant.",
logger: _logger);
await AttachEvents(_client);
await _client.ConnectAsync();
}
public async Task Disconnect()
{
if (_client != null)
await _client.DisconnectAsync();
}
public async Task AppenAudioBuffer(string message)
{
await _client.SendAudioAsync(Convert.FromBase64String(message));
}
public async Task TriggerModelInference(string? instructions = null)
{
await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
{
TurnComplete = true,
});
}
public async Task CancelModelResponse()
{
}
public async Task RemoveConversationItem(string itemId)
{
}
private Task AttachEvents(MultiModalLiveClient client)
{
client.Connected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client connected");
onModelReady();
};
client.Disconnected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client disconnected");
};
client.MessageReceived += async (sender, e) =>
{
if (e.Payload.SetupComplete != null)
{
onConversationItemCreated(_client.ConnectionId.ToString());
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
onModelResponseDone(responseDone);
}
}
};
client.AudioChunkReceived += (sender, e) =>
{
onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
};
client.TextChunkReceived += (sender, e) =>
{
onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
client.GenerationInterrupted += (sender, e) =>
{
onUserInterrupted();
};
client.AudioReceiveCompleted += (sender, e) =>
{
onModelAudioResponseDone();
};
client.ErrorOccurred += (sender, e) =>
{
var ex = e.GetException();
_logger.LogError(ex, "Error occurred in Google Realtime Client");
};
return Task.CompletedTask;
}
private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
BidiGenerateContentServerContent serverContent)
{
var outputs = new List<RoleDialogModel>();
var parts = serverContent.ModelTurn?.Parts;
if (parts != null)
{
foreach (var part in parts)
{
var call = part.FunctionCall;
if (call != null)
{
var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
else
{
var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = call.Name,
FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
ToolCallId = call.Id ?? String.Empty,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
});
}
public async Task CancelModelResponse()
return outputs;
}
public async Task SendEventToModel(object message)
{
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var (prompt, request) = PrepareOptions(_chatClient, agent, new List<RoleDialogModel>());
var config = request.GenerationConfig;
//Output Modality can either be text or audio
if (config != null)
{
}
public async Task RemoveConversationItem(string itemId)
{
}
private async Task AttachEvents()
{
_client.MessageReceived += async (sender, e) =>
{
if (e.Payload.SetupComplete != null)
{
onModelReady();
onConversationItemCreated(_client.ConnectionId.ToString());
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
onModelResponseDone(responseDone);
}
}
};
_client.AudioChunkReceived += async (sender, e) =>
{
onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
};
_client.TextChunkReceived += async (sender, e) =>
{
onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
_client.GenerationInterrupted += async (sender, e) => { onUserInterrupted(); };
_client.AudioReceiveCompleted += async (sender, e) => { onModelAudioResponseDone(); };
}
private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
BidiGenerateContentServerContent serverContent)
{
var outputs = new List<RoleDialogModel>();
var parts = serverContent.ModelTurn?.Parts;
if (parts != null)
{
foreach (var part in parts)
{
var call = part.FunctionCall;
if (call != null)
{
var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
else
{
var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = call.Name,
FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
ToolCallId = call.Id ?? String.Empty,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
});
}
return outputs;
}
public async Task SendEventToModel(object message)
{
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
}
public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
{
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGenerativeModel(_model);
var (prompt, request) = PrepareOptions(chatClient, agent, conversations);
var config = request.GenerationConfig;
//Output Modality can either be text or audio
config.ResponseModalities = new List<Modality>([Modality.AUDIO]);
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
_client = chatClient.CreateMultiModalLiveClient(config,
systemInstruction: request.SystemInstruction?.Parts.FirstOrDefault()?.Text);
_client.UseGoogleSearch = _settings.Gemini.UseGoogleSearch;
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
if (_settings.Gemini.UseGoogleSearch)
config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
var fn = new FunctionDef
{
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools.Add(new Tool()
{
GoogleSearch = new GoogleSearchTool()
});
}
await AttachEvents();
await _client.ConnectAsync();
await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
GenerationConfig = config,
Model = Model,
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
});
return new RealtimeSession()
{
Id = _client.ConnectionId.ToString(),
Model = _model,
Voice = "default"
Name = x.Name ?? string.Empty,
Description = x.Description ?? string.Empty,
};
}
fn.Parameters = x.Parameters != null
? JsonSerializer.Deserialize<FunctionParametersDef>(JsonSerializer.Serialize(x.Parameters))
: null;
return fn;
}).ToArray();
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
await HookEmitter.Emit<IContentGeneratingHook>(_services,
async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
if (_settings.Gemini.UseGoogleSearch)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGenerativeModel(_model);
var (prompt, request) = PrepareOptions(chatClient, agent, new List<RoleDialogModel>());
var config = request.GenerationConfig;
//Output Modality can either be text or audio
if (config != null)
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools.Add(new Tool()
{
config.ResponseModalities = new List<Modality>([Modality.AUDIO]);
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
var fn = new FunctionDef
{
Name = x.Name ?? string.Empty,
Description = x.Description ?? string.Empty,
};
fn.Parameters = x.Parameters != null
? JsonSerializer.Deserialize<FunctionParametersDef>(JsonSerializer.Serialize(x.Parameters))
: null;
return fn;
}).ToArray();
await HookEmitter.Emit<IContentGeneratingHook>(_services,
async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
if (_settings.Gemini.UseGoogleSearch)
{
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools.Add(new Tool()
{
GoogleSearch = new GoogleSearchTool()
});
}
//ToDo: Not sure what's the purpose of UpdateSession, Google Realtime conversion works right away after sending the message!
// await _client.SendSetupAsync(new BidiGenerateContentSetup()
// {
// GenerationConfig = config,
// Model = Model,
// SystemInstruction = request.SystemInstruction,
// Tools = request.Tools?.ToArray(),
// });
return prompt;
GoogleSearch = new GoogleSearchTool()
});
}
public async Task InsertConversationItem(RoleDialogModel message)
await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
if (_client == null)
throw new Exception("Client is not initialized");
if (message.Role == AgentRole.Function)
{
var function = new FunctionResponse()
{
Name = message.FunctionName ?? string.Empty,
Response = JsonNode.Parse(message.Content ?? "{}")
};
GenerationConfig = config,
Model = Model,
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
});
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
{
FunctionResponses = [function]
});
}
else if (message.Role == AgentRole.Assistant)
return prompt;
}
public async Task InsertConversationItem(RoleDialogModel message)
{
if (_client == null)
throw new Exception("Client is not initialized");
if (message.Role == AgentRole.Function)
{
var function = new FunctionResponse()
{
}
else if (message.Role == AgentRole.User)
Name = message.FunctionName ?? string.Empty,
Response = JsonNode.Parse(message.Content ?? "{}")
};
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
{
await _client.SentTextAsync(message.Content);
}
else
{
throw new NotImplementedException("");
}
FunctionResponses = [function]
});
}
public Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
else if (message.Role == AgentRole.Assistant)
{
}
else if (message.Role == AgentRole.User)
{
await _client.SentTextAsync(message.Content);
}
else
{
throw new NotImplementedException("");
}
}
public Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
{
throw new NotImplementedException("");
}
public Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
public Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
{
return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
}
private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var googleSettings = _settings;
renderedInstructions = [];
// Add settings
aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
{
return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
AutoCallFunction = false
};
// Assembly messages
var contents = new List<Content>();
var tools = new List<Tool>();
var funcDeclarations = new List<FunctionDeclaration>();
var systemPrompts = new List<string>();
if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
{
var instruction = agentService.RenderedInstruction(agent);
renderedInstructions.Add(instruction);
systemPrompts.Add(instruction);
}
private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
List<RoleDialogModel> conversations)
var funcPrompts = new List<string>();
var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
foreach (var function in functions)
{
var agentService = _services.GetRequiredService<IAgentService>();
var googleSettings = _settings;
renderedInstructions = [];
if (!agentService.RenderFunction(agent, function)) continue;
// Add settings
aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
var def = agentService.RenderFunctionProperty(agent, function);
var props = JsonSerializer.Serialize(def?.Properties);
var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}"
? new Schema()
{
Type = "object",
Properties = JsonSerializer.Deserialize<Dictionary<string, Schema>>(props),
Required = def?.Required ?? []
}
: null;
aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
funcDeclarations.Add(new FunctionDeclaration
{
AutoCallFunction = false
};
Name = function.Name,
Description = function.Description,
Parameters = parameters
});
// Assembly messages
var contents = new List<Content>();
var tools = new List<Tool>();
var funcDeclarations = new List<FunctionDeclaration>();
funcPrompts.Add($"{function.Name}: {function.Description} {def}");
}
var systemPrompts = new List<string>();
if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
if (!funcDeclarations.IsNullOrEmpty())
{
tools.Add(new Tool { FunctionDeclarations = funcDeclarations });
}
var convPrompts = new List<string>();
foreach (var message in conversations)
{
if (message.Role == AgentRole.Function)
{
var instruction = agentService.RenderedInstruction(agent);
renderedInstructions.Add(instruction);
systemPrompts.Add(instruction);
}
var funcPrompts = new List<string>();
var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
foreach (var function in functions)
{
if (!agentService.RenderFunction(agent, function)) continue;
var def = agentService.RenderFunctionProperty(agent, function);
var props = JsonSerializer.Serialize(def?.Properties);
var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}"
? new Schema()
contents.Add(new Content([
new Part()
{
Type = "object",
Properties = JsonSerializer.Deserialize<Dictionary<string, Schema>>(props),
Required = def?.Required ?? []
FunctionCall = new FunctionCall
{
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
}
}
: null;
], AgentRole.Model));
funcDeclarations.Add(new FunctionDeclaration
{
Name = function.Name,
Description = function.Description,
Parameters = parameters
});
funcPrompts.Add($"{function.Name}: {function.Description} {def}");
}
if (!funcDeclarations.IsNullOrEmpty())
{
tools.Add(new Tool { FunctionDeclarations = funcDeclarations });
}
var convPrompts = new List<string>();
foreach (var message in conversations)
{
if (message.Role == AgentRole.Function)
{
contents.Add(new Content([
new Part()
contents.Add(new Content([
new Part()
{
FunctionResponse = new FunctionResponse
{
FunctionCall = new FunctionCall
Name = message.FunctionName ?? string.Empty,
Response = new JsonObject()
{
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
["result"] = message.Content ?? string.Empty
}
}
], AgentRole.Model));
}
], AgentRole.Function));
contents.Add(new Content([
new Part()
{
FunctionResponse = new FunctionResponse
{
Name = message.FunctionName ?? string.Empty,
Response = new JsonObject()
{
["result"] = message.Content ?? string.Empty
}
}
}
], AgentRole.Function));
convPrompts.Add(
$"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
contents.Add(new Content(text, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)
{
contents.Add(new Content(message.Content, AgentRole.Model));
convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
}
convPrompts.Add(
$"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
else if (message.Role == AgentRole.User)
{
SystemInstruction = !systemPrompts.IsNullOrEmpty()
? new Content(systemPrompts[0], AgentRole.System)
: null,
Contents = contents,
Tools = tools,
GenerationConfig = new()
{
Temperature = temperature,
MaxOutputTokens = maxTokens
}
};
var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
return (prompt, request);
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
contents.Add(new Content(text, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)
{
contents.Add(new Content(message.Content, AgentRole.Model));
convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
}
}
private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string> funcPrompts,
IEnumerable<string> convPrompts)
var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
{
string prompt = string.Join("\r\n\r\n", systemPrompts);
if (!funcPrompts.IsNullOrEmpty())
SystemInstruction = !systemPrompts.IsNullOrEmpty()
? new Content(systemPrompts[0], AgentRole.System)
: null,
Contents = contents,
Tools = tools,
GenerationConfig = new()
{
prompt += "\r\n\r\n[FUNCTIONS]\r\n";
prompt += string.Join("\r\n", funcPrompts);
Temperature = temperature,
MaxOutputTokens = maxTokens
}
};
if (!convPrompts.IsNullOrEmpty())
{
prompt += "\r\n\r\n[CONVERSATION]\r\n";
prompt += string.Join("\r\n", convPrompts);
}
var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
return (prompt, request);
}
return prompt;
private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string> funcPrompts,
IEnumerable<string> convPrompts)
{
string prompt = string.Join("\r\n\r\n", systemPrompts);
if (!funcPrompts.IsNullOrEmpty())
{
prompt += "\r\n\r\n[FUNCTIONS]\r\n";
prompt += string.Join("\r\n", funcPrompts);
}
if (!convPrompts.IsNullOrEmpty())
{
prompt += "\r\n\r\n[CONVERSATION]\r\n";
prompt += string.Join("\r\n", convPrompts);
}
return prompt;
}
}

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
View file

@ -3,7 +3,6 @@ using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;

22
src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
View file

@ -4,12 +4,26 @@ global using System.Text;
global using System.Threading.Tasks;
global using System.Linq;
global using System.Text.Json;
global using System.Text.Json.Serialization;
global using System.Text.Json.Nodes;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using System.Text.Json.Serialization;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Abstraction.Realtime;
global using BotSharp.Abstraction.Realtime.Models;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.GoogleAi.Providers.Chat;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Enums;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Loggers;

2
src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj
View file

@ -12,8 +12,6 @@
<ItemGroup>
<PackageReference Include="OpenAI" />
<PackageReference Include="Refit" />
<PackageReference Include="Refit.HttpClientFactory" />
</ItemGroup>
<ItemGroup>

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Functions.Models;
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class RealtimeSessionBody

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Functions.Models;
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class RealtimeSessionCreationRequest

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Realtime.Models;
namespace BotSharp.Plugin.OpenAI.Models.Realtime;
public class RealtimeSessionUpdate

6
src/Plugins/BotSharp.Plugin.OpenAI/Models/SpeechToTextRequest.cs
View file

@ -1,9 +1,3 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Plugin.OpenAI.Models;
public class SpeechToTextRequest

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/TextCompletionRequest.cs
View file

@ -1,5 +1,3 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.OpenAI.Models;
public class TextCompletionRequest

2
src/Plugins/BotSharp.Plugin.OpenAI/Models/TextCompletionResponse.cs
View file

@ -1,5 +1,3 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.OpenAI.Models;
public class TextCompletionResponse

4
src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs
View file

@ -6,7 +6,6 @@ using BotSharp.Plugin.OpenAI.Providers.Text;
using BotSharp.Plugin.OpenAI.Providers.Chat;
using BotSharp.Plugin.OpenAI.Providers.Audio;
using Microsoft.Extensions.Configuration;
using Refit;
using BotSharp.Plugin.OpenAI.Providers.Realtime;
namespace BotSharp.Plugin.OpenAI;
@ -36,8 +35,5 @@ public class OpenAiPlugin : IBotSharpPlugin
services.AddScoped<IAudioTranscription, AudioTranscriptionProvider>();
services.AddScoped<IAudioSynthesis, AudioSynthesisProvider>();
services.AddScoped<IRealTimeCompletion, RealTimeCompletionProvider>();
services.AddRefitClient<IOpenAiRealtimeApi>()
.ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com"));
}
}

1
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Files.Utilities;
using OpenAI.Chat;
namespace BotSharp.Plugin.OpenAI.Providers.Chat;

11
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs
View file

@ -1,11 +0,0 @@
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using Refit;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
public interface IOpenAiRealtimeApi
{
[Post("/v1/realtime/sessions")]
Task<RealtimeSession> CreateSessionAsync(RealtimeSessionCreationRequest model, [Authorize("Bearer")] string token);
}

55
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,17 +1,6 @@
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.OpenAI.Models.Realtime;
using OpenAI.Chat;
using System.Net.WebSockets;
using System.Text;
using System.Text.Json;
using System.Threading;
namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
@ -27,7 +16,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
protected readonly IServiceProvider _services;
protected readonly ILogger<RealTimeCompletionProvider> _logger;
protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17";
protected string _model = "gpt-4o-mini-realtime-preview";
private ClientWebSocket _webSocket;
public RealTimeCompletionProvider(
@ -50,14 +39,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
_model = llmProviderService.GetProviderModel(Provider, "gpt-4o", modelType: LlmModelType.Realtime).Name;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(provider: "openai", conn.Model);
var settings = settingsService.GetSetting(Provider, _model);
_webSocket = new ClientWebSocket();
_webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}");
_webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");
await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={conn.Model}"), CancellationToken.None);
await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={_model}"), CancellationToken.None);
if (_webSocket.State == WebSocketState.Open)
{
@ -284,41 +276,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
await _webSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
}
public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
var client = ProviderHelper.GetClient(Provider, _model, _services);
var chatClient = client.GetChatClient(_model);
var (prompt, messages, options) = PrepareOptions(agent, conversations);
var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description;
var args = new RealtimeSessionCreationRequest
{
Model = _model,
Instructions = instruction,
ToolChoice = "auto",
Tools = options.Tools.Select(x =>
{
var fn = new FunctionDef
{
Name = x.FunctionName,
Description = x.FunctionDescription
};
fn.Parameters = JsonSerializer.Deserialize<FunctionParametersDef>(x.FunctionParameters);
return fn;
}).ToArray(),
};
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, args.Model ?? _model);
var api = _services.GetRequiredService<IOpenAiRealtimeApi>();
var session = await api.CreateSessionAsync(args, settings.ApiKey);
return session;
}
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
{
var convService = _services.GetRequiredService<IConversationService>();

4
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs
View file

@ -1,9 +1,5 @@
using BotSharp.Abstraction.MLTasks.Settings;
using System.Net.Http;
using System.Net.Mime;
using System.Text.Json;
using System.Text;
using BotSharp.Abstraction.Options;
namespace BotSharp.Plugin.OpenAI.Providers.Text;

11
src/Plugins/BotSharp.Plugin.OpenAI/Using.cs
View file

@ -4,6 +4,9 @@ global using System.Linq;
global using System.IO;
global using System.Threading.Tasks;
global using System.Text.Json.Serialization;
global using System.Text;
global using System.Text.Json;
global using System.Threading;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
@ -19,5 +22,13 @@ global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Conversations.Enums;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.MLTasks.Settings;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Realtime;
global using BotSharp.Abstraction.Realtime.Models;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.OpenAI.Models;
global using BotSharp.Plugin.OpenAI.Settings;

3
tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs
View file

@ -1,4 +1,4 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.MLTasks;
@ -45,7 +45,6 @@ namespace BotSharp.Plugin.Google.Core
(list => { Console.WriteLine(list); }),
(s => { Console.WriteLine(s); }),
(model => { Console.WriteLine(model); }), (() => { Console.WriteLine("UserInterrupted"); }));
var session = await realTimeCompleter.CreateSession(agent, new List<RoleDialogModel>());
Thread.Sleep(1000);
modelReady.ShouldBeTrue();