diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
index efaec3a6..66b7e24c 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
@@ -11,6 +11,8 @@
+
+
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs
index c97283ce..7b311886 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs
@@ -1,6 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
-using BotSharp.Plugin.GoogleAi.Providers.Chat;
using BotSharp.Plugin.GoogleAI.Providers.Embedding;
using BotSharp.Plugin.GoogleAi.Providers.Realtime;
using BotSharp.Plugin.GoogleAi.Providers.Text;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
index 3409121a..d296d0aa 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
@@ -6,7 +6,6 @@ using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Types;
-using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
index 91e797cc..de7e316f 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
@@ -5,7 +5,6 @@ using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using LLMSharp.Google.Palm;
using LLMSharp.Google.Palm.DiscussService;
-using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs
index 0f8d823e..f61f69d6 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs
@@ -1,7 +1,6 @@
using BotSharp.Plugin.GoogleAi.Providers;
using GenerativeAI;
using GenerativeAI.Types;
-using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAI.Providers.Embedding;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs
index 5f7e1312..28b4a03b 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs
@@ -1,5 +1,4 @@
using LLMSharp.Google.Palm;
-using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
index bbda17db..3d7fcc97 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
@@ -1,509 +1,509 @@
-using System.Net.WebSockets;
-using System.Text.Json.Nodes;
-using System.Threading;
-using BotSharp.Abstraction.Agents;
-using BotSharp.Abstraction.Agents.Enums;
-using BotSharp.Abstraction.Conversations;
-using BotSharp.Abstraction.Conversations.Enums;
-using BotSharp.Abstraction.Files;
-using BotSharp.Abstraction.Files.Utilities;
-using BotSharp.Abstraction.Functions.Models;
-using BotSharp.Abstraction.Loggers;
-using BotSharp.Abstraction.Options;
-using BotSharp.Abstraction.Realtime;
-using BotSharp.Abstraction.Realtime.Models;
-using BotSharp.Core.Infrastructures;
-using BotSharp.Plugin.GoogleAi.Providers.Chat;
+using BotSharp.Abstraction.MLTasks.Settings;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
-using Microsoft.Extensions.Logging;
-namespace BotSharp.Plugin.GoogleAi.Providers.Realtime
+namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
+
+public class GoogleRealTimeProvider : IRealTimeCompletion
{
- public class GoogleRealTimeProvider : IRealTimeCompletion
+ public string Provider => "google-ai";
+ private string _model = GoogleAIModels.Gemini2FlashExp;
+ public string Model => _model;
+ private MultiModalLiveClient _client;
+ private GenerativeModel _chatClient;
+ private readonly IServiceProvider _services;
+ private readonly ILogger _logger;
+ private List renderedInstructions = [];
+
+ private readonly GoogleAiSettings _settings;
+
+ public GoogleRealTimeProvider(
+ IServiceProvider services,
+ GoogleAiSettings settings,
+ ILogger logger)
{
- public string Provider => "google-ai";
- private string _model = GoogleAIModels.Gemini2FlashExp;
- public string Model { get; }
- private MultiModalLiveClient? _client;
- private readonly IServiceProvider _services;
- private readonly ILogger _logger;
- private List renderedInstructions = [];
+ _settings = settings;
+ _services = services;
+ _logger = logger;
+ }
- private readonly GoogleAiSettings _settings;
+ public void SetModelName(string model)
+ {
+ _model = model;
+ }
- public GoogleRealTimeProvider(
- IServiceProvider services,
- GoogleAiSettings googleSettings,
- ILogger logger)
- {
- _settings = googleSettings;
- _services = services;
- _logger = logger;
- }
+ private Action onModelReady;
+ Action onModelAudioDeltaReceived;
+ private Action onModelAudioResponseDone;
+ Action onModelAudioTranscriptDone;
+ private Action> onModelResponseDone;
+ Action onConversationItemCreated;
+ private Action onInputAudioTranscriptionCompleted;
+ Action onUserInterrupted;
+ RealtimeHubConnection conn;
- public void SetModelName(string model)
- {
- _model = model;
- }
+ public async Task Connect(RealtimeHubConnection conn,
+ Action onModelReady,
+ Action onModelAudioDeltaReceived,
+ Action onModelAudioResponseDone,
+ Action onModelAudioTranscriptDone,
+ Action> onModelResponseDone,
+ Action onConversationItemCreated,
+ Action onInputAudioTranscriptionCompleted,
+ Action onUserInterrupted)
+ {
+ this.conn = conn;
+ this.onModelReady = onModelReady;
+ this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
+ this.onModelAudioResponseDone = onModelAudioResponseDone;
+ this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
+ this.onModelResponseDone = onModelResponseDone;
+ this.onConversationItemCreated = onConversationItemCreated;
+ this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
+ this.onUserInterrupted = onUserInterrupted;
- private Action onModelReady;
- Action onModelAudioDeltaReceived;
- private Action onModelAudioResponseDone;
- Action onModelAudioTranscriptDone;
- private Action> onModelResponseDone;
- Action onConversationItemCreated;
- private Action onInputAudioTranscriptionCompleted;
- Action onUserInterrupted;
- RealtimeHubConnection conn;
+ var llmProviderService = _services.GetRequiredService();
+ _model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name;
- public async Task Connect(RealtimeHubConnection conn,
- Action onModelReady,
- Action onModelAudioDeltaReceived,
- Action onModelAudioResponseDone,
- Action onModelAudioTranscriptDone,
- Action> onModelResponseDone,
- Action onConversationItemCreated,
- Action onInputAudioTranscriptionCompleted,
- Action onUserInterrupted)
- {
- this.conn = conn;
- this.onModelReady = onModelReady;
- this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
- this.onModelAudioResponseDone = onModelAudioResponseDone;
- this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
- this.onModelResponseDone = onModelResponseDone;
- this.onConversationItemCreated = onConversationItemCreated;
- this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
- this.onUserInterrupted = onUserInterrupted;
- }
-
- public async Task Disconnect()
- {
- if (_client != null)
- await _client.DisconnectAsync();
- }
-
- public async Task AppenAudioBuffer(string message)
- {
- await _client.SendAudioAsync(Convert.FromBase64String(message));
- }
-
- public async Task TriggerModelInference(string? instructions = null)
- {
- await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
+ var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
+ _chatClient = client.CreateGenerativeModel(_model);
+ _client = _chatClient.CreateMultiModalLiveClient(
+ config: new GenerationConfig
{
- TurnComplete = true,
+ ResponseModalities = [Modality.AUDIO],
+ },
+ systemInstruction: "You are a helpful assistant.",
+ logger: _logger);
+
+ await AttachEvents(_client);
+
+ await _client.ConnectAsync();
+ }
+
+ public async Task Disconnect()
+ {
+ if (_client != null)
+ await _client.DisconnectAsync();
+ }
+
+ public async Task AppenAudioBuffer(string message)
+ {
+ await _client.SendAudioAsync(Convert.FromBase64String(message));
+ }
+
+ public async Task TriggerModelInference(string? instructions = null)
+ {
+ await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
+ {
+ TurnComplete = true,
+ });
+ }
+
+ public async Task CancelModelResponse()
+ {
+ }
+
+ public async Task RemoveConversationItem(string itemId)
+ {
+ }
+
+ private Task AttachEvents(MultiModalLiveClient client)
+ {
+ client.Connected += (sender, e) =>
+ {
+ _logger.LogInformation("Google Realtime Client connected");
+ onModelReady();
+ };
+
+ client.Disconnected += (sender, e) =>
+ {
+ _logger.LogInformation("Google Realtime Client disconnected");
+ };
+
+ client.MessageReceived += async (sender, e) =>
+ {
+ if (e.Payload.SetupComplete != null)
+ {
+ onConversationItemCreated(_client.ConnectionId.ToString());
+ }
+
+ if (e.Payload.ServerContent != null)
+ {
+ if (e.Payload.ServerContent.TurnComplete == true)
+ {
+ var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
+ onModelResponseDone(responseDone);
+ }
+ }
+ };
+
+ client.AudioChunkReceived += (sender, e) =>
+ {
+ onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
+ };
+
+ client.TextChunkReceived += (sender, e) =>
+ {
+ onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
+ };
+
+ client.GenerationInterrupted += (sender, e) =>
+ {
+ onUserInterrupted();
+ };
+
+ client.AudioReceiveCompleted += (sender, e) =>
+ {
+ onModelAudioResponseDone();
+ };
+
+ client.ErrorOccurred += (sender, e) =>
+ {
+ var ex = e.GetException();
+ _logger.LogError(ex, "Error occurred in Google Realtime Client");
+ };
+
+ return Task.CompletedTask;
+ }
+
+ private async Task> ResponseDone(RealtimeHubConnection conn,
+ BidiGenerateContentServerContent serverContent)
+ {
+ var outputs = new List();
+
+ var parts = serverContent.ModelTurn?.Parts;
+ if (parts != null)
+ {
+ foreach (var part in parts)
+ {
+ var call = part.FunctionCall;
+ if (call != null)
+ {
+ var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
+ {
+ CurrentAgentId = conn.CurrentAgentId,
+ MessageId = call.Id ?? String.Empty,
+ MessageType = MessageTypeName.FunctionCall
+ };
+ outputs.Add(item);
+ }
+ else
+ {
+ var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
+ {
+ CurrentAgentId = conn.CurrentAgentId,
+ FunctionName = call.Name,
+ FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
+ ToolCallId = call.Id ?? String.Empty,
+ MessageId = call.Id ?? String.Empty,
+ MessageType = MessageTypeName.FunctionCall
+ };
+ outputs.Add(item);
+ }
+ }
+ }
+
+ var contentHooks = _services.GetServices().ToList();
+ // After chat completion hook
+ foreach (var hook in contentHooks)
+ {
+ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
+ {
+ CurrentAgentId = conn.CurrentAgentId
+ }, new TokenStatsModel
+ {
+ Provider = Provider,
+ Model = _model,
});
}
- public async Task CancelModelResponse()
+ return outputs;
+ }
+
+ public async Task SendEventToModel(object message)
+ {
+ //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
+ }
+
+ public async Task CreateSession(Agent agent, List conversations)
+ {
+ var (prompt, request) = PrepareOptions(_chatClient, agent, conversations);
+
+ var config = request.GenerationConfig;
+
+ // Output Modality can either be text or audio
+ config.ResponseModalities = new List([Modality.AUDIO]);
+
+ await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
- }
-
- public async Task RemoveConversationItem(string itemId)
+ GenerationConfig = config,
+ Model = Model,
+ SystemInstruction = request.SystemInstruction,
+ Tools = request.Tools?.ToArray(),
+ });
+
+ return new RealtimeSession()
{
- }
+ Id = _client.ConnectionId.ToString(),
+ Model = _model,
+ Voice = "default"
+ };
+ }
- private async Task AttachEvents()
+ public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
+ {
+ var convService = _services.GetRequiredService();
+ var conv = await convService.GetConversation(conn.ConversationId);
+
+ var agentService = _services.GetRequiredService();
+ var agent = await agentService.LoadAgent(conn.CurrentAgentId);
+
+ var (prompt, request) = PrepareOptions(_chatClient, agent, new List());
+
+ var config = request.GenerationConfig;
+ //Output Modality can either be text or audio
+ if (config != null)
{
- _client.MessageReceived += async (sender, e) =>
- {
- if (e.Payload.SetupComplete != null)
- {
- onModelReady();
- onConversationItemCreated(_client.ConnectionId.ToString());
- }
-
- if (e.Payload.ServerContent != null)
- {
- if (e.Payload.ServerContent.TurnComplete == true)
- {
- var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
- onModelResponseDone(responseDone);
- }
- }
- };
- _client.AudioChunkReceived += async (sender, e) =>
- {
- onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
- };
-
- _client.TextChunkReceived += async (sender, e) =>
- {
- onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
- };
- _client.GenerationInterrupted += async (sender, e) => { onUserInterrupted(); };
- _client.AudioReceiveCompleted += async (sender, e) => { onModelAudioResponseDone(); };
- }
-
- private async Task> ResponseDone(RealtimeHubConnection conn,
- BidiGenerateContentServerContent serverContent)
- {
- var outputs = new List();
-
- var parts = serverContent.ModelTurn?.Parts;
- if (parts != null)
- {
- foreach (var part in parts)
- {
- var call = part.FunctionCall;
- if (call != null)
- {
- var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
- {
- CurrentAgentId = conn.CurrentAgentId,
- MessageId = call.Id ?? String.Empty,
- MessageType = MessageTypeName.FunctionCall
- };
- outputs.Add(item);
- }
- else
- {
- var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
- {
- CurrentAgentId = conn.CurrentAgentId,
- FunctionName = call.Name,
- FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
- ToolCallId = call.Id ?? String.Empty,
- MessageId = call.Id ?? String.Empty,
- MessageType = MessageTypeName.FunctionCall
- };
- outputs.Add(item);
- }
- }
- }
-
- var contentHooks = _services.GetServices().ToList();
- // After chat completion hook
- foreach (var hook in contentHooks)
- {
- await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
- {
- CurrentAgentId = conn.CurrentAgentId
- }, new TokenStatsModel
- {
- Provider = Provider,
- Model = _model,
- });
- }
-
- return outputs;
- }
-
- public async Task SendEventToModel(object message)
- {
- //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
- }
-
- public async Task CreateSession(Agent agent, List conversations)
- {
- var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
- var chatClient = client.CreateGenerativeModel(_model);
- var (prompt, request) = PrepareOptions(chatClient, agent, conversations);
-
- var config = request.GenerationConfig;
-
- //Output Modality can either be text or audio
config.ResponseModalities = new List([Modality.AUDIO]);
- var settingsService = _services.GetRequiredService();
- var settings = settingsService.GetSetting(Provider, _model);
+ var words = new List();
+ HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
- _client = chatClient.CreateMultiModalLiveClient(config,
- systemInstruction: request.SystemInstruction?.Parts.FirstOrDefault()?.Text);
- _client.UseGoogleSearch = _settings.Gemini.UseGoogleSearch;
+ var realtimeModelSettings = _services.GetRequiredService();
- if (_settings.Gemini.UseGoogleSearch)
+ config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
+ config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
+ }
+
+
+ var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
+ {
+ var fn = new FunctionDef
{
- if (request.Tools == null)
- request.Tools = new List();
- request.Tools.Add(new Tool()
- {
- GoogleSearch = new GoogleSearchTool()
- });
- }
-
- await AttachEvents();
-
- await _client.ConnectAsync();
-
- await _client.SendSetupAsync(new BidiGenerateContentSetup()
- {
- GenerationConfig = config,
- Model = Model,
- SystemInstruction = request.SystemInstruction,
- Tools = request.Tools?.ToArray(),
- });
-
- return new RealtimeSession()
- {
- Id = _client.ConnectionId.ToString(),
- Model = _model,
- Voice = "default"
+ Name = x.Name ?? string.Empty,
+ Description = x.Description ?? string.Empty,
};
- }
+ fn.Parameters = x.Parameters != null
+ ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters))
+ : null;
+ return fn;
+ }).ToArray();
- public async Task UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
+ await HookEmitter.Emit(_services,
+ async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
+
+ if (_settings.Gemini.UseGoogleSearch)
{
- var convService = _services.GetRequiredService();
- var conv = await convService.GetConversation(conn.ConversationId);
-
- var agentService = _services.GetRequiredService();
- var agent = await agentService.LoadAgent(conn.CurrentAgentId);
-
- var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
- var chatClient = client.CreateGenerativeModel(_model);
- var (prompt, request) = PrepareOptions(chatClient, agent, new List());
-
-
- var config = request.GenerationConfig;
- //Output Modality can either be text or audio
- if (config != null)
+ if (request.Tools == null)
+ request.Tools = new List();
+ request.Tools.Add(new Tool()
{
- config.ResponseModalities = new List([Modality.AUDIO]);
-
- var words = new List();
- HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
-
- var realtimeModelSettings = _services.GetRequiredService();
-
- config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
- config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
- }
-
-
- var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
- {
- var fn = new FunctionDef
- {
- Name = x.Name ?? string.Empty,
- Description = x.Description ?? string.Empty,
- };
- fn.Parameters = x.Parameters != null
- ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters))
- : null;
- return fn;
- }).ToArray();
-
- await HookEmitter.Emit(_services,
- async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
-
- if (_settings.Gemini.UseGoogleSearch)
- {
- if (request.Tools == null)
- request.Tools = new List();
- request.Tools.Add(new Tool()
- {
- GoogleSearch = new GoogleSearchTool()
- });
- }
-
- //ToDo: Not sure what's the purpose of UpdateSession, Google Realtime conversion works right away after sending the message!
-
- // await _client.SendSetupAsync(new BidiGenerateContentSetup()
- // {
- // GenerationConfig = config,
- // Model = Model,
- // SystemInstruction = request.SystemInstruction,
- // Tools = request.Tools?.ToArray(),
- // });
-
- return prompt;
+ GoogleSearch = new GoogleSearchTool()
+ });
}
- public async Task InsertConversationItem(RoleDialogModel message)
+ await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
- if (_client == null)
- throw new Exception("Client is not initialized");
- if (message.Role == AgentRole.Function)
- {
- var function = new FunctionResponse()
- {
- Name = message.FunctionName ?? string.Empty,
- Response = JsonNode.Parse(message.Content ?? "{}")
- };
+ GenerationConfig = config,
+ Model = Model,
+ SystemInstruction = request.SystemInstruction,
+ Tools = request.Tools?.ToArray(),
+ });
- await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
- {
- FunctionResponses = [function]
- });
- }
- else if (message.Role == AgentRole.Assistant)
+ return prompt;
+ }
+
+ public async Task InsertConversationItem(RoleDialogModel message)
+ {
+ if (_client == null)
+ throw new Exception("Client is not initialized");
+ if (message.Role == AgentRole.Function)
+ {
+ var function = new FunctionResponse()
{
- }
- else if (message.Role == AgentRole.User)
+ Name = message.FunctionName ?? string.Empty,
+ Response = JsonNode.Parse(message.Content ?? "{}")
+ };
+
+ await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
{
- await _client.SentTextAsync(message.Content);
- }
- else
- {
- throw new NotImplementedException("");
- }
+ FunctionResponses = [function]
+ });
}
-
- public Task> OnResponsedDone(RealtimeHubConnection conn, string response)
+ else if (message.Role == AgentRole.Assistant)
+ {
+ }
+ else if (message.Role == AgentRole.User)
+ {
+ await _client.SentTextAsync(message.Content);
+ }
+ else
{
throw new NotImplementedException("");
}
+ }
+
+ public Task> OnResponsedDone(RealtimeHubConnection conn, string response)
+ {
+ throw new NotImplementedException("");
+ }
- public Task OnConversationItemCreated(RealtimeHubConnection conn, string response)
+ public Task OnConversationItemCreated(RealtimeHubConnection conn, string response)
+ {
+ return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
+ }
+
+ private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
+ List conversations)
+ {
+ var agentService = _services.GetRequiredService();
+ var googleSettings = _settings;
+ renderedInstructions = [];
+
+ // Add settings
+ aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
+ aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
+
+ aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
{
- return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
+ AutoCallFunction = false
+ };
+
+ // Assembly messages
+ var contents = new List();
+ var tools = new List();
+ var funcDeclarations = new List();
+
+ var systemPrompts = new List();
+ if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
+ {
+ var instruction = agentService.RenderedInstruction(agent);
+ renderedInstructions.Add(instruction);
+ systemPrompts.Add(instruction);
}
- private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
- List conversations)
+ var funcPrompts = new List();
+ var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
+ foreach (var function in functions)
{
- var agentService = _services.GetRequiredService();
- var googleSettings = _settings;
- renderedInstructions = [];
+ if (!agentService.RenderFunction(agent, function)) continue;
- // Add settings
- aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
- aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
+ var def = agentService.RenderFunctionProperty(agent, function);
+ var props = JsonSerializer.Serialize(def?.Properties);
+ var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}"
+ ? new Schema()
+ {
+ Type = "object",
+ Properties = JsonSerializer.Deserialize>(props),
+ Required = def?.Required ?? []
+ }
+ : null;
- aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
+ funcDeclarations.Add(new FunctionDeclaration
{
- AutoCallFunction = false
- };
+ Name = function.Name,
+ Description = function.Description,
+ Parameters = parameters
+ });
- // Assembly messages
- var contents = new List();
- var tools = new List();
- var funcDeclarations = new List();
+ funcPrompts.Add($"{function.Name}: {function.Description} {def}");
+ }
- var systemPrompts = new List();
- if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
+ if (!funcDeclarations.IsNullOrEmpty())
+ {
+ tools.Add(new Tool { FunctionDeclarations = funcDeclarations });
+ }
+
+ var convPrompts = new List();
+ foreach (var message in conversations)
+ {
+ if (message.Role == AgentRole.Function)
{
- var instruction = agentService.RenderedInstruction(agent);
- renderedInstructions.Add(instruction);
- systemPrompts.Add(instruction);
- }
-
- var funcPrompts = new List();
- var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
- foreach (var function in functions)
- {
- if (!agentService.RenderFunction(agent, function)) continue;
-
- var def = agentService.RenderFunctionProperty(agent, function);
- var props = JsonSerializer.Serialize(def?.Properties);
- var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}"
- ? new Schema()
+ contents.Add(new Content([
+ new Part()
{
- Type = "object",
- Properties = JsonSerializer.Deserialize>(props),
- Required = def?.Required ?? []
+ FunctionCall = new FunctionCall
+ {
+ Name = message.FunctionName,
+ Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
+ }
}
- : null;
+ ], AgentRole.Model));
- funcDeclarations.Add(new FunctionDeclaration
- {
- Name = function.Name,
- Description = function.Description,
- Parameters = parameters
- });
-
- funcPrompts.Add($"{function.Name}: {function.Description} {def}");
- }
-
- if (!funcDeclarations.IsNullOrEmpty())
- {
- tools.Add(new Tool { FunctionDeclarations = funcDeclarations });
- }
-
- var convPrompts = new List();
- foreach (var message in conversations)
- {
- if (message.Role == AgentRole.Function)
- {
- contents.Add(new Content([
- new Part()
+ contents.Add(new Content([
+ new Part()
+ {
+ FunctionResponse = new FunctionResponse
{
- FunctionCall = new FunctionCall
+ Name = message.FunctionName ?? string.Empty,
+ Response = new JsonObject()
{
- Name = message.FunctionName,
- Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
+ ["result"] = message.Content ?? string.Empty
}
}
- ], AgentRole.Model));
+ }
+ ], AgentRole.Function));
- contents.Add(new Content([
- new Part()
- {
- FunctionResponse = new FunctionResponse
- {
- Name = message.FunctionName ?? string.Empty,
- Response = new JsonObject()
- {
- ["result"] = message.Content ?? string.Empty
- }
- }
- }
- ], AgentRole.Function));
-
- convPrompts.Add(
- $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
- }
- else if (message.Role == AgentRole.User)
- {
- var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
- contents.Add(new Content(text, AgentRole.User));
- convPrompts.Add($"{AgentRole.User}: {text}");
- }
- else if (message.Role == AgentRole.Assistant)
- {
- contents.Add(new Content(message.Content, AgentRole.Model));
- convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
- }
+ convPrompts.Add(
+ $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
-
- var state = _services.GetRequiredService();
- var temperature = float.Parse(state.GetState("temperature", "0.0"));
- var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
- ? tokens
- : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
- var request = new GenerateContentRequest
+ else if (message.Role == AgentRole.User)
{
- SystemInstruction = !systemPrompts.IsNullOrEmpty()
- ? new Content(systemPrompts[0], AgentRole.System)
- : null,
- Contents = contents,
- Tools = tools,
- GenerationConfig = new()
- {
- Temperature = temperature,
- MaxOutputTokens = maxTokens
- }
- };
-
- var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
- return (prompt, request);
+ var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+ contents.Add(new Content(text, AgentRole.User));
+ convPrompts.Add($"{AgentRole.User}: {text}");
+ }
+ else if (message.Role == AgentRole.Assistant)
+ {
+ contents.Add(new Content(message.Content, AgentRole.Model));
+ convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
+ }
}
- private string GetPrompt(IEnumerable systemPrompts, IEnumerable funcPrompts,
- IEnumerable convPrompts)
+ var state = _services.GetRequiredService();
+ var temperature = float.Parse(state.GetState("temperature", "0.0"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
+ var request = new GenerateContentRequest
{
- string prompt = string.Join("\r\n\r\n", systemPrompts);
-
- if (!funcPrompts.IsNullOrEmpty())
+ SystemInstruction = !systemPrompts.IsNullOrEmpty()
+ ? new Content(systemPrompts[0], AgentRole.System)
+ : null,
+ Contents = contents,
+ Tools = tools,
+ GenerationConfig = new()
{
- prompt += "\r\n\r\n[FUNCTIONS]\r\n";
- prompt += string.Join("\r\n", funcPrompts);
+ Temperature = temperature,
+ MaxOutputTokens = maxTokens
}
+ };
- if (!convPrompts.IsNullOrEmpty())
- {
- prompt += "\r\n\r\n[CONVERSATION]\r\n";
- prompt += string.Join("\r\n", convPrompts);
- }
+ var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
+ return (prompt, request);
+ }
- return prompt;
+ private string GetPrompt(IEnumerable systemPrompts, IEnumerable funcPrompts,
+ IEnumerable convPrompts)
+ {
+ string prompt = string.Join("\r\n\r\n", systemPrompts);
+
+ if (!funcPrompts.IsNullOrEmpty())
+ {
+ prompt += "\r\n\r\n[FUNCTIONS]\r\n";
+ prompt += string.Join("\r\n", funcPrompts);
}
+
+ if (!convPrompts.IsNullOrEmpty())
+ {
+ prompt += "\r\n\r\n[CONVERSATION]\r\n";
+ prompt += string.Join("\r\n", convPrompts);
+ }
+
+ return prompt;
}
}
\ No newline at end of file
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
index 7d4173d6..7047fbdc 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
@@ -3,7 +3,6 @@ using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
-using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
index 7df18bb7..ea8ecd5e 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
@@ -1,7 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
-using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
index c7f6c176..a4e6606e 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
@@ -4,12 +4,26 @@ global using System.Text;
global using System.Threading.Tasks;
global using System.Linq;
global using System.Text.Json;
+global using System.Text.Json.Serialization;
+global using System.Text.Json.Nodes;
+
+global using Microsoft.Extensions.Logging;
+global using Microsoft.Extensions.Configuration;
+global using Microsoft.Extensions.DependencyInjection;
+
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
-global using Microsoft.Extensions.Configuration;
-global using Microsoft.Extensions.DependencyInjection;
-global using System.Text.Json.Serialization;
global using BotSharp.Abstraction.Utilities;
-global using BotSharp.Plugin.GoogleAi.Settings;
\ No newline at end of file
+global using BotSharp.Plugin.GoogleAi.Settings;
+global using BotSharp.Abstraction.Realtime;
+global using BotSharp.Abstraction.Realtime.Models;
+global using BotSharp.Core.Infrastructures;
+global using BotSharp.Plugin.GoogleAi.Providers.Chat;
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Conversations.Enums;
+global using BotSharp.Abstraction.Functions.Models;
+global using BotSharp.Abstraction.Loggers;
\ No newline at end of file