SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Clean code for GeminiLive

Browse source
This commit is contained in:
Haiping Chen 2025-04-05 15:22:06 -05:00
parent 18c6598af3
commit 6e482e966b
10 changed files with 457 additions and 448 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj
View file

@ -11,6 +11,8 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\..\Research\Google_GenerativeAI\src\GenerativeAI.Live\GenerativeAI.Live.csproj" />
<ProjectReference Include="..\..\..\..\..\Research\Google_GenerativeAI\src\GenerativeAI\GenerativeAI.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>

1
src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.GoogleAi.Providers.Chat;
using BotSharp.Plugin.GoogleAI.Providers.Embedding;
using BotSharp.Plugin.GoogleAi.Providers.Realtime;
using BotSharp.Plugin.GoogleAi.Providers.Text;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -6,7 +6,6 @@ using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Types;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
View file

@ -5,7 +5,6 @@ using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using LLMSharp.Google.Palm;
using LLMSharp.Google.Palm.DiscussService;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Plugin.GoogleAi.Providers;
using GenerativeAI;
using GenerativeAI.Types;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAI.Providers.Embedding;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs
View file

@ -1,5 +1,4 @@
using LLMSharp.Google.Palm;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers;

874
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -1,509 +1,509 @@
using System.Net.WebSockets;
using System.Text.Json.Nodes;
using System.Threading;
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Realtime;
using BotSharp.Abstraction.Realtime.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.GoogleAi.Providers.Chat;
using BotSharp.Abstraction.MLTasks.Settings;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Live;
using GenerativeAI.Live.Extensions;
using GenerativeAI.Types;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime
namespace BotSharp.Plugin.GoogleAi.Providers.Realtime;
public class GoogleRealTimeProvider : IRealTimeCompletion
{
public class GoogleRealTimeProvider : IRealTimeCompletion
public string Provider => "google-ai";
private string _model = GoogleAIModels.Gemini2FlashExp;
public string Model => _model;
private MultiModalLiveClient _client;
private GenerativeModel _chatClient;
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private List<string> renderedInstructions = [];
private readonly GoogleAiSettings _settings;
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings settings,
ILogger<GoogleRealTimeProvider> logger)
{
public string Provider => "google-ai";
private string _model = GoogleAIModels.Gemini2FlashExp;
public string Model { get; }
private MultiModalLiveClient? _client;
private readonly IServiceProvider _services;
private readonly ILogger<GeminiChatCompletionProvider> _logger;
private List<string> renderedInstructions = [];
_settings = settings;
_services = services;
_logger = logger;
}
private readonly GoogleAiSettings _settings;
public void SetModelName(string model)
{
_model = model;
}
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings googleSettings,
ILogger<GeminiChatCompletionProvider> logger)
{
_settings = googleSettings;
_services = services;
_logger = logger;
}
private Action onModelReady;
Action<string, string> onModelAudioDeltaReceived;
private Action onModelAudioResponseDone;
Action<string> onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> onModelResponseDone;
Action<string> onConversationItemCreated;
private Action<RoleDialogModel> onInputAudioTranscriptionCompleted;
Action onUserInterrupted;
RealtimeHubConnection conn;
public void SetModelName(string model)
{
_model = model;
}
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
this.conn = conn;
this.onModelReady = onModelReady;
this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
this.onModelAudioResponseDone = onModelAudioResponseDone;
this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
this.onModelResponseDone = onModelResponseDone;
this.onConversationItemCreated = onConversationItemCreated;
this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
this.onUserInterrupted = onUserInterrupted;
private Action onModelReady;
Action<string, string> onModelAudioDeltaReceived;
private Action onModelAudioResponseDone;
Action<string> onModelAudioTranscriptDone;
private Action<List<RoleDialogModel>> onModelResponseDone;
Action<string> onConversationItemCreated;
private Action<RoleDialogModel> onInputAudioTranscriptionCompleted;
Action onUserInterrupted;
RealtimeHubConnection conn;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
_model = llmProviderService.GetProviderModel(Provider, "gemini-2.0", modelType: LlmModelType.Realtime).Name;
public async Task Connect(RealtimeHubConnection conn,
Action onModelReady,
Action<string, string> onModelAudioDeltaReceived,
Action onModelAudioResponseDone,
Action<string> onModelAudioTranscriptDone,
Action<List<RoleDialogModel>> onModelResponseDone,
Action<string> onConversationItemCreated,
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
Action onUserInterrupted)
{
this.conn = conn;
this.onModelReady = onModelReady;
this.onModelAudioDeltaReceived = onModelAudioDeltaReceived;
this.onModelAudioResponseDone = onModelAudioResponseDone;
this.onModelAudioTranscriptDone = onModelAudioTranscriptDone;
this.onModelResponseDone = onModelResponseDone;
this.onConversationItemCreated = onConversationItemCreated;
this.onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted;
this.onUserInterrupted = onUserInterrupted;
}
public async Task Disconnect()
{
if (_client != null)
await _client.DisconnectAsync();
}
public async Task AppenAudioBuffer(string message)
{
await _client.SendAudioAsync(Convert.FromBase64String(message));
}
public async Task TriggerModelInference(string? instructions = null)
{
await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
_chatClient = client.CreateGenerativeModel(_model);
_client = _chatClient.CreateMultiModalLiveClient(
config: new GenerationConfig
{
TurnComplete = true,
ResponseModalities = [Modality.AUDIO],
},
systemInstruction: "You are a helpful assistant.",
logger: _logger);
await AttachEvents(_client);
await _client.ConnectAsync();
}
public async Task Disconnect()
{
if (_client != null)
await _client.DisconnectAsync();
}
public async Task AppenAudioBuffer(string message)
{
await _client.SendAudioAsync(Convert.FromBase64String(message));
}
public async Task TriggerModelInference(string? instructions = null)
{
await _client.SendClientContentAsync(new BidiGenerateContentClientContent()
{
TurnComplete = true,
});
}
public async Task CancelModelResponse()
{
}
public async Task RemoveConversationItem(string itemId)
{
}
private Task AttachEvents(MultiModalLiveClient client)
{
client.Connected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client connected");
onModelReady();
};
client.Disconnected += (sender, e) =>
{
_logger.LogInformation("Google Realtime Client disconnected");
};
client.MessageReceived += async (sender, e) =>
{
if (e.Payload.SetupComplete != null)
{
onConversationItemCreated(_client.ConnectionId.ToString());
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
onModelResponseDone(responseDone);
}
}
};
client.AudioChunkReceived += (sender, e) =>
{
onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
};
client.TextChunkReceived += (sender, e) =>
{
onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
client.GenerationInterrupted += (sender, e) =>
{
onUserInterrupted();
};
client.AudioReceiveCompleted += (sender, e) =>
{
onModelAudioResponseDone();
};
client.ErrorOccurred += (sender, e) =>
{
var ex = e.GetException();
_logger.LogError(ex, "Error occurred in Google Realtime Client");
};
return Task.CompletedTask;
}
private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
BidiGenerateContentServerContent serverContent)
{
var outputs = new List<RoleDialogModel>();
var parts = serverContent.ModelTurn?.Parts;
if (parts != null)
{
foreach (var part in parts)
{
var call = part.FunctionCall;
if (call != null)
{
var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
else
{
var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = call.Name,
FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
ToolCallId = call.Id ?? String.Empty,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
});
}
public async Task CancelModelResponse()
return outputs;
}
public async Task SendEventToModel(object message)
{
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
}
public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
{
var (prompt, request) = PrepareOptions(_chatClient, agent, conversations);
var config = request.GenerationConfig;
// Output Modality can either be text or audio
config.ResponseModalities = new List<Modality>([Modality.AUDIO]);
await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
}
public async Task RemoveConversationItem(string itemId)
GenerationConfig = config,
Model = Model,
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
});
return new RealtimeSession()
{
}
Id = _client.ConnectionId.ToString(),
Model = _model,
Voice = "default"
};
}
private async Task AttachEvents()
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var (prompt, request) = PrepareOptions(_chatClient, agent, new List<RoleDialogModel>());
var config = request.GenerationConfig;
//Output Modality can either be text or audio
if (config != null)
{
_client.MessageReceived += async (sender, e) =>
{
if (e.Payload.SetupComplete != null)
{
onModelReady();
onConversationItemCreated(_client.ConnectionId.ToString());
}
if (e.Payload.ServerContent != null)
{
if (e.Payload.ServerContent.TurnComplete == true)
{
var responseDone = await ResponseDone(conn, e.Payload.ServerContent);
onModelResponseDone(responseDone);
}
}
};
_client.AudioChunkReceived += async (sender, e) =>
{
onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString());
};
_client.TextChunkReceived += async (sender, e) =>
{
onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text));
};
_client.GenerationInterrupted += async (sender, e) => { onUserInterrupted(); };
_client.AudioReceiveCompleted += async (sender, e) => { onModelAudioResponseDone(); };
}
private async Task<List<RoleDialogModel>> ResponseDone(RealtimeHubConnection conn,
BidiGenerateContentServerContent serverContent)
{
var outputs = new List<RoleDialogModel>();
var parts = serverContent.ModelTurn?.Parts;
if (parts != null)
{
foreach (var part in parts)
{
var call = part.FunctionCall;
if (call != null)
{
var item = new RoleDialogModel(AgentRole.Assistant, part.Text)
{
CurrentAgentId = conn.CurrentAgentId,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
else
{
var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty)
{
CurrentAgentId = conn.CurrentAgentId,
FunctionName = call.Name,
FunctionArgs = call.Args?.ToJsonString() ?? string.Empty,
ToolCallId = call.Id ?? String.Empty,
MessageId = call.Id ?? String.Empty,
MessageType = MessageTypeName.FunctionCall
};
outputs.Add(item);
}
}
}
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// After chat completion hook
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done")
{
CurrentAgentId = conn.CurrentAgentId
}, new TokenStatsModel
{
Provider = Provider,
Model = _model,
});
}
return outputs;
}
public async Task SendEventToModel(object message)
{
//todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete
}
public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
{
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGenerativeModel(_model);
var (prompt, request) = PrepareOptions(chatClient, agent, conversations);
var config = request.GenerationConfig;
//Output Modality can either be text or audio
config.ResponseModalities = new List<Modality>([Modality.AUDIO]);
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
_client = chatClient.CreateMultiModalLiveClient(config,
systemInstruction: request.SystemInstruction?.Parts.FirstOrDefault()?.Text);
_client.UseGoogleSearch = _settings.Gemini.UseGoogleSearch;
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
if (_settings.Gemini.UseGoogleSearch)
config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
var fn = new FunctionDef
{
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools.Add(new Tool()
{
GoogleSearch = new GoogleSearchTool()
});
}
await AttachEvents();
await _client.ConnectAsync();
await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
GenerationConfig = config,
Model = Model,
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
});
return new RealtimeSession()
{
Id = _client.ConnectionId.ToString(),
Model = _model,
Voice = "default"
Name = x.Name ?? string.Empty,
Description = x.Description ?? string.Empty,
};
}
fn.Parameters = x.Parameters != null
? JsonSerializer.Deserialize<FunctionParametersDef>(JsonSerializer.Serialize(x.Parameters))
: null;
return fn;
}).ToArray();
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
await HookEmitter.Emit<IContentGeneratingHook>(_services,
async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
if (_settings.Gemini.UseGoogleSearch)
{
var convService = _services.GetRequiredService<IConversationService>();
var conv = await convService.GetConversation(conn.ConversationId);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGenerativeModel(_model);
var (prompt, request) = PrepareOptions(chatClient, agent, new List<RoleDialogModel>());
var config = request.GenerationConfig;
//Output Modality can either be text or audio
if (config != null)
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools.Add(new Tool()
{
config.ResponseModalities = new List<Modality>([Modality.AUDIO]);
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f);
config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
var fn = new FunctionDef
{
Name = x.Name ?? string.Empty,
Description = x.Description ?? string.Empty,
};
fn.Parameters = x.Parameters != null
? JsonSerializer.Deserialize<FunctionParametersDef>(JsonSerializer.Serialize(x.Parameters))
: null;
return fn;
}).ToArray();
await HookEmitter.Emit<IContentGeneratingHook>(_services,
async hook => { await hook.OnSessionUpdated(agent, prompt, functions); });
if (_settings.Gemini.UseGoogleSearch)
{
if (request.Tools == null)
request.Tools = new List<Tool>();
request.Tools.Add(new Tool()
{
GoogleSearch = new GoogleSearchTool()
});
}
//ToDo: Not sure what's the purpose of UpdateSession, Google Realtime conversion works right away after sending the message!
// await _client.SendSetupAsync(new BidiGenerateContentSetup()
// {
// GenerationConfig = config,
// Model = Model,
// SystemInstruction = request.SystemInstruction,
// Tools = request.Tools?.ToArray(),
// });
return prompt;
GoogleSearch = new GoogleSearchTool()
});
}
public async Task InsertConversationItem(RoleDialogModel message)
await _client.SendSetupAsync(new BidiGenerateContentSetup()
{
if (_client == null)
throw new Exception("Client is not initialized");
if (message.Role == AgentRole.Function)
{
var function = new FunctionResponse()
{
Name = message.FunctionName ?? string.Empty,
Response = JsonNode.Parse(message.Content ?? "{}")
};
GenerationConfig = config,
Model = Model,
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
});
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
{
FunctionResponses = [function]
});
}
else if (message.Role == AgentRole.Assistant)
return prompt;
}
public async Task InsertConversationItem(RoleDialogModel message)
{
if (_client == null)
throw new Exception("Client is not initialized");
if (message.Role == AgentRole.Function)
{
var function = new FunctionResponse()
{
}
else if (message.Role == AgentRole.User)
Name = message.FunctionName ?? string.Empty,
Response = JsonNode.Parse(message.Content ?? "{}")
};
await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse()
{
await _client.SentTextAsync(message.Content);
}
else
{
throw new NotImplementedException("");
}
FunctionResponses = [function]
});
}
public Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
else if (message.Role == AgentRole.Assistant)
{
}
else if (message.Role == AgentRole.User)
{
await _client.SentTextAsync(message.Content);
}
else
{
throw new NotImplementedException("");
}
}
public Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
{
throw new NotImplementedException("");
}
public Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
public Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response)
{
return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
}
private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var googleSettings = _settings;
renderedInstructions = [];
// Add settings
aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
{
return Task.FromResult(new RoleDialogModel(AgentRole.User, response));
AutoCallFunction = false
};
// Assembly messages
var contents = new List<Content>();
var tools = new List<Tool>();
var funcDeclarations = new List<FunctionDeclaration>();
var systemPrompts = new List<string>();
if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
{
var instruction = agentService.RenderedInstruction(agent);
renderedInstructions.Add(instruction);
systemPrompts.Add(instruction);
}
private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent,
List<RoleDialogModel> conversations)
var funcPrompts = new List<string>();
var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
foreach (var function in functions)
{
var agentService = _services.GetRequiredService<IAgentService>();
var googleSettings = _settings;
renderedInstructions = [];
if (!agentService.RenderFunction(agent, function)) continue;
// Add settings
aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch;
aiModel.UseGrounding = googleSettings.Gemini.UseGrounding;
var def = agentService.RenderFunctionProperty(agent, function);
var props = JsonSerializer.Serialize(def?.Properties);
var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}"
? new Schema()
{
Type = "object",
Properties = JsonSerializer.Deserialize<Dictionary<string, Schema>>(props),
Required = def?.Required ?? []
}
: null;
aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour()
funcDeclarations.Add(new FunctionDeclaration
{
AutoCallFunction = false
};
Name = function.Name,
Description = function.Description,
Parameters = parameters
});
// Assembly messages
var contents = new List<Content>();
var tools = new List<Tool>();
var funcDeclarations = new List<FunctionDeclaration>();
funcPrompts.Add($"{function.Name}: {function.Description} {def}");
}
var systemPrompts = new List<string>();
if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
if (!funcDeclarations.IsNullOrEmpty())
{
tools.Add(new Tool { FunctionDeclarations = funcDeclarations });
}
var convPrompts = new List<string>();
foreach (var message in conversations)
{
if (message.Role == AgentRole.Function)
{
var instruction = agentService.RenderedInstruction(agent);
renderedInstructions.Add(instruction);
systemPrompts.Add(instruction);
}
var funcPrompts = new List<string>();
var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
foreach (var function in functions)
{
if (!agentService.RenderFunction(agent, function)) continue;
var def = agentService.RenderFunctionProperty(agent, function);
var props = JsonSerializer.Serialize(def?.Properties);
var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}"
? new Schema()
contents.Add(new Content([
new Part()
{
Type = "object",
Properties = JsonSerializer.Deserialize<Dictionary<string, Schema>>(props),
Required = def?.Required ?? []
FunctionCall = new FunctionCall
{
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
}
}
: null;
], AgentRole.Model));
funcDeclarations.Add(new FunctionDeclaration
{
Name = function.Name,
Description = function.Description,
Parameters = parameters
});
funcPrompts.Add($"{function.Name}: {function.Description} {def}");
}
if (!funcDeclarations.IsNullOrEmpty())
{
tools.Add(new Tool { FunctionDeclarations = funcDeclarations });
}
var convPrompts = new List<string>();
foreach (var message in conversations)
{
if (message.Role == AgentRole.Function)
{
contents.Add(new Content([
new Part()
contents.Add(new Content([
new Part()
{
FunctionResponse = new FunctionResponse
{
FunctionCall = new FunctionCall
Name = message.FunctionName ?? string.Empty,
Response = new JsonObject()
{
Name = message.FunctionName,
Args = JsonNode.Parse(message.FunctionArgs ?? "{}")
["result"] = message.Content ?? string.Empty
}
}
], AgentRole.Model));
}
], AgentRole.Function));
contents.Add(new Content([
new Part()
{
FunctionResponse = new FunctionResponse
{
Name = message.FunctionName ?? string.Empty,
Response = new JsonObject()
{
["result"] = message.Content ?? string.Empty
}
}
}
], AgentRole.Function));
convPrompts.Add(
$"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
contents.Add(new Content(text, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)
{
contents.Add(new Content(message.Content, AgentRole.Model));
convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
}
convPrompts.Add(
$"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
}
var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
else if (message.Role == AgentRole.User)
{
SystemInstruction = !systemPrompts.IsNullOrEmpty()
? new Content(systemPrompts[0], AgentRole.System)
: null,
Contents = contents,
Tools = tools,
GenerationConfig = new()
{
Temperature = temperature,
MaxOutputTokens = maxTokens
}
};
var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
return (prompt, request);
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
contents.Add(new Content(text, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)
{
contents.Add(new Content(message.Content, AgentRole.Model));
convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
}
}
private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string> funcPrompts,
IEnumerable<string> convPrompts)
var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
? tokens
: agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
{
string prompt = string.Join("\r\n\r\n", systemPrompts);
if (!funcPrompts.IsNullOrEmpty())
SystemInstruction = !systemPrompts.IsNullOrEmpty()
? new Content(systemPrompts[0], AgentRole.System)
: null,
Contents = contents,
Tools = tools,
GenerationConfig = new()
{
prompt += "\r\n\r\n[FUNCTIONS]\r\n";
prompt += string.Join("\r\n", funcPrompts);
Temperature = temperature,
MaxOutputTokens = maxTokens
}
};
if (!convPrompts.IsNullOrEmpty())
{
prompt += "\r\n\r\n[CONVERSATION]\r\n";
prompt += string.Join("\r\n", convPrompts);
}
var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
return (prompt, request);
}
return prompt;
private string GetPrompt(IEnumerable<string> systemPrompts, IEnumerable<string> funcPrompts,
IEnumerable<string> convPrompts)
{
string prompt = string.Join("\r\n\r\n", systemPrompts);
if (!funcPrompts.IsNullOrEmpty())
{
prompt += "\r\n\r\n[FUNCTIONS]\r\n";
prompt += string.Join("\r\n", funcPrompts);
}
if (!convPrompts.IsNullOrEmpty())
{
prompt += "\r\n\r\n[CONVERSATION]\r\n";
prompt += string.Join("\r\n", convPrompts);
}
return prompt;
}
}

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs
View file

@ -3,7 +3,6 @@ using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;

1
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;

22
src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
View file

@ -4,12 +4,26 @@ global using System.Text;
global using System.Threading.Tasks;
global using System.Linq;
global using System.Text.Json;
global using System.Text.Json.Serialization;
global using System.Text.Json.Nodes;
global using Microsoft.Extensions.Logging;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using System.Text.Json.Serialization;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Plugin.GoogleAi.Settings;
global using BotSharp.Abstraction.Realtime;
global using BotSharp.Abstraction.Realtime.Models;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Plugin.GoogleAi.Providers.Chat;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Enums;
global using BotSharp.Abstraction.Functions.Models;
global using BotSharp.Abstraction.Loggers;