From be521997cfb7971e24b865b4669896d0ff1a2687 Mon Sep 17 00:00:00 2001 From: Haiping Date: Sat, 22 Mar 2025 20:46:14 -0500 Subject: [PATCH 1/2] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index b8162b2f..91cb7a0a 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username -custom: ['https://bit.ly/2op1mu5'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] +custom: ['https://www.paypal.com/pool/9avinmEbbb?sr=wccr'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 439d4b451c19378bb3a2d01e46f0835ed29f59b4 Mon Sep 17 00:00:00 2001 From: Gunpal Jain Date: Sun, 23 Mar 2025 23:57:58 +0530 Subject: [PATCH 2/2] feat(Gemini): Added Text Embedding Model feat: implemented missing methods. changed to Google_GenerativeAI SDK which supports Multimodal Live APIs. --- Directory.Packages.props | 4 +- .../BotSharp.Plugin.GoogleAI.csproj | 10 +- .../GoogleAiPlugin.cs | 2 + .../Chat/GeminiChatCompletionProvider.cs | 162 ++++++++++++++---- .../Chat/PalmChatCompletionProvider.cs | 2 +- .../Embedding/TextEmbeddingProvider.cs | 67 ++++++++ .../Providers/ProviderHelper.cs | 5 +- .../Text/GeminiTextCompletionProvider.cs | 11 +- 8 files changed, 217 insertions(+), 46 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f49939eb..5cd53104 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,6 +6,8 @@ + + @@ -42,8 +44,6 @@ - - diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index 08d1f499..a91995c2 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -9,14 +9,14 @@ $(GenerateDocumentationFile) $(SolutionDir)packages - - - - - + + + + + diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs index 58a384a7..ab23537b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; using BotSharp.Plugin.GoogleAi.Providers.Chat; +using BotSharp.Plugin.GoogleAI.Providers.Embedding; using BotSharp.Plugin.GoogleAi.Providers.Text; namespace BotSharp.Plugin.GoogleAi; @@ -23,5 +24,6 @@ public class GoogleAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 174efcf2..8d5d9423 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -1,9 +1,12 @@ +using System.Text.Json.Nodes; using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Loggers; +using GenerativeAI; +using GenerativeAI.Core; +using GenerativeAI.Types; using Microsoft.Extensions.Logging; -using Mscc.GenerativeAI; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; @@ -37,16 +40,16 @@ public class GeminiChatCompletionProvider : IChatCompletion } var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - var aiModel = client.GenerativeModel(_model); + var aiModel = client.CreateGenerativeModel(_model); var (prompt, request) = PrepareOptions(aiModel, agent, conversations); - var response = await aiModel.GenerateContent(request); - var candidate = response.Candidates.First(); - var part = candidate.Content?.Parts?.FirstOrDefault(); + var response = await aiModel.GenerateContentAsync(request); + var candidate = response.Candidates?.First(); + var part = candidate?.Content?.Parts?.FirstOrDefault(); var text = part?.Text ?? string.Empty; RoleDialogModel responseMessage; - if (part?.FunctionCall != null) + if (response.GetFunction()!=null) { responseMessage = new RoleDialogModel(AgentRole.Function, text) { @@ -54,7 +57,7 @@ public class GeminiChatCompletionProvider : IChatCompletion MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, ToolCallId = part.FunctionCall.Name, FunctionName = part.FunctionCall.Name, - FunctionArgs = part.FunctionCall.Args?.ToString(), + FunctionArgs = part.FunctionCall.Args?.ToJsonString(), RenderedInstruction = string.Join("\r\n", renderedInstructions) }; } @@ -82,14 +85,112 @@ public class GeminiChatCompletionProvider : IChatCompletion return responseMessage; } - public Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) + public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { - throw new NotImplementedException(); + var hooks = _services.GetServices().ToList(); + + // Before chat completion hook + foreach (var hook in hooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); + var chatClient = client.CreateGeminiModel(_model); + var (prompt, messages) = PrepareOptions(chatClient,agent, conversations); + + var response = await chatClient.GenerateContentAsync(messages); + + var candidate = response.Candidates?.First(); + var part = candidate?.Content?.Parts?.FirstOrDefault(); + var text = part?.Text ?? string.Empty; + + var msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }; + + // After chat completion hook + foreach (var hook in hooks) + { + await hook.AfterGenerated(msg, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + PromptCount = response?.UsageMetadata?.PromptTokenCount ?? 0, + CompletionCount = response?.UsageMetadata?.CandidatesTokenCount ?? 0 + }); + } + + if (response.GetFunction()!=null) + { + var toolCall = response.GetFunction(); + _logger.LogInformation($"[{agent.Name}]: {toolCall?.Name}({toolCall?.Args?.ToJsonString()})"); + + var funcContextIn = new RoleDialogModel(AgentRole.Function, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + ToolCallId = toolCall?.Id, + FunctionName = toolCall?.Name, + FunctionArgs = toolCall?.Args?.ToJsonString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }; + + // Somethings LLM will generate a function name with agent name. + if (!string.IsNullOrEmpty(funcContextIn.FunctionName)) + { + funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last(); + } + + // Execute functions + await onFunctionExecuting(funcContextIn); + } + else + { + // Text response received + await onMessageReceived(msg); + } + + return true; } - public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { - throw new NotImplementedException(); + var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); + var chatClient = client.CreateGenerativeModel(_model); + var (prompt, messages) = PrepareOptions(chatClient,agent, conversations); + + var asyncEnumerable = chatClient.StreamContentAsync(messages); + + await foreach (var response in asyncEnumerable) + { + if (response.GetFunction()!=null) + { + var func = response.GetFunction(); + var update =func?.Args?.ToJsonString().ToString() ?? string.Empty; + _logger.LogInformation(update); + + await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); + continue; + } + + if (response.Text().IsNullOrEmpty()) continue; + + _logger.LogInformation(response.Text()); + + await onMessageReceived(new RoleDialogModel(response.Candidates?.LastOrDefault()?.Content?.Role?.ToString() ?? AgentRole.Assistant.ToString(), response.Text() ?? string.Empty) + { + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }); + } + + return true; } public void SetModelName(string model) @@ -107,6 +208,10 @@ public class GeminiChatCompletionProvider : IChatCompletion aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; aiModel.UseGrounding = googleSettings.Gemini.UseGrounding; + aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour() + { + AutoCallFunction = false + }; // Assembly messages var contents = new List(); var tools = new List(); @@ -116,11 +221,7 @@ public class GeminiChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty()) { var instruction = agentService.RenderedInstruction(agent); - contents.Add(new Content(instruction) - { - Role = AgentRole.User - }); - + contents.Add(new Content(instruction, AgentRole.User)); renderedInstructions.Add(instruction); systemPrompts.Add(instruction); } @@ -135,7 +236,7 @@ public class GeminiChatCompletionProvider : IChatCompletion var props = JsonSerializer.Serialize(def?.Properties); var parameters = !string.IsNullOrWhiteSpace(props) && props != "{}" ? new Schema() { - Type = ParameterType.Object, + Type = "object", Properties = JsonSerializer.Deserialize(props), Required = def?.Required ?? [] } : null; @@ -160,17 +261,20 @@ public class GeminiChatCompletionProvider : IChatCompletion { if (message.Role == AgentRole.Function) { - contents.Add(new Content(message.Content) + contents.Add( new Content(message.Content,AgentRole.Function) { Role = AgentRole.Function, - Parts = new() - { - new FunctionCall + Parts = + [ + new Part() { - Name = message.FunctionName, - Args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") + FunctionCall = new FunctionCall + { + Name = message.FunctionName, + Args = JsonNode.Parse(message.FunctionArgs ?? "{}") + } } - } + ] }); convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs})"); @@ -178,18 +282,12 @@ public class GeminiChatCompletionProvider : IChatCompletion else if (message.Role == AgentRole.User) { var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; - contents.Add(new Content(text) - { - Role = AgentRole.User - }); + contents.Add(new Content(text, AgentRole.User)); convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { - contents.Add(new Content(message.Content) - { - Role = AgentRole.Model - }); + contents.Add(new Content(message.Content, AgentRole.Model)); convPrompts.Add($"{AgentRole.Assistant}: {message.Content}"); } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index 4fe6ad17..91e797cc 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -19,7 +19,7 @@ public class PalmChatCompletionProvider : IChatCompletion public string Provider => "google-palm"; public string Model => _model; - + public PalmChatCompletionProvider( IServiceProvider services, ILogger logger) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs new file mode 100644 index 00000000..0f8d823e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -0,0 +1,67 @@ +using BotSharp.Plugin.GoogleAi.Providers; +using GenerativeAI; +using GenerativeAI.Types; +using Microsoft.Extensions.Logging; + +namespace BotSharp.Plugin.GoogleAI.Providers.Embedding; + +public class TextEmbeddingProvider : ITextEmbedding +{ + protected readonly GoogleAiSettings _settings; + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + + private const int DEFAULT_DIMENSION = 1536; + protected string _model = GoogleAIModels.TextEmbedding; + protected int _dimension = DEFAULT_DIMENSION; + + public virtual string Provider => "google-ai"; + public string Model => _model; + + public TextEmbeddingProvider( + GoogleAiSettings settings, + ILogger logger, + IServiceProvider services) + { + _settings = settings; + _logger = logger; + _services = services; + } + + public async Task GetVectorAsync(string text) + { + var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); + var embeddingClient = client.CreateEmbeddingModel(_model); + + var response = await embeddingClient.EmbedContentAsync(text); + var value = response?.Embedding?.Values; + return value.ToArray(); + } + + public async Task> GetVectorsAsync(List texts) + { + var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); + var embeddingClient = client.CreateEmbeddingModel(_model); + + var response = await embeddingClient.BatchEmbedContentAsync(texts.Select(s=>new Content(s, Roles.User))); + var value = response.Embeddings; + if (value == null) + return new List(); + return value.Select(x => x.Values?.ToArray()??[]).ToList(); + } + + public void SetModelName(string model) + { + _model = model; + } + + public void SetDimension(int dimension) + { + _dimension = dimension > 0 ? dimension : DEFAULT_DIMENSION; + } + + public int GetDimension() + { + return _dimension; + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs index 5ca1058a..20ac2105 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs @@ -1,15 +1,14 @@ using LLMSharp.Google.Palm; -using Mscc.GenerativeAI; namespace BotSharp.Plugin.GoogleAi.Providers; public static class ProviderHelper { - public static GoogleAI GetGeminiClient(string provider, string model, IServiceProvider services) + public static GenerativeAI.GoogleAi GetGeminiClient(string provider, string model, IServiceProvider services) { var settingsService = services.GetRequiredService(); var settings = settingsService.GetSetting(provider, model); - var client = new GoogleAI(settings.ApiKey); + var client = new GenerativeAI.GoogleAi(settings.ApiKey); return client; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index ebbca98a..4c956c97 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -1,8 +1,9 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Loggers; +using GenerativeAI; +using GenerativeAI.Core; using Microsoft.Extensions.Logging; -using Mscc.GenerativeAI; namespace BotSharp.Plugin.GoogleAi.Providers.Text; @@ -47,11 +48,11 @@ public class GeminiTextCompletionProvider : ITextCompletion } var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - var aiModel = client.GenerativeModel(_model); + var aiModel = client.CreateGenerativeModel(_model); PrepareOptions(aiModel); _tokenStatistics.StartTimer(); - var response = await aiModel.GenerateContent(text); + var response = await aiModel.GenerateContentAsync(text); _tokenStatistics.StopTimer(); var completion = response.Text ?? string.Empty; @@ -80,5 +81,9 @@ public class GeminiTextCompletionProvider : ITextCompletion var settings = _services.GetRequiredService(); aiModel.UseGoogleSearch = settings.Gemini.UseGoogleSearch; aiModel.UseGrounding = settings.Gemini.UseGrounding; + aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour() + { + AutoCallFunction = false + }; } }