From 99e949032aeb3fccf63be7cd96fa504f927a0845 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Sun, 22 Dec 2024 20:12:35 -0600 Subject: [PATCH 1/4] add gemini --- .../Agents/Enums/AgentRole.cs | 1 + .../BotSharp.Plugin.GoogleAI.csproj | 1 + .../GoogleAiPlugin.cs | 12 +- .../Chat/GeminiChatCompletionProvider.cs | 194 ++++++++++++++++++ .../PalmChatCompletionProvider.cs} | 43 ++-- .../Providers/ProviderHelper.cs | 21 ++ .../Text/GeminiTextCompletionProvider.cs | 84 ++++++++ .../Text/PalmTextCompletionProvider.cs | 66 ++++++ .../Providers/TextCompletionProvider.cs | 68 ------ .../Settings/GoogleAiSettings.cs | 18 +- .../Settings/PaLMSetting.cs | 7 - src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs | 3 +- src/WebStarter/appsettings.json | 5 + 13 files changed, 421 insertions(+), 102 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs rename src/Plugins/BotSharp.Plugin.GoogleAI/Providers/{ChatCompletionProvider.cs => Chat/PalmChatCompletionProvider.cs} (85%) create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs delete mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs delete mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Settings/PaLMSetting.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentRole.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentRole.cs index f9547d5a..226313e0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentRole.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentRole.cs @@ -6,4 +6,5 @@ public class AgentRole public const string Assistant = "assistant"; public const string User = "user"; public const string Function = "function"; + public const string Model = "model"; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj index 93c0f9df..b6e0b24e 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/BotSharp.Plugin.GoogleAI.csproj @@ -12,6 +12,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs index aac88308..58a384a7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/GoogleAiPlugin.cs @@ -1,9 +1,9 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Plugin.GoogleAI.Providers; -using BotSharp.Plugin.GoogleAI.Settings; +using BotSharp.Plugin.GoogleAi.Providers.Chat; +using BotSharp.Plugin.GoogleAi.Providers.Text; -namespace BotSharp.Plugin.GoogleAI; +namespace BotSharp.Plugin.GoogleAi; public class GoogleAiPlugin : IBotSharpPlugin { @@ -19,7 +19,9 @@ public class GoogleAiPlugin : IBotSharpPlugin return settingService.Bind("GoogleAi"); }); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs new file mode 100644 index 00000000..b94dd534 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -0,0 +1,194 @@ +using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Loggers; +using Microsoft.Extensions.Logging; +using Mscc.GenerativeAI; + +namespace BotSharp.Plugin.GoogleAi.Providers.Chat; + +public class GeminiChatCompletionProvider : IChatCompletion +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private string _model; + + public string Provider => "google-gemini"; + + public GeminiChatCompletionProvider( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task GetChatCompletions(Agent agent, List conversations) + { + var contentHooks = _services.GetServices().ToList(); + + // Before chat completion hook + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetGeminiClient(_services); + var aiModel = client.GenerativeModel(_model); + var (prompt, request) = PrepareOptions(aiModel, agent, conversations); + + var response = await aiModel.GenerateContent(request); + var candidate = response.Candidates.First(); + var part = candidate.Content?.Parts?.FirstOrDefault(); + var text = part?.Text ?? string.Empty; + + RoleDialogModel responseMessage; + if (part?.FunctionCall != null) + { + responseMessage = new RoleDialogModel(AgentRole.Function, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + ToolCallId = part.FunctionCall.Name, + FunctionName = part.FunctionCall.Name, + FunctionArgs = part.FunctionCall.Args?.ToString() + }; + } + else + { + responseMessage = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + }; + } + + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model + }); + } + + return responseMessage; + } + + public Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) + { + throw new NotImplementedException(); + } + + public Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + { + throw new NotImplementedException(); + } + + public void SetModelName(string model) + { + _model = model; + } + + private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent, List conversations) + { + var agentService = _services.GetRequiredService(); + var googleSettings = _services.GetRequiredService(); + + // Add settings + aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; + aiModel.UseGrounding = googleSettings.Gemini.UseGrounding; + + // Assembly messages + var prompt = string.Empty; + var contents = new List(); + var tools = new List(); + var funcDeclarations = new List(); + + if (!string.IsNullOrEmpty(agent.Instruction)) + { + var instruction = agentService.RenderedInstruction(agent); + contents.Add(new Content(instruction) + { + Role = AgentRole.User + }); + + prompt += $"{instruction}\r\n"; + } + + prompt += "\r\n[FUNCTIONS]\r\n"; + foreach (var function in agent.Functions) + { + if (!agentService.RenderFunction(agent, function)) continue; + + var def = agentService.RenderFunctionProperty(agent, function); + + funcDeclarations.Add(new FunctionDeclaration + { + Name = function.Name, + Description = function.Description, + Parameters = new() + { + Type = ParameterType.Object, + Properties = def.Properties, + Required = def.Required + } + }); + + prompt += $"{function.Name}: {function.Description} {def}\r\n\r\n"; + } + + if (!funcDeclarations.IsNullOrEmpty()) + { + tools.Add(new Tool { FunctionDeclarations = funcDeclarations }); + } + + prompt += "\r\n[CONVERSATIONS]\r\n"; + foreach (var message in conversations) + { + if (message.Role == AgentRole.Function) + { + contents.Add(new Content(message.Content) + { + Role = AgentRole.Function, + Parts = new() + { + new FunctionCall + { + Name = message.FunctionName, + Args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}") + } + } + }); + + prompt += $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs})\r\n"; + } + else if (message.Role == AgentRole.User) + { + var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content; + contents.Add(new Content(text) + { + Role = AgentRole.User + }); + prompt += $"{AgentRole.User}: {text}\r\n"; + } + else if (message.Role == AgentRole.Assistant) + { + contents.Add(new Content(message.Content) + { + Role = AgentRole.Model + }); + prompt += $"{AgentRole.Assistant}: {message.Content}\r\n"; + } + } + + var request = new GenerateContentRequest + { + Contents = contents, + Tools = tools + }; + return (prompt, request); + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs similarity index 85% rename from src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs rename to src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index d278b110..851792a6 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -3,44 +3,44 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing; -using BotSharp.Plugin.GoogleAI.Settings; using LLMSharp.Google.Palm; -using Microsoft.Extensions.Logging; using LLMSharp.Google.Palm.DiscussService; +using Microsoft.Extensions.Logging; -namespace BotSharp.Plugin.GoogleAI.Providers; +namespace BotSharp.Plugin.GoogleAi.Providers.Chat; -public class ChatCompletionProvider : IChatCompletion +public class PalmChatCompletionProvider : IChatCompletion { - public string Provider => "google-ai"; private readonly IServiceProvider _services; - private readonly GoogleAiSettings _settings; - private readonly ILogger _logger; + private readonly ILogger _logger; + private string _model; - public ChatCompletionProvider(IServiceProvider services, - GoogleAiSettings settings, - ILogger logger) + public string Provider => "google-ai"; + + public PalmChatCompletionProvider( + IServiceProvider services, + ILogger logger) { _services = services; - _settings = settings; _logger = logger; } public async Task GetChatCompletions(Agent agent, List conversations) { - var hooks = _services.GetServices().ToList(); + var contentHooks = _services.GetServices().ToList(); // Before chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.BeforeGenerating(agent, conversations)).ToArray()); - - var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey); + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + var client = ProviderHelper.GetPalmClient(_services); var (prompt, messages, hasFunctions) = PrepareOptions(agent, conversations); RoleDialogModel msg; - + if (hasFunctions) { // use text completion @@ -80,12 +80,15 @@ public class ChatCompletionProvider : IChatCompletion } // After chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.AfterGenerated(msg, new TokenStatsModel + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(msg, new TokenStatsModel { Prompt = prompt, + Provider = Provider, Model = _model - })).ToArray()); + }); + } return msg; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs new file mode 100644 index 00000000..75435f90 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs @@ -0,0 +1,21 @@ +using LLMSharp.Google.Palm; +using Mscc.GenerativeAI; + +namespace BotSharp.Plugin.GoogleAi.Providers; + +public static class ProviderHelper +{ + public static GoogleAI GetGeminiClient(IServiceProvider services) + { + var settings = services.GetRequiredService(); + var client = new GoogleAI(settings.Gemini.ApiKey); + return client; + } + + public static GooglePalmClient GetPalmClient(IServiceProvider services) + { + var settings = services.GetRequiredService(); + var client = new GooglePalmClient(settings.PaLM.ApiKey); + return client; + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs new file mode 100644 index 00000000..e6e3f4b3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -0,0 +1,84 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Loggers; +using Microsoft.Extensions.Logging; +using Mscc.GenerativeAI; + +namespace BotSharp.Plugin.GoogleAi.Providers.Text; + +public class GeminiTextCompletionProvider : ITextCompletion +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly ITokenStatistics _tokenStatistics; + private string _model; + + public string Provider => "google-gemini"; + + public GeminiTextCompletionProvider( + IServiceProvider services, + ILogger logger, + ITokenStatistics tokenStatistics) + { + _services = services; + _logger = logger; + _tokenStatistics = tokenStatistics; + } + + + public async Task GetCompletion(string text, string agentId, string messageId) + { + var contentHooks = _services.GetServices().ToList(); + + // Before completion hook + var agent = new Agent() + { + Id = agentId + }; + var userMessage = new RoleDialogModel(AgentRole.User, text) + { + MessageId = messageId + }; + + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, new List { userMessage }); + } + + var client = ProviderHelper.GetGeminiClient(_services); + var aiModel = client.GenerativeModel(_model); + PrepareOptions(aiModel); + + _tokenStatistics.StartTimer(); + var response = await aiModel.GenerateContent(text); + _tokenStatistics.StopTimer(); + + var completion = response.Text ?? string.Empty; + + // After completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel + { + Prompt = text, + Provider = Provider, + Model = _model + }); + } + + return completion; + } + + public void SetModelName(string model) + { + _model = model; + } + + + private void PrepareOptions(GenerativeModel aiModel) + { + var settings = _services.GetRequiredService(); + aiModel.UseGoogleSearch = settings.Gemini.UseGoogleSearch; + aiModel.UseGrounding = settings.Gemini.UseGrounding; + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs new file mode 100644 index 00000000..c7e64fe8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -0,0 +1,66 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Loggers; +using Microsoft.Extensions.Logging; + +namespace BotSharp.Plugin.GoogleAi.Providers.Text; + +public class PalmTextCompletionProvider : ITextCompletion +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly ITokenStatistics _tokenStatistics; + + private string _model; + + public string Provider => "google-ai"; + + public PalmTextCompletionProvider( + IServiceProvider services, + ILogger logger, + ITokenStatistics tokenStatistics) + { + _services = services; + _logger = logger; + _tokenStatistics = tokenStatistics; + } + + public async Task GetCompletion(string text, string agentId, string messageId) + { + var contentHooks = _services.GetServices().ToList(); + + // Before completion hook + var agent = new Agent() { Id = agentId }; + var userMessage = new RoleDialogModel(AgentRole.User, text) { MessageId = messageId }; + + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, new List { userMessage }); + } + + var client = ProviderHelper.GetPalmClient(_services); + _tokenStatistics.StartTimer(); + var response = await client.GenerateTextAsync(text, null); + _tokenStatistics.StopTimer(); + + var message = response.Candidates.First(); + var completion = message.Output.Trim(); + + // After completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel + { + Prompt = text, + Provider = Provider + }); + } + + return completion; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs deleted file mode 100644 index c35df64f..00000000 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/TextCompletionProvider.cs +++ /dev/null @@ -1,68 +0,0 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Conversations; -using BotSharp.Abstraction.Loggers; -using BotSharp.Plugin.GoogleAI.Settings; -using LLMSharp.Google.Palm; -using Microsoft.Extensions.Logging; - -namespace BotSharp.Plugin.GoogleAI.Providers; - -public class TextCompletionProvider : ITextCompletion -{ - public string Provider => "google-ai"; - private readonly IServiceProvider _services; - private readonly GoogleAiSettings _settings; - private readonly ILogger _logger; - private readonly ITokenStatistics _tokenStatistics; - private string _model; - - public TextCompletionProvider(IServiceProvider services, - GoogleAiSettings settings, - ILogger logger, - ITokenStatistics tokenStatistics) - { - _services = services; - _settings = settings; - _logger = logger; - _tokenStatistics = tokenStatistics; - } - - public async Task GetCompletion(string text, string agentId, string messageId) - { - var hooks = _services.GetServices().ToList(); - - // Before chat completion hook - var agent = new Agent() - { - Id = agentId - }; - var userMessage = new RoleDialogModel(AgentRole.User, text) - { - MessageId = messageId - }; - Task.WaitAll(hooks.Select(hook => - hook.BeforeGenerating(agent, new List { userMessage })).ToArray()); - - var client = new GooglePalmClient(apiKey: _settings.PaLM.ApiKey); - _tokenStatistics.StartTimer(); - var response = await client.GenerateTextAsync(text, null); - _tokenStatistics.StopTimer(); - - var message = response.Candidates.First(); - var completion = message.Output.Trim(); - - // After chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, completion), new TokenStatsModel - { - Model = _model - })).ToArray()); - - return completion; - } - - public void SetModelName(string model) - { - _model = model; - } -} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/GoogleAiSettings.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/GoogleAiSettings.cs index d515b23f..a4e3468a 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/GoogleAiSettings.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/GoogleAiSettings.cs @@ -1,6 +1,22 @@ -namespace BotSharp.Plugin.GoogleAI.Settings; +namespace BotSharp.Plugin.GoogleAi.Settings; public class GoogleAiSettings { public PaLMSetting PaLM { get; set; } + + public GeminiSetting Gemini { get; set; } +} + +public class PaLMSetting +{ + public string Endpoint { get; set; } = string.Empty; + public string ApiKey { get; set; } +} + + +public class GeminiSetting +{ + public string ApiKey { get; set; } + public bool UseGoogleSearch { get; set; } + public bool UseGrounding { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/PaLMSetting.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/PaLMSetting.cs deleted file mode 100644 index 29894614..00000000 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Settings/PaLMSetting.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Plugin.GoogleAI.Settings; - -public class PaLMSetting -{ - public string Endpoint { get; set; } = string.Empty; - public string ApiKey { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs index 8cb9a723..17152bf4 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs @@ -10,4 +10,5 @@ global using BotSharp.Abstraction.MLTasks; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using System.Text.Json.Serialization; -global using BotSharp.Abstraction.Utilities; \ No newline at end of file +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Plugin.GoogleAi.Settings; \ No newline at end of file diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c3d4bdd5..c600215a 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -197,6 +197,11 @@ "PaLM": { "Endpoint": "https://generativelanguage.googleapis.com", "ApiKey": "" + }, + "Gemini": { + "ApiKey": "", + "UseGoogleSearch": false, + "UseGrounding": false } }, From 9171ea310cd85049f800b6a3030dcdb2507da048 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 22 Dec 2024 23:04:09 -0600 Subject: [PATCH 2/4] refine code --- .../Chat/GeminiChatCompletionProvider.cs | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index b94dd534..87286f77 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -102,11 +102,11 @@ public class GeminiChatCompletionProvider : IChatCompletion aiModel.UseGrounding = googleSettings.Gemini.UseGrounding; // Assembly messages - var prompt = string.Empty; var contents = new List(); var tools = new List(); var funcDeclarations = new List(); + var systemPrompts = new List(); if (!string.IsNullOrEmpty(agent.Instruction)) { var instruction = agentService.RenderedInstruction(agent); @@ -115,10 +115,10 @@ public class GeminiChatCompletionProvider : IChatCompletion Role = AgentRole.User }); - prompt += $"{instruction}\r\n"; + systemPrompts.Add(instruction); } - prompt += "\r\n[FUNCTIONS]\r\n"; + var funcPrompts = new List(); foreach (var function in agent.Functions) { if (!agentService.RenderFunction(agent, function)) continue; @@ -137,7 +137,7 @@ public class GeminiChatCompletionProvider : IChatCompletion } }); - prompt += $"{function.Name}: {function.Description} {def}\r\n\r\n"; + funcPrompts.Add($"{function.Name}: {function.Description} {def}"); } if (!funcDeclarations.IsNullOrEmpty()) @@ -145,7 +145,7 @@ public class GeminiChatCompletionProvider : IChatCompletion tools.Add(new Tool { FunctionDeclarations = funcDeclarations }); } - prompt += "\r\n[CONVERSATIONS]\r\n"; + var convPrompts = new List(); foreach (var message in conversations) { if (message.Role == AgentRole.Function) @@ -163,7 +163,7 @@ public class GeminiChatCompletionProvider : IChatCompletion } }); - prompt += $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs})\r\n"; + convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs})"); } else if (message.Role == AgentRole.User) { @@ -172,7 +172,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { Role = AgentRole.User }); - prompt += $"{AgentRole.User}: {text}\r\n"; + convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { @@ -180,7 +180,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { Role = AgentRole.Model }); - prompt += $"{AgentRole.Assistant}: {message.Content}\r\n"; + convPrompts.Add($"{AgentRole.Assistant}: {message.Content}"); } } @@ -189,6 +189,29 @@ public class GeminiChatCompletionProvider : IChatCompletion Contents = contents, Tools = tools }; + + var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts); return (prompt, request); } + + private string GetPrompt(IEnumerable systemPrompts, IEnumerable funcPrompts, IEnumerable convPrompts) + { + var prompt = string.Empty; + + prompt = string.Join("\r\n\r\n", systemPrompts); + + if (!funcPrompts.IsNullOrEmpty()) + { + prompt += "\r\n[FUNCTIONS]\r\n"; + prompt += string.Join("\r\n", funcPrompts); + } + + if (!convPrompts.IsNullOrEmpty()) + { + prompt += "\r\n[CONVERSATION]\r\n"; + prompt += string.Join("\r\n", convPrompts); + } + + return prompt; + } } From ac600612284041c1b6662155a927dab497233dab Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 22 Dec 2024 23:06:52 -0600 Subject: [PATCH 3/4] minor change --- .../Providers/Chat/GeminiChatCompletionProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 87286f77..4419048b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -202,13 +202,13 @@ public class GeminiChatCompletionProvider : IChatCompletion if (!funcPrompts.IsNullOrEmpty()) { - prompt += "\r\n[FUNCTIONS]\r\n"; + prompt += "\r\n\r\n[FUNCTIONS]\r\n"; prompt += string.Join("\r\n", funcPrompts); } if (!convPrompts.IsNullOrEmpty()) { - prompt += "\r\n[CONVERSATION]\r\n"; + prompt += "\r\n\r\n[CONVERSATION]\r\n"; prompt += string.Join("\r\n", convPrompts); } From 5c09571b6558cfc0418fbbe56a2fb817aa789149 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 24 Dec 2024 14:12:47 -0600 Subject: [PATCH 4/4] rise image detail level --- .../Providers/Chat/ChatCompletionProvider.cs | 6 +++--- .../Providers/Chat/ChatCompletionProvider.cs | 6 +++--- .../BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs | 6 ++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 3384e3b7..a1684c9d 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -290,20 +290,20 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 50151513..c5d45f66 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -268,20 +268,20 @@ public class ChatCompletionProvider : IChatCompletion if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) { var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs index 7ab56b5d..b385b380 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs @@ -9,10 +9,8 @@ public class ProviderHelper { var settingsService = services.GetRequiredService(); var settings = settingsService.GetSetting(provider, model); - var options = string.IsNullOrEmpty(settings.Endpoint) - ? null - : new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) }; - return new OpenAIClient(new ApiKeyCredential(settings.ApiKey), options); + var client = new OpenAIClient(new ApiKeyCredential(settings.ApiKey)); + return client; } public static List GetChatSamples(List lines)