SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1064 from iceljc/test/google-realtime

Browse source 
add gemini file content
This commit is contained in:
Haiping 2025-05-27 09:40:05 -05:00 committed by GitHub
parent f5c5f8ee35 c79e35b653
commit 1672374e48
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 90 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Files.Converters;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Infrastructures;
namespace BotSharp.Core.Files.Services;
@ -22,14 +21,24 @@ public partial class FileInstructService
try
{
var provider = options?.Provider ?? "openai";
var pdfFiles = await DownloadFiles(sessionDir, files);
var images = await ConvertPdfToImages(pdfFiles);
if (images.IsNullOrEmpty()) return content;
var targetFiles = pdfFiles;
if (provider != "google-ai")
{
targetFiles = await ConvertPdfToImages(pdfFiles);
}
if (targetFiles.IsNullOrEmpty())
{
return content;
}
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai",
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider,
model: options?.Model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
@ -39,7 +48,7 @@ public partial class FileInstructService
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
Files = targetFiles.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
}
});

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -35,7 +35,7 @@ public class ReadImageFn : IFunctionCallback
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Agent",
Instruction = fromAgent?.Instruction ?? args.UserRequest ?? "Please describe the image(s).",
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).",
TemplateDict = new Dictionary<string, object>()
};

29
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Routing;
namespace BotSharp.Plugin.FileHandler.Functions;
public class ReadPdfFn : IFunctionCallback
@ -25,20 +27,31 @@ public class ReadPdfFn : IFunctionCallback
{
var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
var conv = _services.GetRequiredService<IConversationService>();
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
Agent? fromAgent = null;
if (!string.IsNullOrEmpty(message.CurrentAgentId))
{
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the pdf file(s).",
fromAgent = await agentService.LoadAgent(message.CurrentAgentId);
}
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Agent",
Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).",
TemplateDict = new Dictionary<string, object>()
};
var response = await GetChatCompletion(fileAgent, dialogs);
var wholeDialogs = routingCtx.GetDialogs();
if (wholeDialogs.IsNullOrEmpty())
{
wholeDialogs = conv.GetDialogHistory();
}
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var response = await GetChatCompletion(agent, dialogs);
message.Content = response;
return true;
}

62
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -1,9 +1,6 @@
using System.Text.Json.Nodes;
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Loggers;
using GenerativeAI;
using GenerativeAI.Core;
using GenerativeAI.Types;
@ -43,7 +40,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
}
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var aiModel = client.CreateGenerativeModel(_model);
var aiModel = client.CreateGenerativeModel(_model.ToModelId());
var (prompt, request) = PrepareOptions(aiModel, agent, conversations);
var response = await aiModel.GenerateContentAsync(request);
@ -101,7 +98,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
}
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGenerativeModel(_model);
var chatClient = client.CreateGenerativeModel(_model.ToModelId());
var (prompt, messages) = PrepareOptions(chatClient, agent, conversations);
var response = await chatClient.GenerateContentAsync(messages);
@ -165,7 +162,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
{
var client = ProviderHelper.GetGeminiClient(Provider, _model, _services);
var chatClient = client.CreateGenerativeModel(_model);
var chatClient = client.CreateGenerativeModel(_model.ToModelId());
var (prompt, messages) = PrepareOptions(chatClient,agent, conversations);
var asyncEnumerable = chatClient.StreamContentAsync(messages);
@ -207,6 +204,10 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var googleSettings = _services.GetRequiredService<GoogleAiSettings>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
renderedInstructions = [];
// Add settings
@ -298,7 +299,50 @@ public class GeminiChatCompletionProvider : IChatCompletion
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
contents.Add(new Content(text, AgentRole.User));
var contentParts = new List<Part> { new() { Text = text } };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType,
Data = Convert.ToBase64String(bytes)
}
});
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType,
Data = Convert.ToBase64String(bytes)
}
});
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
contentParts.Add(new Part()
{
FileData = new()
{
FileUri = file.FileUrl
}
});
}
}
}
contents.Add(new Content(contentParts, AgentRole.User));
convPrompts.Add($"{AgentRole.User}: {text}");
}
else if (message.Role == AgentRole.Assistant)

2
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ProviderHelper.cs
View file

@ -7,7 +7,7 @@ public static class ProviderHelper
public static GenerativeAI.GoogleAi GetGeminiClient(string provider, string model, IServiceProvider services)
{
var aiSettings = services.GetRequiredService<GoogleAiSettings>();
if (aiSettings == null || aiSettings.Gemini ==null || string.IsNullOrEmpty(aiSettings.Gemini.ApiKey))
if (string.IsNullOrEmpty(aiSettings?.Gemini?.ApiKey))
{
var settingsService = services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(provider, model);

1
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -218,7 +218,6 @@ public class ChatCompletionProvider : IChatCompletion
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);