SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #537 from iceljc/features/refine-image-reading

Browse source 
refine chat image reading
This commit is contained in:
C. Oceania 2024-07-11 11:42:05 -05:00 committed by GitHub
parent c27acab711 5aa4595cef
commit b4f87fdd06
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 54 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Core/Using.cs
View file

@ -28,7 +28,6 @@ global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Translation.Attributes;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Http.Settings;
global using BotSharp.Core.Repository;
global using BotSharp.Core.Routing;
global using BotSharp.Core.Agents.Services;

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -85,7 +85,7 @@ public class InstructModeController : ControllerBase
try
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",
modelId: input.ModelId ?? "gpt-4", multiModal: true);
model: input.Model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = Guid.Empty.ToString(),

50
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -193,7 +193,6 @@ public class ChatCompletionProvider : IChatCompletion
return true;
}
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
@ -257,40 +256,34 @@ public class ChatCompletionProvider : IChatCompletion
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
var chat = new UserChatMessage(textPart)
{
ParticipantName = message.FunctionName
};
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal)
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
if (!message.Files.IsNullOrEmpty())
foreach (var file in message.Files)
{
foreach (var file in message.Files)
if (!string.IsNullOrEmpty(file.FileUrl))
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = fileService.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
}
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = fileService.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
}
}
messages.Add(chat);
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
}
else if (message.Role == AgentRole.Assistant)
{
@ -302,7 +295,6 @@ public class ChatCompletionProvider : IChatCompletion
return (prompt, messages, options);
}
private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
{
var prompt = string.Empty;

5
src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs
View file

@ -1,5 +1,4 @@
using System.Net.Http;
using BotSharp.Plugin.HttpHandler.LlmContexts;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
@ -58,7 +57,7 @@ public class HandleHttpRequestFn : IFunctionCallback
{
if (string.IsNullOrEmpty(url)) return null;
var settings = _services.GetRequiredService<HttpSettings>();
var settings = _services.GetRequiredService<HttpHandlerSettings>();
using var client = _httpClientFactory.CreateClient();
AddRequestHeaders(client);
@ -82,7 +81,7 @@ public class HandleHttpRequestFn : IFunctionCallback
{
client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}");
var settings = _services.GetRequiredService<HttpSettings>();
var settings = _services.GetRequiredService<HttpHandlerSettings>();
var origin = !string.IsNullOrEmpty(settings.Origin) ? settings.Origin : $"{_context.HttpContext.Request.Headers["Origin"]}";
if (!string.IsNullOrEmpty(origin))
{

1
src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs
View file

@ -3,7 +3,6 @@ using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Plugin.HttpHandler.Enums;
namespace BotSharp.Plugin.HttpHandler.Hooks;

1
src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.HttpHandler.Enums;
namespace BotSharp.Plugin.HttpHandler.Hooks;

3
src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.HttpHandler.Hooks;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Plugin.HttpHandler;
@ -18,7 +17,7 @@ public class HttpHandlerPlugin : IBotSharpPlugin
services.AddScoped(provider =>
{
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<HttpSettings>("Http");
return settingService.Bind<HttpHandlerSettings>("HttpHandler");
});
services.AddScoped<IAgentHook, HttpHandlerHook>();

4
src/Infrastructure/BotSharp.Abstraction/Http/Settings/HttpSettings.cs → src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Http.Settings;
namespace BotSharp.Plugin.HttpHandler.Settings;
public class HttpSettings
public class HttpHandlerSettings
{
public string BaseAddress { get; set; } = string.Empty;
public string Origin { get; set; } = string.Empty;

7
src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs
View file

@ -15,5 +15,8 @@ global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Messaging;
global using BotSharp.Abstraction.Messaging.Models.RichContent;
global using BotSharp.Abstraction.Options;
global using BotSharp.Abstraction.Http.Settings;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Plugin.HttpHandler.Hooks;
global using BotSharp.Plugin.HttpHandler.Settings;
global using BotSharp.Plugin.HttpHandler.LlmContexts;
global using BotSharp.Plugin.HttpHandler.Enums;

48
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -257,40 +257,34 @@ public class ChatCompletionProvider : IChatCompletion
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
var chat = new UserChatMessage(textPart)
{
ParticipantName = message.FunctionName
};
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal)
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
if (!message.Files.IsNullOrEmpty())
foreach (var file in message.Files)
{
foreach (var file in message.Files)
if (!string.IsNullOrEmpty(file.FileUrl))
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = fileService.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
}
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = fileService.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
}
}
messages.Add(chat);
messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
}
else if (message.Role == AgentRole.Assistant)
{

2
src/WebStarter/appsettings.json
View file

@ -154,7 +154,7 @@
"Driver": "Playwright"
},
"Http": {
"HttpHandler": {
"BaseAddress": "",
"Origin": ""
},