SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add image process fn

Browse source
This commit is contained in:
Jicheng Lu 2024-06-06 13:30:39 -05:00
parent b336ef57a4
commit a66df13706
8 changed files with 196 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -3,9 +3,10 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2);
IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int? offset = null);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false);
string GetMessageFile(string conversationId, string messageId, string fileName);
bool HasConversationFiles(string conversationId);
Task<bool> SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files);
string GetUserAvatar();

6
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -14,4 +14,10 @@ public class BotSharpFile
[JsonPropertyName("file_url")]
public string FileUrl { get; set; } = string.Empty;
[JsonPropertyName("content_type")]
public string ContentType { get; set; } = string.Empty;
[JsonPropertyName("file_storage_url")]
public string FileStorageUrl { get; set; } = string.Empty;
}

3
src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs
View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Google.Settings;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Messaging;
@ -6,7 +5,6 @@ using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Routing.Planning;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Templating;
using BotSharp.Core.Files;
using BotSharp.Core.Instructs;
using BotSharp.Core.Messaging;
using BotSharp.Core.Routing.Planning;
@ -44,7 +42,6 @@ public class ConversationPlugin : IBotSharpPlugin
services.AddScoped<IConversationStorage, ConversationStorage>();
services.AddScoped<IConversationService, ConversationService>();
services.AddScoped<IConversationStateService, ConversationStateService>();
services.AddScoped<IBotSharpFileService, BotSharpFileService>();
services.AddScoped<ITranslationService, TranslationService>();
// Rich content messaging

27
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs
View file

@ -1,14 +1,14 @@
using BotSharp.Abstraction.Browsing;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.EntityFrameworkCore;
using System.IO;
using System.Linq;
using System.Threading;
namespace BotSharp.Core.Files;
public partial class BotSharpFileService
{
public IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 1)
public IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int? offset = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
@ -25,7 +25,16 @@ public partial class BotSharpFileService
offset = MAX_OFFSET;
}
var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList();
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList();
return files;
}
@ -83,6 +92,16 @@ public partial class BotSharpFileService
return found;
}
public bool HasConversationFiles(string conversationId)
{
if (string.IsNullOrEmpty(conversationId)) return false;
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
if (!ExistDirectory(dir)) return false;
return Directory.GetDirectories(dir).Count() > 0;
}
public async Task<bool> SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files)
{
if (files.IsNullOrEmpty()) return false;
@ -122,7 +141,7 @@ public partial class BotSharpFileService
{
var path = Path.Combine(preFixPath, fileName);
await web.GoToPage(contextId, path);
path = Path.Combine(preFixPath, $"{Guid.NewGuid()}.png");
path = Path.Combine(preFixPath, $"{Guid.NewGuid()}.{i + 1}.png");
await web.ScreenshotAsync(contextId, path);
}
}

21
src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs Normal file
View file

@ -0,0 +1,21 @@
using BotSharp.Core.Files.Hooks;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Core.Files;
public class FilePlugin : IBotSharpPlugin
{
public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d";
public string Name => "File";
public string Description => "Provides file processing funcationality.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
services.AddScoped<IBotSharpFileService, BotSharpFileService>();
services.AddScoped<IAgentHook, AttachmentProcessingHook>();
}
}

89
src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs Normal file
View file

@ -0,0 +1,89 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.MLTasks;
namespace BotSharp.Core.Files.Functions;
public class LoadAttachmentFn : IFunctionCallback
{
public string Name => "load_attachment";
public string Indication => "Analyzing files";
private readonly IServiceProvider _services;
private readonly ILogger<LoadAttachmentFn> _logger;
private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
public LoadAttachmentFn(
IServiceProvider services,
ILogger<LoadAttachmentFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var conv = _services.GetRequiredService<IConversationService>();
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(AIAssistant);
var fileAgent = new Agent
{
Id = agent.Id,
Name = agent.Name,
Instruction = "Please describe the images.",
TemplateDict = new Dictionary<string, object>()
};
var response = await GetChatCompletion(fileAgent, dialogs);
message.Content = response;
message.StopCompletion = true;
return true;
}
private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var files = fileService.GetChatImages(conversationId, dialogs);
foreach (var dialog in dialogs)
{
var found = files.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
{
FileName = x.FileName,
ContentType = x.ContentType,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}
return dialogs;
}
private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs)
{
try
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;
}
catch (Exception ex)
{
var error = $"Error when analyzing files.";
_logger.LogWarning($"{error} {ex.Message}");
return error;
}
}
}

50
src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs Normal file
View file

@ -0,0 +1,50 @@
namespace BotSharp.Core.Files.Hooks;
public class AttachmentProcessingHook : AgentHookBase
{
private readonly IServiceProvider _services;
private readonly AgentSettings _agentSettings;
public override string SelfId => string.Empty;
public AttachmentProcessingHook(IServiceProvider services, AgentSettings settings)
: base(services, settings)
{
_services = services;
_agentSettings = settings;
}
public override bool OnFunctionsLoaded(List<FunctionDef> functions)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var conv = _services.GetRequiredService<IConversationService>();
var hasConvFiles = fileService.HasConversationFiles(conv.ConversationId);
if (hasConvFiles)
{
var json = JsonSerializer.Serialize(new
{
user_question = new
{
type = "string",
description = $"The question asked by user, which is related to analyzing images or other files."
}
});
functions.Add(new FunctionDef
{
Name = "load_attachment",
Description = $"If the user's request is related to analyzing files, you can call this function to analyze files.",
Parameters =
{
Properties = JsonSerializer.Deserialize<JsonDocument>(json),
Required = new List<string>
{
"user_question"
}
}
});
}
return true;
}
}

21
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -229,12 +229,6 @@ public class ChatCompletionProvider : IChatCompletion
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
var chatFiles = new List<MessageFileModel>();
if (allowMultiModal)
{
chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList();
}
var chatCompletionsOptions = new ChatCompletionsOptions();
if (!string.IsNullOrEmpty(agent.Instruction))
@ -304,16 +298,6 @@ public class ChatCompletionProvider : IChatCompletion
new ChatMessageTextContentItem(text)
};
var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList();
if (!files.IsNullOrEmpty())
{
foreach (var file in files)
{
using var stream = File.OpenRead(file.FileStorageUrl);
chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low));
}
}
if (!message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
@ -329,6 +313,11 @@ public class ChatCompletionProvider : IChatCompletion
using var stream = new MemoryStream(bytes, 0, bytes.Length);
chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low));
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
using var stream = File.OpenRead(file.FileStorageUrl);
chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low));
}
}
}