SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add instruct multi modal

Browse source
This commit is contained in:
Jicheng Lu 2024-05-14 11:51:39 -05:00
parent f3bbb0259f
commit 83a78d2b5a
6 changed files with 104 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -18,4 +18,11 @@ public interface IBotSharpFileService
/// <returns></returns>
bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
/// <summary>
/// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
(string, byte[]) GetFileInfoFromData(string data);
}

11
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -4,14 +4,11 @@ namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile
{
[JsonPropertyName("file_name")]
public string FileName { get; set; }
public string FileName { get; set; } = string.Empty;
[JsonPropertyName("file_data")]
public string FileData { get; set; }
public string FileData { get; set; } = string.Empty;
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
[JsonPropertyName("file_size")]
public int FileSize { get; set; }
[JsonPropertyName("file_url")]
public string FileUrl { get; set; } = string.Empty;
}

59
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
View file

@ -8,6 +8,7 @@ public class BotSharpFileService : IBotSharpFileService
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
private readonly ILogger<BotSharpFileService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _allowedTypes = new List<string> { "image/png", "image/jpeg" };
@ -18,9 +19,11 @@ public class BotSharpFileService : IBotSharpFileService
public BotSharpFileService(
BotSharpDatabaseSettings dbSettings,
ILogger<BotSharpFileService> logger,
IServiceProvider services)
{
_dbSettings = dbSettings;
_logger = logger;
_services = services;
_baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
}
@ -117,19 +120,26 @@ public class BotSharpFileService : IBotSharpFileService
var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true);
if (string.IsNullOrEmpty(dir)) return;
for (int i = 0; i < files.Count; i++)
try
{
var file = files[i];
if (string.IsNullOrEmpty(file.FileData))
for (int i = 0; i < files.Count; i++)
{
continue;
}
var file = files[i];
if (string.IsNullOrEmpty(file.FileData))
{
continue;
}
var bytes = GetFileBytes(file.FileData);
var fileType = Path.GetExtension(file.FileName);
var fileName = $"{i + 1}{fileType}";
Thread.Sleep(100);
File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
var (_, bytes) = GetFileInfoFromData(file.FileData);
var fileType = Path.GetExtension(file.FileName);
var fileName = $"{i + 1}{fileType}";
Thread.Sleep(100);
File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
}
}
catch (Exception ex)
{
_logger.LogError($"Error when saving conversation files: {ex.Message}");
}
}
@ -179,6 +189,23 @@ public class BotSharpFileService : IBotSharpFileService
return true;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
#region Private methods
private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false)
{
@ -212,18 +239,6 @@ public class BotSharpFileService : IBotSharpFileService
return dir;
}
private byte[] GetFileBytes(string data)
{
if (string.IsNullOrEmpty(data))
{
return new byte[0];
}
var startIdx = data.IndexOf(',');
var base64Str = data.Substring(startIdx + 1);
return Convert.FromBase64String(base64Str);
}
private string GetFileContentType(string filePath)
{
string contentType;

2
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -47,7 +47,7 @@ public class CompletionProvider
logger.LogError($"Can't resolve completion provider by {provider}");
}
completer.SetModelName(model);
completer?.SetModelName(model);
return completer;
}

35
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -11,10 +11,12 @@ namespace BotSharp.OpenAPI.Controllers;
public class InstructModeController : ControllerBase
{
private readonly IServiceProvider _services;
private readonly ILogger<InstructModeController> _logger;
public InstructModeController(IServiceProvider services)
public InstructModeController(IServiceProvider services, ILogger<InstructModeController> logger)
{
_services = services;
_logger = logger;
}
[HttpPost("/instruct/{agentId}")]
@ -72,4 +74,35 @@ public class InstructModeController : ControllerBase
});
return message.Content;
}
[HttpPost("/instruct/multi-modal")]
public async Task<string> MultiModalCompletion([FromBody] IncomingMessageModel input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
state.SetState("provider", input.Provider, source: StateSource.External)
.SetState("model", input.Model, source: StateSource.External)
.SetState("model_id", input.ModelId, source: StateSource.External);
try
{
var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo");
var message = await completion.GetChatCompletions(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, input.Text)
{
Files = input.Files
}
});
return message.Content;
}
catch (Exception ex)
{
_logger.LogError($"Error in analyzing files. {ex.Message}");
return $"Error in analyzing files. {ex.Message}";
}
}
}

22
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -16,6 +16,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.ComTypes;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
@ -226,9 +227,10 @@ public class ChatCompletionProvider : IChatCompletion
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
var chatFiles = new List<MessageFileModel>();
if (settings != null && settings.MultiModal)
if (allowMultiModal)
{
chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList();
}
@ -308,6 +310,24 @@ public class ChatCompletionProvider : IChatCompletion
}
}
if (allowMultiModal && !message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
using var stream = new MemoryStream(bytes, 0, bytes.Length);
chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low));
}
}
}
//if (!string.IsNullOrEmpty(message.ImageUrl))
//{
// var uri = new Uri(message.ImageUrl);