diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
index edc04b7b..272abf0d 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
@@ -18,4 +18,11 @@ public interface IBotSharpFileService
///
bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null);
bool DeleteConversationFiles(IEnumerable conversationIds);
+
+ ///
+ /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
+ ///
+ ///
+ ///
+ (string, byte[]) GetFileInfoFromData(string data);
}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
index f679d52e..9581b83f 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
@@ -4,14 +4,11 @@ namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile
{
[JsonPropertyName("file_name")]
- public string FileName { get; set; }
+ public string FileName { get; set; } = string.Empty;
[JsonPropertyName("file_data")]
- public string FileData { get; set; }
+ public string FileData { get; set; } = string.Empty;
- [JsonPropertyName("content_type")]
- public string ContentType { get; set; }
-
- [JsonPropertyName("file_size")]
- public int FileSize { get; set; }
+ [JsonPropertyName("file_url")]
+ public string FileUrl { get; set; } = string.Empty;
}
diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
index 581569a1..d7e961be 100644
--- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
+++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
@@ -8,6 +8,7 @@ public class BotSharpFileService : IBotSharpFileService
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
+ private readonly ILogger _logger;
private readonly string _baseDir;
private readonly IEnumerable _allowedTypes = new List { "image/png", "image/jpeg" };
@@ -18,9 +19,11 @@ public class BotSharpFileService : IBotSharpFileService
public BotSharpFileService(
BotSharpDatabaseSettings dbSettings,
+ ILogger logger,
IServiceProvider services)
{
_dbSettings = dbSettings;
+ _logger = logger;
_services = services;
_baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
}
@@ -117,19 +120,26 @@ public class BotSharpFileService : IBotSharpFileService
var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true);
if (string.IsNullOrEmpty(dir)) return;
- for (int i = 0; i < files.Count; i++)
+ try
{
- var file = files[i];
- if (string.IsNullOrEmpty(file.FileData))
+ for (int i = 0; i < files.Count; i++)
{
- continue;
- }
+ var file = files[i];
+ if (string.IsNullOrEmpty(file.FileData))
+ {
+ continue;
+ }
- var bytes = GetFileBytes(file.FileData);
- var fileType = Path.GetExtension(file.FileName);
- var fileName = $"{i + 1}{fileType}";
- Thread.Sleep(100);
- File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
+ var (_, bytes) = GetFileInfoFromData(file.FileData);
+ var fileType = Path.GetExtension(file.FileName);
+ var fileName = $"{i + 1}{fileType}";
+ Thread.Sleep(100);
+ File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError($"Error when saving conversation files: {ex.Message}");
}
}
@@ -179,6 +189,23 @@ public class BotSharpFileService : IBotSharpFileService
return true;
}
+ public (string, byte[]) GetFileInfoFromData(string data)
+ {
+ if (string.IsNullOrEmpty(data))
+ {
+ return (string.Empty, new byte[0]);
+ }
+
+ var typeStartIdx = data.IndexOf(':');
+ var typeEndIdx = data.IndexOf(';');
+ var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
+
+ var base64startIdx = data.IndexOf(',');
+ var base64Str = data.Substring(base64startIdx + 1);
+
+ return (contentType, Convert.FromBase64String(base64Str));
+ }
+
#region Private methods
private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false)
{
@@ -212,18 +239,6 @@ public class BotSharpFileService : IBotSharpFileService
return dir;
}
- private byte[] GetFileBytes(string data)
- {
- if (string.IsNullOrEmpty(data))
- {
- return new byte[0];
- }
-
- var startIdx = data.IndexOf(',');
- var base64Str = data.Substring(startIdx + 1);
- return Convert.FromBase64String(base64Str);
- }
-
private string GetFileContentType(string filePath)
{
string contentType;
diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
index c55ed9a8..bc0266ed 100644
--- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
+++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
@@ -47,7 +47,7 @@ public class CompletionProvider
logger.LogError($"Can't resolve completion provider by {provider}");
}
- completer.SetModelName(model);
+ completer?.SetModelName(model);
return completer;
}
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
index 985526d9..9f00a3e0 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
@@ -11,10 +11,12 @@ namespace BotSharp.OpenAPI.Controllers;
public class InstructModeController : ControllerBase
{
private readonly IServiceProvider _services;
+ private readonly ILogger _logger;
- public InstructModeController(IServiceProvider services)
+ public InstructModeController(IServiceProvider services, ILogger logger)
{
_services = services;
+ _logger = logger;
}
[HttpPost("/instruct/{agentId}")]
@@ -72,4 +74,35 @@ public class InstructModeController : ControllerBase
});
return message.Content;
}
+
+ [HttpPost("/instruct/multi-modal")]
+ public async Task MultiModalCompletion([FromBody] IncomingMessageModel input)
+ {
+ var state = _services.GetRequiredService();
+ input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
+ state.SetState("provider", input.Provider, source: StateSource.External)
+ .SetState("model", input.Model, source: StateSource.External)
+ .SetState("model_id", input.ModelId, source: StateSource.External);
+
+ try
+ {
+ var completion = CompletionProvider.GetChatCompletion(_services, input.Provider ?? "openai", input.Model ?? "gpt-4-turbo");
+ var message = await completion.GetChatCompletions(new Agent()
+ {
+ Id = Guid.Empty.ToString(),
+ }, new List
+ {
+ new RoleDialogModel(AgentRole.User, input.Text)
+ {
+ Files = input.Files
+ }
+ });
+ return message.Content;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError($"Error in analyzing files. {ex.Message}");
+ return $"Error in analyzing files. {ex.Message}";
+ }
+ }
}
diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
index b2bbe71b..d884e386 100644
--- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
@@ -16,6 +16,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using System.Runtime.InteropServices.ComTypes;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
@@ -226,9 +227,10 @@ public class ChatCompletionProvider : IChatCompletion
var state = _services.GetRequiredService();
var settingsService = _services.GetRequiredService();
var settings = settingsService.GetSetting(Provider, _model);
+ var allowMultiModal = settings != null && settings.MultiModal;
var chatFiles = new List();
- if (settings != null && settings.MultiModal)
+ if (allowMultiModal)
{
chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList();
}
@@ -308,6 +310,24 @@ public class ChatCompletionProvider : IChatCompletion
}
}
+ if (allowMultiModal && !message.Files.IsNullOrEmpty())
+ {
+ foreach (var file in message.Files)
+ {
+ if (!string.IsNullOrEmpty(file.FileUrl))
+ {
+ var uri = new Uri(file.FileUrl);
+ chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
+ }
+ else if (!string.IsNullOrEmpty(file.FileData))
+ {
+ var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
+ using var stream = new MemoryStream(bytes, 0, bytes.Length);
+ chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low));
+ }
+ }
+ }
+
//if (!string.IsNullOrEmpty(message.ImageUrl))
//{
// var uri = new Uri(message.ImageUrl);