SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1068 from iceljc/features/refine-file-instruct

Browse source 
Features/refine file instruct
This commit is contained in:
Haiping 2025-06-11 08:58:19 -05:00 committed by GitHub
parent 4977cc76fc 0c820aac8e
commit 9f8767b706
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 413 additions and 261 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs
View file

@ -13,6 +13,7 @@ public interface IAgentService
Task<string> RefreshAgents();
Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null, bool byName = false);
Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions();
/// <summary>
/// Load agent configurations and trigger hooks

7
src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentUtility.cs
View file

@ -26,8 +26,7 @@ public class AgentUtility
public class UtilityItem
{
[JsonPropertyName("function_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionName { get; set; }
public string FunctionName { get; set; } = null!;
[JsonPropertyName("template_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
@ -36,4 +35,8 @@ public class UtilityItem
[JsonPropertyName("visibility_expression")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? VisibilityExpression { get; set; }
[JsonPropertyName("description")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Description { get; set; }
}

4
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -7,9 +7,9 @@ public interface IFileStorageService
#region Common
string GetDirectory(string conversationId);
IEnumerable<string> GetFiles(string relativePath, string? searchQuery = null);
byte[] GetFileBytes(string fileStorageUrl);
BinaryData GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
bool SaveFileBytesToPath(string filePath, BinaryData binary);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);

11
src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs
View file

@ -3,16 +3,23 @@ namespace BotSharp.Abstraction.Files.Models;
public class InstructFileModel : FileBase
{
/// <summary>
/// File extension without dot
/// File extension
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
/// <summary>
/// External file url
/// File url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
/// <summary>
/// File url
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; }
}

15
src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs
View file

@ -11,11 +11,16 @@ public static class FileUtility
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public static (string, byte[]) GetFileInfoFromData(string data)
public static (string?, BinaryData) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
return (null, BinaryData.Empty);
}
if (!data.StartsWith("data:"))
{
return (null, BinaryData.FromBytes(Convert.FromBase64String(data)));
}
var typeStartIdx = data.IndexOf(':');
@ -25,13 +30,13 @@ public static class FileUtility
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
return (contentType, BinaryData.FromBytes(Convert.FromBase64String(base64Str)));
}
public static string BuildFileDataFromFile(string fileName, byte[] bytes)
public static string BuildFileDataFromFile(string fileName, BinaryData binary)
{
var contentType = GetFileContentType(fileName);
var base64 = Convert.ToBase64String(bytes);
var base64 = Convert.ToBase64String(binary.ToArray());
return $"data:{contentType};base64,{base64}";
}

101
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Utility.cs Normal file
View file

@ -0,0 +1,101 @@
using BotSharp.Abstraction.Repositories.Settings;
using System.IO;
namespace BotSharp.Core.Agents.Services;
public partial class AgentService
{
public async Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions()
{
var utilities = new List<AgentUtility>();
var hooks = _services.GetServices<IAgentUtilityHook>();
foreach (var hook in hooks)
{
hook.AddUtilities(utilities);
}
utilities = utilities.Where(x => !string.IsNullOrWhiteSpace(x.Category)
&& !string.IsNullOrWhiteSpace(x.Name)
&& !x.Items.IsNullOrEmpty()).ToList();
var allItems = utilities.SelectMany(x => x.Items).ToList();
var functionNames = allItems.Select(x => x.FunctionName).Distinct().ToList();
var mapper = await GetAgentDocs(functionNames);
allItems.ForEach(x =>
{
if (mapper.ContainsKey(x.FunctionName))
{
x.Description = mapper[x.FunctionName];
}
});
return utilities;
}
#region Private methods
private async ValueTask<IDictionary<string, string>> GetAgentDocs(IEnumerable<string> names)
{
var mapper = new Dictionary<string, string>();
if (names.IsNullOrEmpty())
{
return mapper;
}
var dir = GetAgentDocDir(BuiltInAgentId.UtilityAssistant);
if (string.IsNullOrEmpty(dir))
{
return mapper;
}
var matchDocs = Directory.GetFiles(dir, "*.md")
.Where(x => names.Contains(Path.GetFileNameWithoutExtension(x)))
.ToList();
if (matchDocs.IsNullOrEmpty())
{
return mapper;
}
await foreach (var item in GetUtilityDescriptions(matchDocs))
{
mapper[item.Key] = item.Value;
}
return mapper;
}
private string GetAgentDocDir(string agentId)
{
var dbSettings = _services.GetRequiredService<BotSharpDatabaseSettings>();
var agentSettings = _services.GetRequiredService<AgentSettings>();
var dir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, agentSettings.DataDir, agentId, "docs");
if (!Directory.Exists(dir))
{
dir = string.Empty;
}
return dir;
}
private async IAsyncEnumerable<KeyValuePair<string, string>> GetUtilityDescriptions(IEnumerable<string> docs)
{
foreach (var doc in docs)
{
var content = string.Empty;
try
{
content = await File.ReadAllTextAsync(doc);
}
catch { }
if (string.IsNullOrWhiteSpace(content))
{
continue;
}
var fileName = Path.GetFileNameWithoutExtension(doc);
yield return new KeyValuePair<string, string>(fileName, content);
}
}
#endregion
}

6
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -145,11 +145,7 @@ public class ConversationStateService : IConversationStateService
newPair.Values = new List<StateValue> { newValue };
_curStates[name] = newPair;
}
else if (isNoChange)
{
// do nothing
}
else
else if (!isNoChange)
{
_curStates[name].Values.Add(newValue);
}

8
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Instructs.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
@ -14,12 +13,11 @@ public partial class FileInstructService
}
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model);
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
var audioBinary = await DownloadFile(audio);
using var stream = audioBinary.ToStream();
stream.Position = 0;
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var fileName = BuildFileName(audio.FileName, audio.FileExtension, "audio", "wav");
var content = await completion.TranscriptTextAsync(stream, fileName, text ?? string.Empty);
stream.Close();
return content;

37
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -1,7 +1,5 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using System.IO;
using BotSharp.Abstraction.Infrastructures;
namespace BotSharp.Core.Files.Services;
@ -21,7 +19,12 @@ public partial class FileInstructService
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
Files = images?.Select(x => new BotSharpFile
{
FileUrl = x.FileUrl,
FileData = x.FileData,
ContentType = x.ContentType
}).ToList() ?? []
}
});
@ -76,12 +79,11 @@ public partial class FileInstructService
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
var binary = await DownloadFile(image);
using var stream = binary.ToStream();
stream.Position = 0;
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var message = await completion.GetImageVariation(new Agent()
{
Id = innerAgentId
@ -113,12 +115,11 @@ public partial class FileInstructService
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
var binary = await DownloadFile(image);
using var stream = binary.ToStream();
stream.Position = 0;
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
@ -153,19 +154,17 @@ public partial class FileInstructService
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var imageBytes = await DownloadFile(image);
var maskBytes = await DownloadFile(mask);
var imageBinary = await DownloadFile(image);
var maskBinary = await DownloadFile(mask);
using var imageStream = new MemoryStream();
imageStream.Write(imageBytes, 0, imageBytes.Length);
using var imageStream = imageBinary.ToStream();
imageStream.Position = 0;
using var maskStream = new MemoryStream();
maskStream.Write(maskBytes, 0, maskBytes.Length);
using var maskStream = maskBinary.ToStream();
maskStream.Position = 0;
var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}";
var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png");
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId

44
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -22,7 +22,7 @@ public partial class FileInstructService
try
{
var provider = options?.Provider ?? "openai";
var pdfFiles = await DownloadFiles(sessionDir, files);
var pdfFiles = await DownloadAndSaveFiles(sessionDir, files);
var targetFiles = pdfFiles;
if (provider != "google-ai")
@ -78,44 +78,38 @@ public partial class FileInstructService
}
#region Private methods
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
private async Task<IEnumerable<string>> DownloadAndSaveFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
}
var downloadTasks = files.Select(x => DownloadFile(x));
await Task.WhenAll(downloadTasks);
var locs = new List<string>();
foreach (var file in files)
for (int i = 0; i < files.Count; i++)
{
var binary = downloadTasks.ElementAt(i).Result;
if (binary == null || binary.IsEmpty)
{
continue;
}
try
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
}
var guid = Guid.NewGuid().ToString();
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, createNew: true);
if (!bytes.IsNullOrEmpty())
{
var guid = Guid.NewGuid().ToString();
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, true);
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(outputDir, bytes);
locs.Add(outputDir);
}
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(outputDir, binary);
locs.Add(outputDir);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when saving pdf file.");
_logger.LogWarning(ex, $"Error when saving #{i + 1} {extension} file.");
continue;
}
}

46
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
View file

@ -1,4 +1,6 @@
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService : IFileInstructService
@ -32,21 +34,31 @@ public partial class FileInstructService : IFileInstructService
}
}
private async Task<byte[]> DownloadFile(InstructFileModel file)
private async Task<BinaryData> DownloadFile(InstructFileModel file)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
}
var binary = BinaryData.Empty;
return bytes;
try
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
var bytes = await client.GetByteArrayAsync(file.FileUrl);
binary = BinaryData.FromBytes(bytes);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
}
return binary;
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when downloading file {file.FileUrl}");
return binary;
}
}
private async Task<string?> GetAgentTemplate(string agentId, string? templateName)
@ -66,5 +78,13 @@ public partial class FileInstructService : IFileInstructService
var instruction = agentService.RenderedTemplate(agent, templateName);
return instruction;
}
private string BuildFileName(string? name, string? extension, string defaultName, string defaultExtension)
{
var fname = name.IfNullOrEmptyAs(defaultName);
var fextension = extension.IfNullOrEmptyAs(defaultExtension);
fextension = fextension.StartsWith(".") ? fextension.Substring(1) : fextension;
return $"{name}.{fextension}";
}
#endregion
}

8
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs
View file

@ -30,12 +30,12 @@ public partial class LocalFileStorageService
return Directory.GetFiles(path);
}
public byte[] GetFileBytes(string fileStorageUrl)
public BinaryData GetFileBytes(string fileStorageUrl)
{
using var stream = File.OpenRead(fileStorageUrl);
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
return bytes;
return BinaryData.FromBytes(bytes);
}
public bool SaveFileStreamToPath(string filePath, Stream stream)
@ -49,11 +49,11 @@ public partial class LocalFileStorageService
return true;
}
public bool SaveFileBytesToPath(string filePath, byte[] bytes)
public bool SaveFileBytesToPath(string filePath, BinaryData binary)
{
using (var fs = new FileStream(filePath, FileMode.Create))
{
fs.Write(bytes, 0, bytes.Length);
fs.Write(binary.ToArray(), 0, binary.Length);
fs.Flush();
fs.Close();
}

4
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -136,7 +136,7 @@ public partial class LocalFileStorageService
try
{
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var subDir = Path.Combine(dir, source, $"{i + 1}");
if (!ExistDirectory(subDir))
{
@ -145,7 +145,7 @@ public partial class LocalFileStorageService
using (var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.Create))
{
fs.Write(bytes, 0, bytes.Length);
fs.Write(binary.ToArray(), 0, binary.Length);
fs.Flush(true);
fs.Close();
Thread.Sleep(100);

4
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs
View file

@ -34,8 +34,8 @@ public partial class LocalFileStorageService
}
dir = GetUserAvatarDir(user?.Id, createNewDir: true);
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes);
var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
File.WriteAllBytes(Path.Combine(dir, file.FileName), binary.ToArray());
return true;
}
catch (Exception ex)

13
src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs
View file

@ -159,17 +159,10 @@ public class AgentController : ControllerBase
}
[HttpGet("/agent/utility/options")]
public IEnumerable<AgentUtility> GetAgentUtilityOptions()
public async Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions()
{
var utilities = new List<AgentUtility>();
var hooks = _services.GetServices<IAgentUtilityHook>();
foreach (var hook in hooks)
{
hook.AddUtilities(utilities);
}
return utilities.Where(x => !string.IsNullOrWhiteSpace(x.Category)
&& !string.IsNullOrWhiteSpace(x.Name)
&& !x.Items.IsNullOrEmpty()).ToList();
var agentService = _services.GetRequiredService<IAgentService>();
return await agentService.GetAgentUtilityOptions();
}
[HttpGet("/agent/labels")]

203
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -1,11 +1,9 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.OpenAPI.ViewModels.Instructs;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.OpenAPI.Controllers;
@ -93,7 +91,12 @@ public class InstructModeController : ControllerBase
{
new RoleDialogModel(AgentRole.User, input.Text)
{
Files = input.Files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
Files = input.Files?.Select(x => new BotSharpFile
{
FileUrl = x.FileUrl,
FileData = x.FileData,
ContentType = x.ContentType
}).ToList() ?? []
}
});
@ -115,10 +118,10 @@ public class InstructModeController : ControllerBase
#region Read image
[HttpPost("/instruct/multi-modal")]
public async Task<string> MultiModalCompletion([FromBody] MultiModalRequest input)
public async Task<string> MultiModalCompletion([FromBody] MultiModalFileRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
try
{
@ -135,34 +138,32 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in reading images. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
return error;
}
}
[HttpPost("/instruct/multi-modal/upload")]
public async Task<MultiModalViewModel> MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
public async Task<MultiModalViewModel> MultiModalCompletion([FromForm] IEnumerable<IFormFile> files, [FromForm] MultiModalRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new MultiModalViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var files = new List<InstructFileModel>
var fileModels = files.Select(x => new InstructFileModel
{
new InstructFileModel { FileData = data }
};
FileData = FileUtility.BuildFileDataFromFile(x)
}).ToList();
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadImages(text, files, new InstructOptions
var content = await fileInstruct.ReadImages(request?.Text ?? string.Empty, fileModels, new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
});
viewModel.Content = content;
return viewModel;
@ -170,7 +171,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in reading image upload. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
viewModel.Message = error;
return viewModel;
}
@ -182,7 +183,7 @@ public class InstructModeController : ControllerBase
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
@ -202,7 +203,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image generation. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
@ -214,7 +215,7 @@ public class InstructModeController : ControllerBase
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] ImageVariationRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
@ -239,30 +240,34 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image variation. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-variation/upload")]
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null)
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] MultiModalRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var fileData = FileUtility.BuildFileDataFromFile(file);
var message = await fileInstruct.VaryImage(new InstructFileModel { FileData = fileData }, new InstructOptions
var message = await fileInstruct.VaryImage(new InstructFileModel
{
Provider = provider,
Model = model,
AgentId = agentId
FileData = fileData,
FileName = Path.GetFileNameWithoutExtension(file.FileName),
FileExtension = Path.GetExtension(file.FileName)
},
new InstructOptions
{
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId
});
imageViewModel.Content = message.Content;
@ -272,7 +277,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image variation upload. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
@ -283,7 +288,7 @@ public class InstructModeController : ControllerBase
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
@ -306,31 +311,35 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image edit. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-edit/upload")]
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null,
[FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] MultiModalRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var fileData = FileUtility.BuildFileDataFromFile(file);
var message = await fileInstruct.EditImage(text, new InstructFileModel { FileData = fileData }, new InstructOptions
var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, new InstructFileModel
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
FileData = fileData,
FileName = Path.GetFileNameWithoutExtension(file.FileName),
FileExtension = Path.GetExtension(file.FileName)
},
new InstructOptions
{
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
});
imageViewModel.Content = message.Content;
@ -341,7 +350,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image edit upload. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
@ -352,7 +361,7 @@ public class InstructModeController : ControllerBase
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
@ -377,35 +386,44 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image mask edit. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
}
[HttpPost("/instruct/image-mask-edit/upload")]
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask,
[FromForm] string text, [FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] List<MessageState>? states = null, [FromForm] string? agentId = null, [FromForm] string? templateName = null)
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var imageData = FileUtility.BuildFileDataFromFile(image);
var maskData = FileUtility.BuildFileDataFromFile(mask);
var message = await fileInstruct.EditImage(text,
new InstructFileModel { FileData = imageData },
new InstructFileModel { FileData = maskData }, new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
});
var message = await fileInstruct.EditImage(request?.Text ?? string.Empty,
new InstructFileModel
{
FileData = imageData,
FileName = Path.GetFileNameWithoutExtension(image.FileName),
FileExtension = Path.GetExtension(image.FileName)
},
new InstructFileModel
{
FileData = maskData,
FileName = Path.GetFileNameWithoutExtension(mask.FileName),
FileExtension = Path.GetExtension(mask.FileName)
},
new InstructOptions
{
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
});
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
@ -415,7 +433,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in image mask edit upload. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
imageViewModel.Message = error;
return imageViewModel;
}
@ -424,10 +442,10 @@ public class InstructModeController : ControllerBase
#region Pdf
[HttpPost("/instruct/pdf-completion")]
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] MultiModalRequest input)
public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] MultiModalFileRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new PdfCompletionViewModel();
try
@ -446,36 +464,33 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in pdf completion. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
viewModel.Message = error;
return viewModel;
}
}
[HttpPost("/instruct/pdf-completion/upload")]
public async Task<PdfCompletionViewModel> PdfCompletion(IFormFile file, [FromForm] string text,
[FromForm] string? provider = null, [FromForm] string? model = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
public async Task<PdfCompletionViewModel> PdfCompletion([FromForm] IEnumerable<IFormFile> files, [FromForm] MultiModalRequest request)
{
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new PdfCompletionViewModel();
try
{
var data = FileUtility.BuildFileDataFromFile(file);
var files = new List<InstructFileModel>
var fileModels = files.Select(x => new InstructFileModel
{
new InstructFileModel { FileData = data }
};
FileData = FileUtility.BuildFileDataFromFile(x)
}).ToList();
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(text, files, new InstructOptions
var content = await fileInstruct.ReadPdf(request?.Text ?? string.Empty, fileModels, new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
});
viewModel.Content = content;
return viewModel;
@ -483,7 +498,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in pdf completion upload. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
viewModel.Message = error;
return viewModel;
}
@ -496,7 +511,7 @@ public class InstructModeController : ControllerBase
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new SpeechToTextViewModel();
try
@ -519,32 +534,36 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in speech to text. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
viewModel.Message = error;
return viewModel;
}
}
[HttpPost("/instruct/speech-to-text/upload")]
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file,
[FromForm] string? provider = null, [FromForm] string? model = null,
[FromForm] string? text = null, [FromForm] List<MessageState>? states = null,
[FromForm] string? agentId = null, [FromForm] string? templateName = null)
public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm] MultiModalRequest request)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var viewModel = new SpeechToTextViewModel();
try
{
var audioData = FileUtility.BuildFileDataFromFile(file);
var content = await fileInstruct.SpeechToText(new InstructFileModel { FileData = audioData }, text, new InstructOptions
var content = await fileInstruct.SpeechToText(new InstructFileModel
{
FileData = audioData,
FileName = Path.GetFileNameWithoutExtension(file.FileName),
FileExtension = Path.GetExtension(file.FileName)
},
request?.Text ?? string.Empty,
new InstructOptions
{
Provider = provider,
Model = model,
AgentId = agentId,
TemplateName = templateName
Provider = request?.Provider,
Model = request?.Model,
AgentId = request?.AgentId,
TemplateName = request?.TemplateName
});
viewModel.Content = content;
@ -553,7 +572,7 @@ public class InstructModeController : ControllerBase
catch (Exception ex)
{
var error = $"Error in speech-to-text upload. {ex.Message}";
_logger.LogError(error);
_logger.LogError(ex, error);
viewModel.Message = error;
return viewModel;
}
@ -563,7 +582,7 @@ public class InstructModeController : ControllerBase
public async Task<IActionResult> TextToSpeech([FromBody] TextToSpeechRequest input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model);
var binaryData = await completion.GenerateAudioAsync(input.Text);

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -245,8 +245,8 @@ public class UserController : ControllerBase
private FileContentResult BuildFileResult(string file)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var bytes = fileStorage.GetFileBytes(file);
return File(bytes, "application/octet-stream", Path.GetFileName(file));
var binary = fileStorage.GetFileBytes(file);
return File(binary.ToArray(), "application/octet-stream", Path.GetFileName(file));
}
#endregion
}

6
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs
View file

@ -1,3 +1,4 @@
using BotSharp.OpenAPI.ViewModels.Instructs.Request;
using System.Text.Json.Serialization;
namespace BotSharp.Abstraction.Instructs.Models;
@ -17,14 +18,17 @@ public class InstructBaseRequest
public virtual string? TemplateName { get; set; }
[JsonPropertyName("states")]
public List<MessageState> States { get; set; } = [];
public List<InstructState> States { get; set; } = [];
}
public class MultiModalRequest : InstructBaseRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
}
public class MultiModalFileRequest : MultiModalRequest
{
[JsonPropertyName("files")]
public List<InstructFileModel> Files { get; set; } = [];
}

7
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructState.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.OpenAPI.ViewModels.Instructs.Request;
public class InstructState
{
public string Key { get; set; }
public string Value { get; set; }
}

4
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -87,8 +87,8 @@ public class HandleAudioRequestFn : IFunctionCallback
var fileName = Path.GetFileName(file.FileStorageUrl);
if (!ParseAudioFileType(fileName)) continue;
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = new MemoryStream(bytes);
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
using var stream = binary.ToStream();
stream.Position = 0;
var result = await audioCompletion.TranscriptTextAsync(stream, fileName);

4
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -89,8 +89,8 @@ public class NativeWhisperProvider : IAudioTranscription
DownloadModel(modelType, modelLoc);
}
var bytes = _fileStorage.GetFileBytes(modelLoc);
_whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build();
var binary = _fileStorage.GetFileBytes(modelLoc);
_whisperProcessor = WhisperFactory.FromBuffer(binary.ToArray()).CreateBuilder().WithLanguage("auto").Build();
}
catch (Exception ex)
{

10
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -308,7 +308,7 @@ public class ChatCompletionProvider : IChatCompletion
ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
}));
messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
}
else if (message.Role == AgentRole.User)
{
@ -322,15 +322,15 @@ public class ChatCompletionProvider : IChatCompletion
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))

2
src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs
View file

@ -28,7 +28,7 @@ public class SignalRHub : Hub
_logger.LogInformation($"SignalR Hub: {_user.FirstName} {_user.LastName} ({Context.User.Identity.Name}) connected in {Context.ConnectionId}");
var convService = _services.GetRequiredService<IConversationService>();
_context.HttpContext.Request.Query.TryGetValue("conversationId", out var conversationId);
_context.HttpContext.Request.Query.TryGetValue("conversation-id", out var conversationId);
if (!string.IsNullOrEmpty(conversationId))
{

4
src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -270,10 +270,10 @@ public class ChatCompletionProvider : IChatCompletion
{
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
}));
messages.Add(new ToolChatMessage(message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
}
else if (message.Role == AgentRole.User)
{

4
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -90,8 +90,8 @@ public class HandleEmailSenderFn : IFunctionCallback
{
if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl);
builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType));
var fileBinary = fileStorage.GetFileBytes(file.FileStorageUrl);
builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBinary.ToArray(), ContentType.Parse(file.ContentType));
Thread.Sleep(100);
}
}

4
src/Plugins/BotSharp.Plugin.ExcelHandler/Functions/HandleExcelRequestFn.cs
View file

@ -113,8 +113,8 @@ public class HandleExcelRequestFn : IFunctionCallback
_currentFileName = Path.GetFileName(file.FileStorageUrl);
var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(bytes);
var binary = _fileStorage.GetFileBytes(file.FileStorageUrl);
var workbook = ConvertToWorkBook(binary.ToArray());
var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook);
sqlCommandList.AddRange(currentCommandList);

5
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -77,9 +77,8 @@ public class EditImageFn : IFunctionCallback
};
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl);
using var stream = new MemoryStream();
stream.Write(fileBytes);
var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl);
using var stream = fileBinary.ToStream();
stream.Position = 0;
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty);
stream.Close();

12
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
View file

@ -307,26 +307,26 @@ public class GeminiChatCompletionProvider : IChatCompletion
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType,
Data = Convert.ToBase64String(bytes)
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
contentParts.Add(new Part()
{
InlineData = new()
{
MimeType = contentType,
Data = Convert.ToBase64String(bytes)
MimeType = contentType.IfNullOrEmptyAs(file.ContentType),
Data = Convert.ToBase64String(binary.ToArray())
}
});
}

36
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -48,12 +48,12 @@ public partial class KnowledgeService
try
{
// Get document info
var (contentType, bytes) = await GetFileInfo(file);
var contents = await GetFileContent(contentType, bytes, option ?? ChunkOption.Default());
var (contentType, binary) = await GetFileInfo(file);
var contents = await GetFileContent(contentType, binary, option ?? ChunkOption.Default());
// Save document
var fileId = Guid.NewGuid();
var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, binary);
if (!saved)
{
failedFiles.Add(file.FileName);
@ -342,11 +342,11 @@ public partial class KnowledgeService
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
private async Task<(string, byte[])> GetFileInfo(ExternalFileModel file)
private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file)
{
if (file == null)
{
return (string.Empty, new byte[0]);
return (string.Empty, BinaryData.Empty);
}
if (!string.IsNullOrWhiteSpace(file.FileUrl))
@ -355,37 +355,38 @@ public partial class KnowledgeService
var contentType = FileUtility.GetFileContentType(file.FileName);
using var client = http.CreateClient();
var bytes = await client.GetByteArrayAsync(file.FileUrl);
return (contentType, bytes);
return (contentType, BinaryData.FromBytes(bytes));
}
else if (!string.IsNullOrWhiteSpace(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
return (contentType, bytes);
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
return (contentType, binary);
}
return (string.Empty, new byte[0]);
return (string.Empty, BinaryData.Empty);
}
#region Read doc content
private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes, ChunkOption option)
private async Task<IEnumerable<string>> GetFileContent(string contentType, BinaryData binary, ChunkOption option)
{
IEnumerable<string> results = new List<string>();
if (contentType.IsEqualTo(MediaTypeNames.Text.Plain))
{
results = await ReadTxt(bytes, option);
results = await ReadTxt(binary, option);
}
else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))
{
results = await ReadPdf(bytes);
results = await ReadPdf(binary);
}
return results;
}
private async Task<IEnumerable<string>> ReadTxt(byte[] bytes, ChunkOption option)
private async Task<IEnumerable<string>> ReadTxt(BinaryData binary, ChunkOption option)
{
using var stream = new MemoryStream(bytes);
using var stream = binary.ToStream();
stream.Position = 0;
using var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync();
reader.Close();
@ -395,18 +396,17 @@ public partial class KnowledgeService
return lines;
}
private async Task<IEnumerable<string>> ReadPdf(byte[] bytes)
private async Task<IEnumerable<string>> ReadPdf(BinaryData binary)
{
return Enumerable.Empty<string>();
}
#endregion
private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes)
private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData binary)
{
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
var data = BinaryData.FromBytes(bytes);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, binary);
return saved;
}

4
src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs
View file

@ -128,8 +128,8 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage!.GetFileBytes(file.FileStorageUrl);
contents.Add(new DataContent(bytes, contentType));
var binary = fileStorage!.GetFileBytes(file.FileStorageUrl);
contents.Add(new DataContent(binary.ToMemory(), contentType));
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{

2
src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs
View file

@ -48,7 +48,7 @@ public class AgentUtilityMongoElement
public class AgentUtilityItemMongoElement
{
public string? FunctionName { get; set; }
public string FunctionName { get; set; }
public string? TemplateName { get; set; }
public string? VisibilityExpression { get; set; }
}

10
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -274,7 +274,7 @@ public class ChatCompletionProvider : IChatCompletion
ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
}));
messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
}
else if (message.Role == AgentRole.User)
{
@ -288,15 +288,15 @@ public class ChatCompletionProvider : IChatCompletion
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))

8
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -632,15 +632,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
{
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))

11
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs
View file

@ -25,16 +25,17 @@ public partial class TencentCosService
}
}
public byte[] GetFileBytes(string fileStorageUrl)
public BinaryData GetFileBytes(string fileStorageUrl)
{
try
{
return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl);
var bytes = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl);
return BinaryData.FromBytes(bytes);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when getting file bytes (url: {fileStorageUrl}).");
return Array.Empty<byte>();
return BinaryData.Empty;
}
}
@ -53,13 +54,13 @@ public partial class TencentCosService
}
}
public bool SaveFileBytesToPath(string filePath, byte[] bytes)
public bool SaveFileBytesToPath(string filePath, BinaryData binary)
{
if (string.IsNullOrEmpty(filePath)) return false;
try
{
return _cosClient.BucketClient.UploadBytes(filePath, bytes);
return _cosClient.BucketClient.UploadBytes(filePath, binary.ToArray());
}
catch (Exception ex)
{

4
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -132,9 +132,9 @@ public partial class TencentCosService
try
{
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var subDir = $"{dir}/{source}/{i + 1}";
_cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes);
_cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", binary.ToArray());
}
catch (Exception ex)
{

4
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs
View file

@ -28,11 +28,11 @@ public partial class TencentCosService
if (string.IsNullOrEmpty(dir)) return false;
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData);
var extension = Path.GetExtension(file.FileName);
var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}";
return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes);
return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", binary.ToArray());
}
catch (Exception ex)
{

11
tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs
View file

@ -1,9 +1,9 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
namespace BotSharp.Plugin.Google.Core
{
public class NullFileStorageService:IFileStorageService
public class NullFileStorageService : IFileStorageService
{
public string GetDirectory(string conversationId)
{
@ -15,9 +15,10 @@ namespace BotSharp.Plugin.Google.Core
return new List<string> { "FakeFile1.txt", "FakeFile2.txt" };
}
public byte[] GetFileBytes(string fileStorageUrl)
public BinaryData GetFileBytes(string fileStorageUrl)
{
return new byte[] { 0x00, 0x01, 0x02 };
var bytes = new byte[] { 0x00, 0x01, 0x02 };
return BinaryData.FromBytes(bytes);
}
public bool SaveFileStreamToPath(string filePath, Stream stream)
@ -25,7 +26,7 @@ namespace BotSharp.Plugin.Google.Core
return true;
}
public bool SaveFileBytesToPath(string filePath, byte[] bytes)
public bool SaveFileBytesToPath(string filePath, BinaryData binary)
{
return true;
}

6
tests/BotSharp.LLM.Tests/Core/TestAgentService.cs
View file

@ -6,7 +6,6 @@ using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Utilities;
using NetTopologySuite.Algorithm;
namespace BotSharp.Plugin.Google.Core
{
@ -111,5 +110,10 @@ namespace BotSharp.Plugin.Google.Core
{
return new PluginDef();
}
public Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions()
{
return Task.FromResult(Enumerable.Empty<AgentUtility>());
}
}
}