SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
Haiping 2024-08-07 19:50:28 -05:00 committed by GitHub
parent 98a02caf71 a92c76104e
commit 2f04e2c1dd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 598 additions and 692 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
View file

@ -25,6 +25,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />

9
src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs Normal file
View file

@ -0,0 +1,9 @@
namespace BotSharp.Abstraction.Files.Constants;
public class FileConstants
{
public static readonly IEnumerable<string> AudioExtensions = new List<string>
{
".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma"
};
}

40
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs → src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs
View file

@ -2,7 +2,7 @@ using System.IO;
namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
public interface IFileBasicService
{
#region Conversation
/// <summary>
@ -11,13 +11,13 @@ public interface IBotSharpFileService
/// </summary>
/// <param name="conversationId"></param>
/// <param name="source"></param>
/// <param name="conversations"></param>
/// <param name="dialogs"></param>
/// <param name="contentTypes"></param>
/// <param name="includeScreenShot"></param>
/// <param name="offset"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes,
bool includeScreenShot = false, int? offset = null);
/// <summary>
@ -28,7 +28,7 @@ public interface IBotSharpFileService
/// <param name="source"></param>
/// <param name="imageOnly"></param>
/// <returns></returns>
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
@ -45,38 +45,20 @@ public interface IBotSharpFileService
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
#endregion
#region Image
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask);
#endregion
#region Pdf
/// <summary>
/// Take screenshots of pdf pages and get response from llm
/// </summary>
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
#endregion
#region User
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
#endregion
#region Common
/// <summary>
/// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
(string, byte[]) GetFileInfoFromData(string data);
string GetDirectory(string conversationId);
string GetFileContentType(string filePath);
byte[] GetFileBytes(string fileStorageUrl);
bool SavefileToPath(string filePath, Stream stream);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);
void DeleteDirectory(string dir);
string BuildDirectory(params string[] segments);
#endregion
}

29
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs Normal file
View file

@ -0,0 +1,29 @@
namespace BotSharp.Abstraction.Files;
public interface IFileInstructService
{
#region Image
Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images);
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask);
#endregion
#region Pdf
/// <summary>
/// Take screenshots of pdf pages and get response from llm
/// </summary>
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
#endregion
#region Select file
Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId,
string? agentId = null, string? template = null, string? description = null,
bool includeBotFile = false, bool fromBreakpoint = false,
int? offset = null, IEnumerable<string>? contentTypes = null);
#endregion
}

8
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Files.Models;
public class FileSelectContext
{
[JsonPropertyName("selected_ids")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IEnumerable<int>? Selecteds { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -39,6 +39,6 @@ public class MessageFileModel
public override string ToString()
{
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}";
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}";
}
}

40
src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs Normal file
View file

@ -0,0 +1,40 @@
using Microsoft.AspNetCore.StaticFiles;
namespace BotSharp.Abstraction.Files.Utilities;
public static class FileUtility
{
/// <summary>
/// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public static (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public static string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
}

45
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs
View file

@ -1,7 +1,12 @@
using BotSharp.Abstraction.Routing.Models;
using System.Collections.Concurrent;
namespace BotSharp.Core.Agents.Services;
public partial class AgentService
{
public static ConcurrentDictionary<string, Dictionary<string,string>> AgentParameterTypes = new();
[MemoryCache(10 * 60, perInstanceCache: true)]
public async Task<Agent> LoadAgent(string id)
{
@ -49,6 +54,7 @@ public partial class AgentService
agent.Instruction = inheritedAgent.Instruction;
}
}
AddOrUpdateParameters(agent);
agent.TemplateDict = new Dictionary<string, object>();
@ -96,4 +102,43 @@ public partial class AgentService
dict[t.Key] = t.Value;
}
}
private void AddOrUpdateParameters(Agent agent)
{
var agentId = agent.Id ?? agent.Name;
if (AgentParameterTypes.ContainsKey(agentId)) return;
AddOrUpdateRoutesParameters(agentId, agent.RoutingRules);
AddOrUpdateFunctionsParameters(agentId, agent.Functions);
}
private void AddOrUpdateRoutesParameters(string agentId, List<RoutingRule> routingRules)
{
if(!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new();
foreach (var rule in routingRules.Where(x => x.Required))
{
if (string.IsNullOrEmpty(rule.FieldType)) continue;
parameterTypes.TryAdd(rule.Field, rule.FieldType);
}
AgentParameterTypes.TryAdd(agentId, parameterTypes);
}
private void AddOrUpdateFunctionsParameters(string agentId, List<FunctionDef> functions)
{
if (!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new();
var parameters = functions.Select(p => p.Parameters);
foreach (var param in parameters)
{
foreach (JsonProperty prop in param.Properties.RootElement.EnumerateObject())
{
var name = prop.Name;
var node = prop.Value;
if (node.TryGetProperty("type", out var type))
{
parameterTypes.TryAdd(name, type.GetString());
}
}
}
AgentParameterTypes.TryAdd(agentId, parameterTypes);
}
}

19
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -45,6 +45,15 @@
<NoWarn>1701;1702</NoWarn>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Planning\**" />
<Compile Remove="Translation\Models\**" />
<EmbeddedResource Remove="Planning\**" />
<EmbeddedResource Remove="Translation\Models\**" />
<None Remove="Planning\**" />
<None Remove="Translation\Models\**" />
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\agent.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instruction.liquid" />
@ -69,6 +78,7 @@
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.two_stage.2nd.task.liquid" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\translation_prompt.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instruction.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.executor.liquid" />
@ -155,6 +165,9 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\plugins\config.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
@ -172,7 +185,6 @@
<PackageReference Include="DistributedLock.Redis" Version="1.0.3" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="8.5.1" />
<PackageReference Include="Fluid.Core" Version="2.11.1" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Nanoid" Version="3.1.0" />
</ItemGroup>
@ -181,9 +193,4 @@
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Planning\" />
<Folder Include="Translation\Models\" />
</ItemGroup>
</Project>

1
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs
View file

@ -150,6 +150,7 @@ public partial class ConversationService
await HookEmitter.Emit<IConversationHook>(_services, async hook =>
await hook.OnConversationEnding(response)
);
response.FunctionName = "conversation_end";
}
}

2
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs
View file

@ -5,7 +5,7 @@ public partial class ConversationService : IConversationService
public async Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true);
fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId);

2
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
View file

@ -37,7 +37,7 @@ public partial class ConversationService : IConversationService
public async Task<bool> DeleteConversations(IEnumerable<string> ids)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var isDeleted = db.DeleteConversations(ids);
fileService.DeleteConversationFiles(ids);
return await Task.FromResult(isDeleted);

21
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -360,9 +360,28 @@ public class ConversationStateService : IConversationStateService, IDisposable
stateValue = stateValue?.ToLower();
}
SetState(property.Name, stateValue, source: StateSource.Application);
if (CheckArgType(property.Name, stateValue))
{
SetState(property.Name, stateValue, source: StateSource.Application);
}
}
}
}
}
private bool CheckArgType(string name, string value)
{
var agentTypes = AgentService.AgentParameterTypes.SelectMany(p => p.Value).ToList();
var filed = agentTypes.FirstOrDefault(t => t.Key == name);
if (filed.Key != null)
{
return filed.Value switch
{
"boolean" => bool.TryParse(value, out _),
"number" => long.TryParse(value, out _),
_ => true,
};
}
return true;
}
}

3
src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs
View file

@ -20,7 +20,8 @@ public class FilePlugin : IBotSharpPlugin
if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage)
{
services.AddScoped<IBotSharpFileService, BotSharpFileService>();
services.AddScoped<IFileBasicService, FileBasicService>();
}
services.AddScoped<IFileInstructService, FileInstructService>();
}
}

73
src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs Normal file
View file

@ -0,0 +1,73 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileBasicService
{
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
return dir;
}
public byte[] GetFileBytes(string fileStorageUrl)
{
using var stream = File.OpenRead(fileStorageUrl);
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
return bytes;
}
public bool SaveFileStreamToPath(string filePath, Stream stream)
{
if (string.IsNullOrEmpty(filePath)) return false;
using (var fileStream = new FileStream(filePath, FileMode.Create))
{
stream.CopyTo(fileStream);
}
return true;
}
public bool SaveFileBytesToPath(string filePath, byte[] bytes)
{
using (var fs = new FileStream(filePath, FileMode.Create))
{
fs.Write(bytes, 0, bytes.Length);
fs.Flush();
fs.Close();
}
return true;
}
public string GetParentDir(string dir, int level = 1)
{
var segs = dir.Split(Path.DirectorySeparatorChar);
return string.Join(Path.DirectorySeparatorChar, segs.SkipLast(level));
}
public string BuildDirectory(params string[] segments)
{
var relativePath = Path.Combine(segments);
return Path.Combine(_baseDir, relativePath);
}
public void CreateDirectory(string dir)
{
Directory.CreateDirectory(dir);
}
public bool ExistDirectory(string? dir)
{
return !string.IsNullOrEmpty(dir) && Directory.Exists(dir);
}
public void DeleteDirectory(string dir)
{
Directory.Delete(dir, true);
}
}

51
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs → src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs
View file

@ -4,19 +4,19 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
public partial class FileBasicService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty())
{
return files;
}
var messageIds = GetMessageIds(conversations, offset);
var messageIds = GetMessageIds(dialogs, offset);
var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
foreach (var messageId in messageIds)
@ -29,8 +29,11 @@ public partial class BotSharpFileService
var file = Directory.GetFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = GetFileContentType(file);
if (contentTypes?.Contains(contentType) != true) continue;
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
@ -43,7 +46,7 @@ public partial class BotSharpFileService
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds,
string source, bool imageOnly = false)
string source, IEnumerable<string>? contentTypes = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files;
@ -62,8 +65,8 @@ public partial class BotSharpFileService
foreach (var file in Directory.GetFiles(subDir))
{
var contentType = GetFileContentType(file);
if (imageOnly && !_imageTypes.Contains(contentType))
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
@ -141,7 +144,7 @@ public partial class BotSharpFileService
try
{
var (_, bytes) = GetFileInfoFromData(file.FileData);
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var subDir = Path.Combine(dir, source, $"{i + 1}");
if (!ExistDirectory(subDir))
{
@ -180,7 +183,7 @@ public partial class BotSharpFileService
{
if (ExistDirectory(newDir))
{
Directory.Delete(newDir, true);
DeleteDirectory(newDir);
}
Directory.Move(prevDir, newDir);
@ -189,7 +192,7 @@ public partial class BotSharpFileService
var botDir = Path.Combine(newDir, BOT_FILE_FOLDER);
if (ExistDirectory(botDir))
{
Directory.Delete(botDir, true);
DeleteDirectory(botDir);
}
}
}
@ -200,7 +203,7 @@ public partial class BotSharpFileService
if (!ExistDirectory(dir)) continue;
Thread.Sleep(100);
Directory.Delete(dir, true);
DeleteDirectory(dir);
}
return true;
@ -215,7 +218,7 @@ public partial class BotSharpFileService
var convDir = GetConversationDirectory(conversationId);
if (!ExistDirectory(convDir)) continue;
Directory.Delete(convDir, true);
DeleteDirectory(convDir);
}
return true;
}
@ -244,27 +247,23 @@ public partial class BotSharpFileService
return dir;
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> conversations, int? offset = null)
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> dialogs, int? offset = null)
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (dialogs.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset <= 0)
if (offset.HasValue && offset < 1)
{
offset = MIN_OFFSET;
}
else if (offset > MAX_OFFSET)
{
offset = MAX_OFFSET;
offset = 1;
}
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
}
return messageIds;
@ -285,7 +284,7 @@ public partial class BotSharpFileService
{
foreach (var screenShot in Directory.GetFiles(screenShotDir))
{
contentType = GetFileContentType(screenShot);
contentType = FileUtility.GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
@ -307,7 +306,7 @@ public partial class BotSharpFileService
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = GetFileContentType(image);
contentType = FileUtility.GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()

6
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs → src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs
View file

@ -2,7 +2,7 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
public partial class FileBasicService
{
public string GetUserAvatar()
{
@ -30,11 +30,11 @@ public partial class BotSharpFileService
if (Directory.Exists(dir))
{
Directory.Delete(dir, true);
DeleteDirectory(dir);
}
dir = GetUserAvatarDir(user?.Id, createNewDir: true);
var (_, bytes) = GetFileInfoFromData(file.FileData);
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes);
return true;
}

19
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs → src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs
View file

@ -1,14 +1,13 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService : IBotSharpFileService
public partial class FileBasicService : IFileBasicService
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
private readonly ILogger<BotSharpFileService> _logger;
private readonly ILogger<FileBasicService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _imageTypes = new List<string>
{
@ -25,13 +24,10 @@ public partial class BotSharpFileService : IBotSharpFileService
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const int MIN_OFFSET = 1;
private const int MAX_OFFSET = 5;
public BotSharpFileService(
public FileBasicService(
BotSharpDatabaseSettings dbSettings,
IUserIdentity user,
ILogger<BotSharpFileService> logger,
ILogger<FileBasicService> logger,
IServiceProvider services)
{
_dbSettings = dbSettings;
@ -40,11 +36,4 @@ public partial class BotSharpFileService : IBotSharpFileService
_services = services;
_baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
}
#region Private methods
private bool ExistDirectory(string? dir)
{
return !string.IsNullOrEmpty(dir) && Directory.Exists(dir);
}
#endregion
}

63
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs
View file

@ -1,63 +0,0 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
return dir;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
public byte[] GetFileBytes(string fileStorageUrl)
{
using var stream = File.OpenRead(fileStorageUrl);
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
return bytes;
}
public bool SavefileToPath(string filePath, Stream stream)
{
using (var fileStream = new FileStream(filePath, FileMode.Create))
{
stream.CopyTo(fileStream);
}
return true;
}
}

143
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs
View file

@ -1,143 +0,0 @@
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
{
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
{
var content = string.Empty;
if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty())
{
return content;
}
var guid = Guid.NewGuid().ToString();
var sessionDir = GetSessionDirectory(guid);
if (!ExistDirectory(sessionDir))
{
Directory.CreateDirectory(sessionDir);
}
try
{
var pdfFiles = await DownloadFiles(sessionDir, files);
var images = await ConvertPdfToImages(pdfFiles);
if (images.IsNullOrEmpty()) return content;
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
model: model, modelId: modelId ?? "gpt-4", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, prompt)
{
Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
}
});
content = message.Content;
return content;
}
catch (Exception ex)
{
_logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}");
return content;
}
finally
{
Directory.Delete(sessionDir, true);
}
}
#region Private methods
private string GetSessionDirectory(string id)
{
var dir = Path.Combine(_baseDir, SESSION_FOLDER, id);
return dir;
}
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
}
var locs = new List<string>();
foreach (var file in files)
{
try
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = GetFileInfoFromData(file.FileData);
}
if (!bytes.IsNullOrEmpty())
{
var guid = Guid.NewGuid().ToString();
var fileDir = Path.Combine(dir, guid);
if (!ExistDirectory(fileDir))
{
Directory.CreateDirectory(fileDir);
}
var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}");
using (var fs = new FileStream(pdfDir, FileMode.Create))
{
fs.Write(bytes, 0, bytes.Length);
fs.Close();
locs.Add(pdfDir);
Thread.Sleep(100);
}
}
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}");
continue;
}
}
return locs;
}
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files)
{
var images = new List<string>();
var converter = GetPdf2ImageConverter();
if (converter == null || files.IsNullOrEmpty())
{
return images;
}
foreach (var file in files)
{
try
{
var segs = file.Split(Path.DirectorySeparatorChar);
var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1));
var folder = Path.Combine(dir, "screenshots");
var urls = await converter.ConvertPdfToImages(file, folder);
images.AddRange(urls);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}");
continue;
}
}
return images;
}
#endregion
}

26
src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs → src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -2,8 +2,24 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class BotSharpFileService
public partial class FileInstructService
{
public async Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images)
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images?.ToList() ?? new List<BotSharpFile>()
}
});
return message;
}
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)
{
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3");
@ -31,7 +47,7 @@ public partial class BotSharpFileService
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty);
stream.Close();
return message;
}
@ -53,7 +69,7 @@ public partial class BotSharpFileService
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty);
stream.Close();
return message;
}
@ -82,7 +98,7 @@ public partial class BotSharpFileService
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty);
imageStream.Close();
maskStream.Close();
return message;
@ -100,7 +116,7 @@ public partial class BotSharpFileService
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = GetFileInfoFromData(file.FileData);
(_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
}
return bytes;

41
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs → src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -1,6 +1,9 @@
namespace BotSharp.Plugin.TencentCos.Services;
using BotSharp.Abstraction.Files.Converters;
using System.IO;
public partial class TencentCosService
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
{
@ -12,7 +15,9 @@ public partial class TencentCosService
}
var guid = Guid.NewGuid().ToString();
var sessionDir = GetSessionDirectory(guid);
var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid);
DeleteIfExistDirectory(sessionDir);
try
{
@ -32,9 +37,7 @@ public partial class TencentCosService
Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
}
});
content = message.Content;
return content;
return message.Content;
}
catch (Exception ex)
{
@ -43,17 +46,11 @@ public partial class TencentCosService
}
finally
{
Directory.Delete(sessionDir, true);
_fileBasic.DeleteDirectory(sessionDir);
}
}
#region Private methods
private string GetSessionDirectory(string id)
{
var dir = $"{SESSION_FOLDER}/{id}";
return dir;
}
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
@ -75,18 +72,17 @@ public partial class TencentCosService
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = GetFileInfoFromData(file.FileData);
(_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
}
if (!bytes.IsNullOrEmpty())
{
var guid = Guid.NewGuid().ToString();
var fileDir = $"{dir}/{guid}";
var fileDir = _fileBasic.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir);
var pdfDir = $"{fileDir}/{guid}.{extension}";
_cosClient.BucketClient.UploadBytes(pdfDir, bytes);
var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileBasic.SaveFileBytesToPath(pdfDir, bytes);
locs.Add(pdfDir);
}
}
@ -102,7 +98,7 @@ public partial class TencentCosService
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files)
{
var images = new List<string>();
var converter = GetPdf2ImageConverter();
var converter = _services.GetServices<IPdf2ImageConverter>().FirstOrDefault();
if (converter == null || files.IsNullOrEmpty())
{
return images;
@ -112,9 +108,8 @@ public partial class TencentCosService
{
try
{
var segs = file.Split(Path.DirectorySeparatorChar);
var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1));
var folder = Path.Combine(dir, "screenshots");
var dir = _fileBasic.GetParentDir(file);
var folder = _fileBasic.BuildDirectory(dir, "screenshots");
var urls = await converter.ConvertPdfToImages(file, folder);
images.AddRange(urls);
}

114
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs Normal file
View file

@ -0,0 +1,114 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Templating;
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId,
string? agentId = null, string? template = null, string? description = null,
bool includeBotFile = false, bool fromBreakpoint = false,
int? offset = null, IEnumerable<string>? contentTypes = null)
{
if (string.IsNullOrEmpty(conversationId))
{
return Enumerable.Empty<MessageFileModel>();
}
var convService = _services.GetRequiredService<IConversationService>();
var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint);
var messageIds = GetMessageIds(dialogs, offset);
var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes);
if (includeBotFile)
{
var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes);
files = files.Concat(botFiles);
}
if (files.IsNullOrEmpty())
{
return Enumerable.Empty<MessageFileModel>();
}
return await SelectFiles(agentId, template, description, files, dialogs);
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(string? agentId, string? template, string? description,
IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}";
}).ToList();
agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant;
template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt";
var foundAgent = db.GetAgent(agentId);
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template);
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
var agent = new Agent
{
Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant,
Name = foundAgent?.Name ?? "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var message = dialogs.Last();
if (!string.IsNullOrWhiteSpace(description))
{
message = RoleDialogModel.From(message, AgentRole.User, description);
}
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
var content = response?.Content ?? string.Empty;
var selecteds = JsonSerializer.Deserialize<FileSelectContext>(content);
var fids = selecteds?.Selecteds ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when selecting files. {ex.Message}\r\n{ex.InnerException}");
return new List<MessageFileModel>();
}
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> conversations, int? offset = null)
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset.HasValue && offset < 1)
{
offset = 1;
}
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
}
return messageIds;
}
}

32
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs Normal file
View file

@ -0,0 +1,32 @@
namespace BotSharp.Core.Files.Services;
public partial class FileInstructService : IFileInstructService
{
private readonly IFileBasicService _fileBasic;
private readonly IServiceProvider _services;
private readonly ILogger<FileInstructService> _logger;
private const string SESSION_FOLDER = "sessions";
public FileInstructService(
IFileBasicService fileBasic,
ILogger<FileInstructService> logger,
IServiceProvider services)
{
_fileBasic = fileBasic;
_logger = logger;
_services = services;
}
private void DeleteIfExistDirectory(string? dir)
{
if (_fileBasic.ExistDirectory(dir))
{
_fileBasic.DeleteDirectory(dir);
}
else
{
_fileBasic.CreateDirectory(dir);
}
}
}

4
src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs
View file

@ -26,6 +26,10 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
new ParameterPropertyDef("user_goal_agent",
"agent who can acheive user initial task, must align with user_goal_description.",
required: true),
new ParameterPropertyDef("conversation_end",
"user is ending the conversation.",
type: "boolean",
required: true),
new ParameterPropertyDef("is_new_task",
"whether the user is requesting a new task that is different from the previous topic.",
type: "boolean")

3
src/Infrastructure/BotSharp.Core/Using.cs
View file

@ -29,6 +29,7 @@ global using BotSharp.Abstraction.Translation;
global using BotSharp.Abstraction.Files;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Files.Utilities;
global using BotSharp.Abstraction.Translation.Attributes;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Core.Repository;
@ -37,4 +38,4 @@ global using BotSharp.Core.Agents.Services;
global using BotSharp.Core.Conversations.Services;
global using BotSharp.Core.Infrastructures;
global using BotSharp.Core.Users.Services;
global using Aspects.Cache;
global using Aspects.Cache;

8
src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json
View file

@ -7,5 +7,11 @@
"updatedDateTime": "2024-01-15T14:39:32Z",
"iconUrl": "/images/logo.png",
"disabled": false,
"isPublic": true
"isPublic": true,
"llmConfig": {
"is_inherit": false,
"provider": "openai",
"model": "gpt-4o-mini",
"max_recursion_depth": 3
}
}

2
src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid → src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid
View file

@ -2,7 +2,7 @@ Please take a look at the files in the [FILES] section from the conversation and
** Ensure the output is only in JSON format without any additional text.
** If no files are selected, you must output an empty list [].
** You may need to look at the file_name as a reference to find the correct file id.
** You may need to look at the file_name as a reference to find the correct file id or ids.
Here is the JSON format to use:
{

19
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Files.Constants;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
@ -80,7 +81,7 @@ public class ConversationController : ControllerBase
var userService = _services.GetRequiredService<IUserService>();
var agentService = _services.GetRequiredService<IAgentService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var messageIds = history.Select(x => x.MessageId).Distinct().ToList();
var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds);
@ -348,7 +349,7 @@ public class ConversationController : ControllerBase
{
if (files != null && files.Length > 0)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var dir = fileService.GetDirectory(conversationId);
foreach (var file in files)
{
@ -356,7 +357,7 @@ public class ConversationController : ControllerBase
var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"');
var filePath = Path.Combine(dir, fileName);
fileService.SavefileToPath(filePath, file.OpenReadStream());
fileService.SaveFileStreamToPath(filePath, file.OpenReadStream());
}
return Ok(new { message = "File uploaded successfully." });
@ -371,7 +372,7 @@ public class ConversationController : ControllerBase
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(conversationId, input.States);
var conv = await convService.GetConversationRecordOrCreateNew(agentId);
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var messageId = Guid.NewGuid().ToString();
var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files);
return isSaved ? messageId : string.Empty;
@ -380,15 +381,15 @@ public class ConversationController : ControllerBase
[HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")]
public IEnumerable<MessageFileViewModel> GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var files = fileService.GetMessageFiles(conversationId, new List<string> { messageId }, source, imageOnly: false);
var fileService = _services.GetRequiredService<IFileBasicService>();
var files = fileService.GetMessageFiles(conversationId, new List<string> { messageId }, source);
return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List<MessageFileViewModel>();
}
[HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")]
public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName);
if (string.IsNullOrEmpty(file))
{
@ -413,7 +414,9 @@ public class ConversationController : ControllerBase
using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read);
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
return File(bytes, "application/octet-stream", Path.GetFileName(file));
var fileExtension = Path.GetExtension(file).ToLower();
var enableRangeProcessing = FileConstants.AudioExtensions.Contains(fileExtension);
return File(bytes, "application/octet-stream", Path.GetFileName(file), enableRangeProcessing: enableRangeProcessing);
}
private async Task OnChunkReceived(HttpResponse response, ChatResponseModel message)

36
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Core.Infrastructures;
using BotSharp.OpenAPI.ViewModels.Instructs;
using NetTopologySuite.IO;
namespace BotSharp.OpenAPI.Controllers;
@ -87,18 +86,8 @@ public class InstructModeController : ControllerBase
try
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",
model: input.Model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, input.Text)
{
Files = input.Files
}
});
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files);
return message.Content;
}
catch (Exception ex)
@ -114,14 +103,14 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text);
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -140,7 +129,6 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-variation")]
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
@ -152,7 +140,9 @@ public class InstructModeController : ControllerBase
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
}
var message = await fileService.VaryImage(input.Provider, input.Model, image);
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var message = await fileInstruct.VaryImage(input.Provider, input.Model, image);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -169,7 +159,7 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-edit")]
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
@ -181,7 +171,7 @@ public class InstructModeController : ControllerBase
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
}
var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image);
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -198,7 +188,7 @@ public class InstructModeController : ControllerBase
[HttpPost("/instruct/image-mask-edit")]
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] IncomingMessageModel input)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
@ -211,7 +201,7 @@ public class InstructModeController : ControllerBase
{
return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" };
}
var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image, mask);
var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask);
imageViewModel.Content = message.Content;
imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
return imageViewModel;
@ -236,8 +226,8 @@ public class InstructModeController : ControllerBase
try
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files);
viewModel.Content = content;
return viewModel;
}

6
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -137,14 +137,14 @@ public class UserController : ControllerBase
[HttpPost("/user/avatar")]
public bool UploadUserAvatar([FromBody] BotSharpFile file)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
return fileService.SaveUserAvatar(file);
}
[HttpGet("/user/avatar")]
public IActionResult GetUserAvatar()
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var file = fileService.GetUserAvatar();
if (string.IsNullOrEmpty(file))
{
@ -158,7 +158,7 @@ public class UserController : ControllerBase
#region Private methods
private FileContentResult BuildFileResult(string file)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var bytes = fileService.GetFileBytes(file);
return File(bytes, "application/octet-stream", Path.GetFileName(file));
}

6
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Files.Utilities;
using OpenAI.Chat;
namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat;
@ -196,7 +197,6 @@ public class ChatCompletionProvider : IChatCompletion
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
@ -270,13 +270,13 @@ public class ChatCompletionProvider : IChatCompletion
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = fileService.GetFileContentType(file.FileStorageUrl);
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);

3
src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj
View file

@ -28,9 +28,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_sender.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_attachment_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_reader.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

52
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -74,57 +74,11 @@ public class HandleEmailSenderFn : IFunctionCallback
private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
{
var convService = _services.GetRequiredService<IConversationService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var conversationId = convService.ConversationId;
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot);
return await SelectFiles(userFiles.Concat(botFiles), dialogs);
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}";
}).ToList();
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt");
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "file_list", promptFiles }
});
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var latest = dialogs.LastOrDefault();
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { latest });
var content = response?.Content ?? string.Empty;
var selecteds = JsonSerializer.Deserialize<LlmContextOut>(content);
var fids = selecteds?.Selecteds ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}");
return new List<MessageFileModel>();
}
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true);
return selecteds;
}
private void BuildEmailAttachments(BodyBuilder builder, IEnumerable<MessageFileModel> files)

3
src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj
View file

@ -47,9 +47,6 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\edit_image.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>

65
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -28,7 +28,7 @@ public class EditImageFn : IFunctionCallback
Init(message);
SetImageOptions();
var image = await SelectConversationImage(descrpition);
var image = await SelectImage(descrpition);
var response = await GetImageEditGeneration(message, descrpition, image);
message.Content = response;
return true;
@ -48,64 +48,11 @@ public class EditImageFn : IFunctionCallback
state.SetState("image_count", "1");
}
private async Task<MessageFileModel?> SelectConversationImage(string? description)
private async Task<MessageFileModel?> SelectImage(string? description)
{
var convService = _services.GetRequiredService<IConversationService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var dialogs = convService.GetDialogHistory();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
return await SelectImage(userImages, dialogs.LastOrDefault(), description);
}
private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, RoleDialogModel message, string? description)
{
if (images.IsNullOrEmpty()) return null;
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var render = _services.GetRequiredService<ITemplateRender>();
var db = _services.GetRequiredService<IBotSharpRepository>();
try
{
var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) =>
{
return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}";
}).ToList();
if (promptImages.IsNullOrEmpty()) return null;
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt");
prompt = render.Render(prompt, new Dictionary<string, object>
{
{ "image_list", promptImages }
});
var agent = new Agent
{
Id = BuiltInAgentId.UtilityAssistant,
Name = "Utility Assistant",
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { dialog });
var content = response?.Content ?? string.Empty;
var selected = JsonSerializer.Deserialize<LlmContextOut>(content);
var fid = selected?.Selected ?? -1;
return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}");
return null;
}
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, description: description, contentTypes: new List<string> { MediaTypeNames.Image.Png });
return selecteds?.FirstOrDefault();
}
private async Task<string> GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image)
@ -154,7 +101,7 @@ public class EditImageFn : IFunctionCallback
}
};
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
}
}

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback
FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}"
}).ToList();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
}
}

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -51,7 +51,7 @@ public class ReadImageFn : IFunctionCallback
return new List<RoleDialogModel>();
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes);
foreach (var dialog in dialogs)

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -50,7 +50,7 @@ public class ReadPdfFn : IFunctionCallback
return new List<RoleDialogModel>();
}
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true);
foreach (var dialog in dialogs)

41
src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid
View file

@ -1,41 +0,0 @@
Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user.
** Ensure the output is only in JSON format without any additional text.
** You may need to look at the image_name as a reference to find the correct image id.
Here is the JSON format to use:
{
"selected_id": the id selected from the [IMAGES] section
}
Suppose there are four images:
id: 1, image_name: example_image_a.png
id: 2, image_name: example_image_b.png
id: 3, image_name: example_image_c.png
id: 4, image_name: example_image_d.png
=====
Example 1:
USER: I want to add a dog in the first file.
OUTPUT: { "selected_id": 1 }
Example 2:
USER: Add a coffee cup in the second image I uploaded.
OUTPUT: { "selected_id": 2 }
Example 3:
USER: Please remove the left tree in the third and the first images.
OUTPUT: { "selected_id": 3 }
Example 4:
USER: Circle the head of the dog in example_image_b.png.
OUTPUT: { "selected_id": 4 }
=====
[IMAGES]
{% for image in image_list -%}
{{ image }}{{ "\r\n" }}
{%- endfor %}

6
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Files.Utilities;
using OpenAI.Chat;
namespace BotSharp.Plugin.OpenAI.Providers.Chat;
@ -197,7 +198,6 @@ public class ChatCompletionProvider : IChatCompletion
protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
@ -271,13 +271,13 @@ public class ChatCompletionProvider : IChatCompletion
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = fileService.GetFileContentType(file.FileStorageUrl);
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);

77
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs
View file

@ -1,4 +1,4 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
namespace BotSharp.Plugin.TencentCos.Services;
@ -9,41 +9,11 @@ public partial class TencentCosService
return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/";
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
public string GetFileContentType(string filePath)
{
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
contentType = string.Empty;
}
return contentType;
}
public byte[] GetFileBytes(string fileStorageUrl)
{
try
{
var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl);
return fileData;
}
catch (Exception ex)
@ -53,7 +23,7 @@ public partial class TencentCosService
return Array.Empty<byte>();
}
public bool SavefileToPath(string filePath, Stream stream)
public bool SaveFileStreamToPath(string filePath, Stream stream)
{
if (string.IsNullOrEmpty(filePath)) return false;
@ -63,8 +33,49 @@ public partial class TencentCosService
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving file to path: {ex.Message}\r\n{ex.InnerException}");
_logger.LogWarning($"Error when saving file stream to path: {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public bool SaveFileBytesToPath(string filePath, byte[] bytes)
{
if (string.IsNullOrEmpty(filePath)) return false;
try
{
return _cosClient.BucketClient.UploadBytes(filePath, bytes);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving file bytes to path: {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public string GetParentDir(string dir, int level = 1)
{
var segs = dir.Split("/");
return string.Join("/", segs.SkipLast(level));
}
public string BuildDirectory(params string[] segments)
{
return string.Join("/", segments);
}
public void CreateDirectory(string dir)
{
}
public bool ExistDirectory(string? dir)
{
return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir);
}
public void DeleteDirectory(string dir)
{
_cosClient.BucketClient.DeleteDir(dir);
}
}

34
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Files.Converters;
using BotSharp.Abstraction.Files.Enums;
using BotSharp.Abstraction.Files.Utilities;
using System.Net.Mime;
namespace BotSharp.Plugin.TencentCos.Services;
@ -7,16 +8,16 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty())
{
return files;
}
var messageIds = GetMessageIds(conversations, offset);
var messageIds = GetMessageIds(dialogs, offset);
var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}";
foreach (var messageId in messageIds)
@ -28,8 +29,11 @@ public partial class TencentCosService
var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = GetFileContentType(file);
if (contentTypes?.Contains(contentType) != true) continue;
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
@ -42,7 +46,7 @@ public partial class TencentCosService
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds,
string source, bool imageOnly = false)
string source, IEnumerable<string>? contentTypes = null)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files;
@ -59,8 +63,8 @@ public partial class TencentCosService
{
foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir))
{
var contentType = GetFileContentType(file);
if (imageOnly && !_imageTypes.Contains(contentType))
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
@ -135,7 +139,7 @@ public partial class TencentCosService
try
{
var (_, bytes) = GetFileInfoFromData(file.FileData);
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var subDir = $"{dir}/{source}/{i + 1}";
@ -225,13 +229,9 @@ public partial class TencentCosService
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset <= 0)
if (offset <= 1)
{
offset = MIN_OFFSET;
}
else if (offset > MAX_OFFSET)
{
offset = MAX_OFFSET;
offset = 1;
}
var messageIds = new List<string>();
@ -264,7 +264,7 @@ public partial class TencentCosService
{
foreach (var screenShot in fileList)
{
contentType = GetFileContentType(screenShot);
contentType = FileUtility.GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
@ -286,7 +286,7 @@ public partial class TencentCosService
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = GetFileContentType(image);
contentType = FileUtility.GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()

107
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs
View file

@ -1,107 +0,0 @@
namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)
{
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3");
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
}, new RoleDialogModel(AgentRole.User, text));
return message;
}
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
throw new ArgumentException($"Cannot find image url or data!");
}
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
throw new ArgumentException($"Cannot find image url or data!");
}
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask)
{
if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) ||
(string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData)))
{
throw new ArgumentException($"Cannot find image/mask url or data");
}
var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
var imageBytes = await DownloadFile(image);
var maskBytes = await DownloadFile(mask);
using var imageStream = new MemoryStream();
imageStream.Write(imageBytes, 0, imageBytes.Length);
imageStream.Position = 0;
using var maskStream = new MemoryStream();
maskStream.Write(maskBytes, 0, maskBytes.Length);
maskStream.Position = 0;
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty);
imageStream.Close();
maskStream.Close();
return message;
}
#region Private methods
private async Task<byte[]> DownloadFile(BotSharpFile file)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = GetFileInfoFromData(file.FileData);
}
return bytes;
}
#endregion
}

6
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Files.Utilities;
namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
@ -26,10 +28,8 @@ public partial class TencentCosService
if (string.IsNullOrEmpty(dir)) return false;
var (_, bytes) = GetFileInfoFromData(file.FileData);
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var extension = Path.GetExtension(file.FileName);
var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}";
return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes);

14
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -5,8 +5,9 @@ using System.Net.Mime;
namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService : IBotSharpFileService
public partial class TencentCosService : IFileBasicService
{
private readonly TencentCosClient _cosClient;
private readonly TencentCosSettings _settings;
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
@ -27,10 +28,6 @@ public partial class TencentCosService : IBotSharpFileService
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const int MIN_OFFSET = 1;
private const int MAX_OFFSET = 5;
private readonly TencentCosClient _cosClient;
public TencentCosService(
TencentCosSettings settings,
@ -46,11 +43,4 @@ public partial class TencentCosService : IBotSharpFileService
_fullBuketName = $"{_settings.BucketName}-{_settings.AppId}";
_cosClient = cosClient;
}
#region Private methods
private bool ExistDirectory(string? dir)
{
return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir);
}
#endregion
}

2
src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs
View file

@ -31,7 +31,7 @@ public class TencentCosPlugin : IBotSharpPlugin
services.AddScoped<TencentCosClient>();
services.AddScoped<IBotSharpFileService, TencentCosService>();
services.AddScoped<IFileBasicService, TencentCosService>();
}
}
}