SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

resolve conflict

Browse source
This commit is contained in:
Jicheng Lu 2024-08-08 16:06:12 -05:00
parent f14c634b3e 6f13ec2d73
commit 5b9db8bdd1
35 changed files with 442 additions and 434 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
@ -28,9 +28,11 @@
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageReference Include="System.Memory.Data" Version="8.0.0" />
<PackageReference Include="System.Text.Json" Version="8.0.4" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />

5
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs
View file

@ -21,9 +21,6 @@ public interface IFileInstructService
#endregion
#region Select file
Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId,
string? agentId = null, string? template = null, string? description = null,
bool includeBotFile = false, bool fromBreakpoint = false,
int? offset = null, IEnumerable<string>? contentTypes = null);
Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId, SelectFileOptions options);
#endregion
}

41
src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs → src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -2,23 +2,29 @@ using System.IO;
namespace BotSharp.Abstraction.Files;
public interface IFileBasicService
public interface IFileStorageService
{
#region Common
string GetDirectory(string conversationId);
byte[] GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);
void DeleteDirectory(string dir);
string BuildDirectory(params string[] segments);
#endregion
#region Conversation
/// <summary>
/// Get the files that have been uploaded in the chat.
/// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
/// Get the message file screenshots for specific content types, e.g., pdf
/// </summary>
/// <param name="conversationId"></param>
/// <param name="source"></param>
/// <param name="dialogs"></param>
/// <param name="contentTypes"></param>
/// <param name="includeScreenShot"></param>
/// <param name="offset"></param>
/// <param name="messageIds"></param>
/// <returns></returns>
Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes,
bool includeScreenShot = false, int? offset = null);
Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds);
/// <summary>
/// Get the files that have been uploaded in the chat. No screenshot images are included.
@ -45,20 +51,9 @@ public interface IFileBasicService
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
#endregion
#region User
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
#endregion
#region Common
string GetDirectory(string conversationId);
byte[] GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);
void DeleteDirectory(string dir);
string BuildDirectory(params string[] segments);
#endregion
}

20
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -1,23 +1,7 @@
namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile
public class BotSharpFile : FileBase
{
[JsonPropertyName("file_name")]
public string FileName { get; set; } = string.Empty;
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
public string FileData { get; set; } = string.Empty;
[JsonPropertyName("file_url")]
public string FileUrl { get; set; } = string.Empty;
[JsonPropertyName("content_type")]
public string ContentType { get; set; } = string.Empty;
[JsonPropertyName("file_storage_url")]
public string FileStorageUrl { get; set; } = string.Empty;
}

46
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs Normal file
View file

@ -0,0 +1,46 @@
namespace BotSharp.Abstraction.Files.Models;
public class FileBase
{
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileStorageUrl { get; set; } = string.Empty;
/// <summary>
/// File name without extension
/// </summary>
[JsonPropertyName("file_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileName { get; set; } = string.Empty;
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
/// <summary>
/// File content type
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; } = string.Empty;
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
}

28
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -1,34 +1,10 @@
namespace BotSharp.Abstraction.Files.Models;
public class MessageFileModel
public class MessageFileModel : FileBase
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
public string FileStorageUrl { get; set; }
/// <summary>
/// File name without extension
/// </summary>
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
[JsonPropertyName("file_source")]
public string FileSource { get; set; } = FileSourceType.User;
@ -39,6 +15,6 @@ public class MessageFileModel
public override string ToString()
{
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}";
return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}";
}
}

49
src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs Normal file
View file

@ -0,0 +1,49 @@
namespace BotSharp.Abstraction.Files.Models;
public class SelectFileOptions
{
/// <summary>
/// Llm provider
/// </summary>
public string? Provider { get; set; }
/// <summary>
/// Llm model id
/// </summary>
public string? ModelId { get; set; }
/// <summary>
/// Agent id
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Template (prompt) name
/// </summary>
public string? Template { get; set; }
/// <summary>
/// Description that user provides to select files
/// </summary>
public string? Description { get; set; }
/// <summary>
/// Whether include bot generated files
/// </summary>
public bool IncludeBotFile { get; set; }
/// <summary>
/// Conversation breakpoint
/// </summary>
public bool FromBreakpoint { get; set; }
/// <summary>
/// Message offset from last
/// </summary>
public int? Offset { get; set; }
/// <summary>
/// File content types. If null, all types of files will be retrived
/// </summary>
public IEnumerable<string>? ContentTypes { get; set; }
}

2
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

5
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs
View file

@ -5,10 +5,9 @@ public partial class ConversationService : IConversationService
public async Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true);
fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId);
fileStorage.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId);
var hooks = _services.GetServices<IConversationHook>().ToList();
foreach (var hook in hooks)

4
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
View file

@ -37,9 +37,9 @@ public partial class ConversationService : IConversationService
public async Task<bool> DeleteConversations(IEnumerable<string> ids)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var isDeleted = db.DeleteConversations(ids);
fileService.DeleteConversationFiles(ids);
fileStorage.DeleteConversationFiles(ids);
return await Task.FromResult(isDeleted);
}

5
src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs
View file

@ -10,17 +10,18 @@ public class FilePlugin : IBotSharpPlugin
public string Name => "File";
public string Description => "Provides file analysis.";
public string Description => "Provides file storage and analysis.";
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
var myFileStorageSettings = new FileStorageSettings();
config.Bind("FileStorage", myFileStorageSettings);
services.AddSingleton(myFileStorageSettings);
if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage)
{
services.AddScoped<IFileBasicService, FileBasicService>();
services.AddScoped<IFileStorageService, LocalFileStorageService>();
}
services.AddScoped<IFileInstructService, FileInstructService>();
}

19
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Files.Converters;
using System.IO;
namespace BotSharp.Core.Files.Services;
@ -16,8 +15,8 @@ public partial class FileInstructService
var guid = Guid.NewGuid().ToString();
var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid);
DeleteIfExistDirectory(sessionDir);
var sessionDir = _fileStorage.BuildDirectory(SESSION_FOLDER, guid);
DeleteIfExistDirectory(sessionDir, true);
try
{
@ -46,7 +45,7 @@ public partial class FileInstructService
}
finally
{
_fileBasic.DeleteDirectory(sessionDir);
_fileStorage.DeleteDirectory(sessionDir);
}
}
@ -78,11 +77,11 @@ public partial class FileInstructService
if (!bytes.IsNullOrEmpty())
{
var guid = Guid.NewGuid().ToString();
var fileDir = _fileBasic.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir);
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, true);
var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileBasic.SaveFileBytesToPath(pdfDir, bytes);
var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(pdfDir, bytes);
locs.Add(pdfDir);
}
}
@ -108,8 +107,8 @@ public partial class FileInstructService
{
try
{
var dir = _fileBasic.GetParentDir(file);
var folder = _fileBasic.BuildDirectory(dir, "screenshots");
var dir = _fileStorage.GetParentDir(file);
var folder = _fileStorage.BuildDirectory(dir, "screenshots");
var urls = await converter.ConvertPdfToImages(file, folder);
images.AddRange(urls);
}

74
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs
View file

@ -5,10 +5,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId,
string? agentId = null, string? template = null, string? description = null,
bool includeBotFile = false, bool fromBreakpoint = false,
int? offset = null, IEnumerable<string>? contentTypes = null)
public async Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId, SelectFileOptions options)
{
if (string.IsNullOrEmpty(conversationId))
{
@ -16,14 +13,14 @@ public partial class FileInstructService
}
var convService = _services.GetRequiredService<IConversationService>();
var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint);
var messageIds = GetMessageIds(dialogs, offset);
var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint);
var messageIds = GetMessageIds(dialogs, options.Offset);
var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes);
if (includeBotFile)
var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes);
if (options.IncludeBotFile)
{
var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes);
files = files.Concat(botFiles);
var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes);
files = MergeMessageFiles(messageIds, files, botFiles);
}
if (files.IsNullOrEmpty())
@ -31,11 +28,28 @@ public partial class FileInstructService
return Enumerable.Empty<MessageFileModel>();
}
return await SelectFiles(agentId, template, description, files, dialogs);
return await SelectFiles(files, dialogs, options);
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(string? agentId, string? template, string? description,
IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
private IEnumerable<MessageFileModel> MergeMessageFiles(IEnumerable<string> messageIds, IEnumerable<MessageFileModel> userFiles, IEnumerable<MessageFileModel> botFiles)
{
var files = new List<MessageFileModel>();
if (messageIds.IsNullOrEmpty()) return files;
foreach (var messageId in messageIds)
{
var users = userFiles.Where(x => x.MessageId == messageId).ToList();
var bots = botFiles.Where(x => x.MessageId == messageId).ToList();
if (!users.IsNullOrEmpty()) files.AddRange(users);
if (!bots.IsNullOrEmpty()) files.AddRange(bots);
}
return files;
}
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, IEnumerable<RoleDialogModel> dialogs, SelectFileOptions options)
{
if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
@ -47,11 +61,11 @@ public partial class FileInstructService
{
var promptFiles = files.Select((x, idx) =>
{
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}";
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}";
}).ToList();
agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant;
template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt";
var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant;
var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "select_file_prompt";
var foundAgent = db.GetAgent(agentId);
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template);
@ -67,19 +81,29 @@ public partial class FileInstructService
Instruction = prompt
};
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
var message = dialogs.LastOrDefault();
var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content;
if (message == null)
{
message = new RoleDialogModel(AgentRole.User, text);
}
else
{
message = RoleDialogModel.From(message, AgentRole.User, text);
}
var providerName = options.Provider ?? "openai";
var modelId = options?.ModelId ?? "gpt-4";
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
var model = llmProviderService.GetProviderModel(provider: provider, id: modelId);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var message = dialogs.Last();
if (!string.IsNullOrWhiteSpace(description))
{
message = RoleDialogModel.From(message, AgentRole.User, description);
}
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
var content = response?.Content ?? string.Empty;
var selecteds = JsonSerializer.Deserialize<FileSelectContext>(content);
var selecteds = JsonSerializer.Deserialize<FileSelectContext>(content, new JsonSerializerOptions
{
AllowTrailingCommas = true
});
var fids = selecteds?.Selecteds ?? new List<int>();
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
}

16
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs
View file

@ -2,31 +2,31 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService : IFileInstructService
{
private readonly IFileBasicService _fileBasic;
private readonly IFileStorageService _fileStorage;
private readonly IServiceProvider _services;
private readonly ILogger<FileInstructService> _logger;
private const string SESSION_FOLDER = "sessions";
public FileInstructService(
IFileBasicService fileBasic,
IFileStorageService fileStorate,
ILogger<FileInstructService> logger,
IServiceProvider services)
{
_fileBasic = fileBasic;
_fileStorage = fileStorate;
_logger = logger;
_services = services;
}
private void DeleteIfExistDirectory(string? dir)
private void DeleteIfExistDirectory(string? dir, bool createNew = false)
{
if (_fileBasic.ExistDirectory(dir))
if (_fileStorage.ExistDirectory(dir))
{
_fileBasic.DeleteDirectory(dir);
_fileStorage.DeleteDirectory(dir);
}
else
else if (createNew)
{
_fileBasic.CreateDirectory(dir);
_fileStorage.CreateDirectory(dir);
}
}
}

4
src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs
View file

@ -2,11 +2,11 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileBasicService
public partial class LocalFileStorageService
{
public string GetDirectory(string conversationId)
{
var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, "attachments");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);

169
src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -4,24 +4,22 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileBasicService
public partial class LocalFileStorageService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
{
return files;
}
var messageIds = GetMessageIds(dialogs, offset);
var source = FileSourceType.User;
var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER);
foreach (var messageId in messageIds)
{
var dir = Path.Combine(pathPrefix, messageId, source);
var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User);
if (!ExistDirectory(dir)) continue;
foreach (var subDir in Directory.GetDirectories(dir))
@ -29,22 +27,16 @@ public partial class FileBasicService
var file = Directory.GetFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var screenshots = await GetScreenshots(file, subDir, messageId, source);
if (screenshots.IsNullOrEmpty()) continue;
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
files.AddRange(foundFiles);
files.AddRange(screenshots);
}
}
return files;
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds,
string source, IEnumerable<string>? contentTypes = null)
{
@ -72,14 +64,14 @@ public partial class FileBasicService
}
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var fileExtension = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}",
FileStorageUrl = file,
FileName = fileName,
FileType = fileType,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
};
@ -87,7 +79,6 @@ public partial class FileBasicService
}
}
}
return files;
}
@ -202,8 +193,8 @@ public partial class FileBasicService
var dir = GetConversationFileDirectory(conversationId, messageId);
if (!ExistDirectory(dir)) continue;
Thread.Sleep(100);
DeleteDirectory(dir);
Thread.Sleep(100);
}
return true;
@ -269,85 +260,6 @@ public partial class FileBasicService
return messageIds;
}
private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, string fileDir, string contentType,
string messageId, string source, bool includeScreenShot)
{
var files = new List<MessageFileModel>();
try
{
if (!_imageTypes.Contains(contentType) && includeScreenShot)
{
var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty())
{
foreach (var screenShot in Directory.GetFiles(screenShotDir))
{
contentType = FileUtility.GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = screenShot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = FileUtility.GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
}
else
{
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = file,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
@ -366,5 +278,62 @@ public partial class FileBasicService
var converters = _services.GetServices<IPdf2ImageConverter>();
return converters.FirstOrDefault();
}
private async Task<IEnumerable<MessageFileModel>> GetScreenshots(string file, string parentDir, string messageId, string source)
{
var files = new List<MessageFileModel>();
try
{
var contentType = FileUtility.GetFileContentType(file);
var screenshotDir = Path.Combine(parentDir, SCREENSHOT_FILE_FOLDER);
if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty())
{
foreach (var screenshot in Directory.GetFiles(screenshotDir))
{
var fileName = Path.GetFileNameWithoutExtension(screenshot);
var fileExtension = Path.GetExtension(screenshot).Substring(1);
var screenshotContentType = FileUtility.GetFileContentType(screenshot);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileStorageUrl = screenshot,
ContentType = screenshotContentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenshotDir);
foreach (var image in images)
{
var fileName = Path.GetFileNameWithoutExtension(image);
var fileExtension = Path.GetExtension(image).Substring(1);
var screenshotContentType = FileUtility.GetFileContentType(image);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileStorageUrl = image,
ContentType = screenshotContentType,
FileSource = source
};
files.Add(model);
}
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
#endregion
}

2
src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs
View file

@ -2,7 +2,7 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileBasicService
public partial class LocalFileStorageService
{
public string GetUserAvatar()
{

13
src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs → src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -2,18 +2,13 @@ using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class FileBasicService : IFileBasicService
public partial class LocalFileStorageService : IFileStorageService
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
private readonly IUserIdentity _user;
private readonly ILogger<FileBasicService> _logger;
private readonly ILogger<LocalFileStorageService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _imageTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
};
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
@ -24,10 +19,10 @@ public partial class FileBasicService : IFileBasicService
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
public FileBasicService(
public LocalFileStorageService(
BotSharpDatabaseSettings dbSettings,
IUserIdentity user,
ILogger<FileBasicService> logger,
ILogger<LocalFileStorageService> logger,
IServiceProvider services)
{
_dbSettings = dbSettings;

22
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -81,10 +81,10 @@ public class ConversationController : ControllerBase
var userService = _services.GetRequiredService<IUserService>();
var agentService = _services.GetRequiredService<IAgentService>();
var fileService = _services.GetRequiredService<IFileBasicService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = history.Select(x => x.MessageId).Distinct().ToList();
var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds);
var fileMessages = fileStorage.GetMessagesWithFile(conversationId, messageIds);
var dialogs = new List<ChatResponseModel>();
foreach (var message in history)
@ -349,15 +349,15 @@ public class ConversationController : ControllerBase
{
if (files != null && files.Length > 0)
{
var fileService = _services.GetRequiredService<IFileBasicService>();
var dir = fileService.GetDirectory(conversationId);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var dir = fileStorage.GetDirectory(conversationId);
foreach (var file in files)
{
// Save the file, process it, etc.
var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"');
var filePath = Path.Combine(dir, fileName);
fileService.SaveFileStreamToPath(filePath, file.OpenReadStream());
fileStorage.SaveFileStreamToPath(filePath, file.OpenReadStream());
}
return Ok(new { message = "File uploaded successfully." });
@ -372,25 +372,25 @@ public class ConversationController : ControllerBase
var convService = _services.GetRequiredService<IConversationService>();
convService.SetConversationId(conversationId, input.States);
var conv = await convService.GetConversationRecordOrCreateNew(agentId);
var fileService = _services.GetRequiredService<IFileBasicService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageId = Guid.NewGuid().ToString();
var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files);
var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files);
return isSaved ? messageId : string.Empty;
}
[HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")]
public IEnumerable<MessageFileViewModel> GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source)
{
var fileService = _services.GetRequiredService<IFileBasicService>();
var files = fileService.GetMessageFiles(conversationId, new List<string> { messageId }, source);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var files = fileStorage.GetMessageFiles(conversationId, new List<string> { messageId }, source);
return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List<MessageFileViewModel>();
}
[HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")]
public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName)
{
var fileService = _services.GetRequiredService<IFileBasicService>();
var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var file = fileStorage.GetMessageFile(conversationId, messageId, source, index, fileName);
if (string.IsNullOrEmpty(file))
{
return NotFound();

12
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -137,15 +137,15 @@ public class UserController : ControllerBase
[HttpPost("/user/avatar")]
public bool UploadUserAvatar([FromBody] BotSharpFile file)
{
var fileService = _services.GetRequiredService<IFileBasicService>();
return fileService.SaveUserAvatar(file);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
return fileStorage.SaveUserAvatar(file);
}
[HttpGet("/user/avatar")]
public IActionResult GetUserAvatar()
{
var fileService = _services.GetRequiredService<IFileBasicService>();
var file = fileService.GetUserAvatar();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var file = fileStorage.GetUserAvatar();
if (string.IsNullOrEmpty(file))
{
return NotFound();
@ -158,8 +158,8 @@ public class UserController : ControllerBase
#region Private methods
private FileContentResult BuildFileResult(string file)
{
var fileService = _services.GetRequiredService<IFileBasicService>();
var bytes = fileService.GetFileBytes(file);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var bytes = fileStorage.GetFileBytes(file);
return File(bytes, "application/octet-stream", Path.GetFileName(file));
}
#endregion

6
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs
View file

@ -10,8 +10,8 @@ public class MessageFileViewModel
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("file_extension")]
public string FileExtension { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
@ -30,7 +30,7 @@ public class MessageFileViewModel
{
FileUrl = model.FileUrl,
FileName = model.FileName,
FileType = model.FileType,
FileExtension = model.FileExtension,
ContentType = model.ContentType,
FileSource = model.FileSource
};

19
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -198,6 +198,7 @@ public class ChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
@ -262,13 +263,7 @@ public class ChatCompletionProvider : IChatCompletion
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileData))
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
@ -277,8 +272,14 @@ public class ChatCompletionProvider : IChatCompletion
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
}

25
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs
View file

@ -1,17 +1,7 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Messaging.Models.RichContent.Template;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.EmailHandler.Models;
using BotSharp.Plugin.EmailHandler.Providers;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MailKit.Security;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using MimeKit;
namespace BotSharp.Plugin.EmailReader.Functions;
@ -31,13 +21,14 @@ public class HandleEmailReaderFn : IFunctionCallback
private readonly IConversationStateService _state;
private readonly IEmailReader _emailProvider;
public HandleEmailReaderFn(IServiceProvider services,
ILogger<HandleEmailReaderFn> logger,
IHttpContextAccessor context,
BotSharpOptions options,
EmailReaderSettings emailPluginSettings,
IConversationStateService state,
IEmailReader emailProvider)
public HandleEmailReaderFn(
IServiceProvider services,
ILogger<HandleEmailReaderFn> logger,
IHttpContextAccessor context,
BotSharpOptions options,
EmailReaderSettings emailPluginSettings,
IConversationStateService state,
IEmailReader emailProvider)
{
_services = services;
_logger = logger;

12
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs
View file

@ -1,7 +1,6 @@
using MailKit.Net.Smtp;
using MailKit.Security;
using MimeKit;
using System.IO;
namespace BotSharp.Plugin.EmailHandler.Functions;
@ -65,7 +64,7 @@ public class HandleEmailSenderFn : IFunctionCallback
catch (Exception ex)
{
var msg = $"Failed to send the email. {ex.Message}";
_logger.LogError($"{msg}\n(Error: {ex.Message})");
_logger.LogError($"{msg}\n(Error: {ex.Message}\r\n{ex.InnerException})");
message.Content = msg;
return false;
}
@ -77,7 +76,7 @@ public class HandleEmailSenderFn : IFunctionCallback
var conversationId = convService.ConversationId;
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true);
var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true });
return selecteds;
}
@ -89,10 +88,9 @@ public class HandleEmailSenderFn : IFunctionCallback
{
if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
using var fs = File.OpenRead(file.FileStorageUrl);
var binary = BinaryData.FromStream(fs);
builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType));
fs.Close();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl);
builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType));
Thread.Sleep(100);
}
}

20
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Files.Utilities;
using BotSharp.Abstraction.Templating;
using System.IO;
namespace BotSharp.Plugin.FileHandler.Functions;
@ -50,7 +52,11 @@ public class EditImageFn : IFunctionCallback
private async Task<MessageFileModel?> SelectImage(string? description)
{
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, description: description, contentTypes: new List<string> { MediaTypeNames.Image.Png });
var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions
{
Description = description,
ContentTypes = new List<string> { MediaTypeNames.Image.Png }
});
return selecteds?.FirstOrDefault();
}
@ -72,12 +78,16 @@ public class EditImageFn : IFunctionCallback
Name = "Utility Assistant"
};
using var stream = File.OpenRead(image.FileStorageUrl);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl);
using var stream = new MemoryStream();
stream.Write(fileBytes);
stream.Position = 0;
var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty);
stream.Close();
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted.";
return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted.";
}
catch (Exception ex)
{
@ -100,7 +110,7 @@ public class EditImageFn : IFunctionCallback
}
};
var fileService = _services.GetRequiredService<IFileBasicService>();
fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
}
}

4
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback
FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}"
}).ToList();
var fileService = _services.GetRequiredService<IFileBasicService>();
fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files);
}
}

20
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<ReadImageFn> _logger;
private readonly IEnumerable<string> _imageContentTypes = new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg,
};
public ReadImageFn(
IServiceProvider services,
ILogger<ReadImageFn> logger)
@ -29,7 +23,7 @@ public class ReadImageFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs);
var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs);
var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant);
var fileAgent = new Agent
{
@ -44,15 +38,20 @@ public class ReadImageFn : IFunctionCallback
return true;
}
private async Task<List<RoleDialogModel>> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
private List<RoleDialogModel> AssembleFiles(string conversationId, List<RoleDialogModel> dialogs)
{
if (dialogs.IsNullOrEmpty())
{
return new List<RoleDialogModel>();
}
var fileService = _services.GetRequiredService<IFileBasicService>();
var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List<string>
{
MediaTypeNames.Image.Png,
MediaTypeNames.Image.Jpeg
});
foreach (var dialog in dialogs)
{
@ -62,6 +61,7 @@ public class ReadImageFn : IFunctionCallback
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}

10
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -50,17 +50,21 @@ public class ReadPdfFn : IFunctionCallback
return new List<RoleDialogModel>();
}
var fileService = _services.GetRequiredService<IFileBasicService>();
var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds);
if (screenshots.IsNullOrEmpty()) return dialogs;
foreach (var dialog in dialogs)
{
var found = files.Where(x => x.MessageId == dialog.MessageId).ToList();
var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList();
if (found.IsNullOrEmpty()) continue;
dialog.Files = found.Select(x => new BotSharpFile
{
ContentType = x.ContentType,
FileUrl = x.FileUrl,
FileStorageUrl = x.FileStorageUrl
}).ToList();
}

19
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -199,6 +199,7 @@ public class ChatCompletionProvider : IChatCompletion
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
@ -263,13 +264,7 @@ public class ChatCompletionProvider : IChatCompletion
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileData))
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
@ -278,8 +273,14 @@ public class ChatCompletionProvider : IChatCompletion
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
using var stream = File.OpenRead(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
contentParts.Add(contentPart);
}
}

5
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs
View file

@ -1,5 +1,3 @@
using System.IO;
namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
@ -13,8 +11,7 @@ public partial class TencentCosService
{
try
{
var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl);
return fileData;
return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl);
}
catch (Exception ex)
{

187
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -7,38 +7,28 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,
bool includeScreenShot = false, int? offset = null)
public async Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds)
{
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty())
if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty())
{
return files;
}
var messageIds = GetMessageIds(dialogs, offset);
var source = FileSourceType.User;
var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}";
foreach (var messageId in messageIds)
{
var dir = $"{pathPrefix}/{messageId}/{source}";
foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir))
{
var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault();
if (file == null) continue;
var contentType = FileUtility.GetFileContentType(file);
if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType))
{
continue;
}
var screenshots = await GetScreenshots(file, subDir, messageId, source);
if (screenshots.IsNullOrEmpty()) continue;
var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot);
if (foundFiles.IsNullOrEmpty()) continue;
files.AddRange(foundFiles);
files.AddRange(screenshots);
}
}
@ -70,14 +60,14 @@ public partial class TencentCosService
}
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var fileExtension = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileUrl = $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}",
FileUrl = BuilFileUrl(file),
FileStorageUrl = file,
FileName = fileName,
FileType = fileType,
FileExtension = fileExtension,
ContentType = contentType,
FileSource = source
};
@ -89,12 +79,13 @@ public partial class TencentCosService
return files;
}
public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName)
{
var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/";
var fileList = _cosClient.BucketClient.GetDirFiles(dir);
var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName));
return found;
}
@ -140,9 +131,7 @@ public partial class TencentCosService
try
{
var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var subDir = $"{dir}/{source}/{i + 1}";
_cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes);
}
catch (Exception ex)
@ -225,11 +214,11 @@ public partial class TencentCosService
return dir;
}
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> conversations, int? offset = null)
private IEnumerable<string> GetMessageIds(IEnumerable<RoleDialogModel> dialogs, int? offset = null)
{
if (conversations.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (dialogs.IsNullOrEmpty()) return Enumerable.Empty<string>();
if (offset <= 1)
if (offset.HasValue && offset < 1)
{
offset = 1;
}
@ -237,97 +226,17 @@ public partial class TencentCosService
var messageIds = new List<string>();
if (offset.HasValue)
{
messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList();
}
else
{
messageIds = conversations.Select(x => x.MessageId).Distinct().ToList();
messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
}
return messageIds;
}
private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, string fileDir, string contentType,
string messageId, string source, bool includeScreenShot)
{
var files = new List<MessageFileModel>();
try
{
if (!_imageTypes.Contains(contentType) && includeScreenShot)
{
var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/";
var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir);
if (!fileList.IsNullOrEmpty())
{
foreach (var screenShot in fileList)
{
contentType = FileUtility.GetFileContentType(screenShot);
if (!_imageTypes.Contains(contentType)) continue;
var fileName = Path.GetFileNameWithoutExtension(screenShot);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = screenShot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenShotDir);
foreach (var image in images)
{
contentType = FileUtility.GetFileContentType(image);
var fileName = Path.GetFileNameWithoutExtension(image);
var fileType = Path.GetExtension(image).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
}
else
{
var fileName = Path.GetFileNameWithoutExtension(file);
var fileType = Path.GetExtension(file).Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileName = fileName,
FileType = fileType,
FileStorageUrl = file,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string imageLoc)
{
var converters = _services.GetServices<IPdf2ImageConverter>();
@ -346,5 +255,69 @@ public partial class TencentCosService
var converters = _services.GetServices<IPdf2ImageConverter>();
return converters.FirstOrDefault();
}
private string BuilFileUrl(string file)
{
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}";
}
private async Task<IEnumerable<MessageFileModel>> GetScreenshots(string file, string parentDir, string messageId, string source)
{
var files = new List<MessageFileModel>();
try
{
var contentType = FileUtility.GetFileContentType(file);
var screenshotDir = $"{parentDir}/{SCREENSHOT_FILE_FOLDER}/";
var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir);
if (!screenshots.IsNullOrEmpty())
{
foreach (var screenshot in screenshots)
{
var screenshotContentType = FileUtility.GetFileContentType(screenshot);
var fileName = Path.GetFileNameWithoutExtension(screenshot);
var fileExtension = Path.GetExtension(screenshot).Substring(1);
var model = new MessageFileModel
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileUrl = BuilFileUrl(screenshot),
FileStorageUrl = screenshot,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
else if (contentType == MediaTypeNames.Application.Pdf)
{
var images = await ConvertPdfToImages(file, screenshotDir);
foreach (var image in images)
{
var fileName = Path.GetFileNameWithoutExtension(image);
var fileExtension = Path.GetExtension(image).Substring(1);
var screenshotContentType = FileUtility.GetFileContentType(image);
var model = new MessageFileModel
{
MessageId = messageId,
FileName = fileName,
FileExtension = fileExtension,
FileUrl = BuilFileUrl(image),
FileStorageUrl = image,
ContentType = contentType,
FileSource = source
};
files.Add(model);
}
}
return files;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}");
return files;
}
}
#endregion
}

1
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs
View file

@ -51,7 +51,6 @@ public partial class TencentCosService
}
var dir = $"{USERS_FOLDER}/{userId}/{USER_AVATAR_FOLDER}/";
return dir;
}
#endregion

4
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -5,7 +5,7 @@ using System.Net.Mime;
namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService : IFileBasicService
public partial class TencentCosService : IFileStorageService
{
private readonly TencentCosClient _cosClient;
private readonly TencentCosSettings _settings;
@ -40,7 +40,7 @@ public partial class TencentCosService : IFileBasicService
_user = user;
_logger = logger;
_services = services;
_fullBuketName = $"{_settings.BucketName}-{_settings.AppId}";
_fullBuketName = $"{settings.BucketName}-{settings.AppId}";
_cosClient = cosClient;
}
}

1
src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs
View file

@ -19,7 +19,6 @@ namespace BotSharp.Plugin.TencentCos
settings.SecretId, settings.SecretKey, settings.KeyDurationSecond);
var cosXml = new CosXmlServer(cosXmlConfig, cosCredentialProvider);
BucketClient = new BucketClient(cosXml, $"{settings.BucketName}-{settings.AppId}", settings.AppId, settings.Region);
}
}

3
src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs
View file

@ -30,8 +30,7 @@ public class TencentCosPlugin : IBotSharpPlugin
});
services.AddScoped<TencentCosClient>();
services.AddScoped<IFileBasicService, TencentCosService>();
services.AddScoped<IFileStorageService, TencentCosService>();
}
}
}