SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine file models

Browse source
This commit is contained in:
Jicheng Lu 2024-08-27 12:12:17 -05:00
parent 512d459c6b
commit 1ace4e44ab
22 changed files with 155 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs
View file

@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files;
public interface IFileInstructService
{
#region Image
Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images);
Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images);
Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask);
Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image);
Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask);
#endregion
#region Pdf
@ -17,11 +17,11 @@ public interface IFileInstructService
/// <param name="prompt"></param>
/// <param name="files">Pdf files</param>
/// <returns></returns>
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files);
#endregion
#region Audio
Task<string> ReadAudio(string? provider, string? model, BotSharpFile audio);
Task<string> ReadAudio(string? provider, string? model, InstructFileModel audio);
#endregion
#region Select file

6
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -16,7 +16,6 @@ public interface IFileStorageService
string BuildDirectory(params string[] segments);
#endregion
#region Conversation
/// <summary>
/// Get the message file screenshots for specific content types, e.g., pdf
@ -37,7 +36,7 @@ public interface IFileStorageService
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files);
/// <summary>
/// Delete files under messages
@ -54,8 +53,9 @@ public interface IFileStorageService
#region User
string GetUserAvatar();
bool SaveUserAvatar(BotSharpFile file);
bool SaveUserAvatar(InputFileModel file);
#endregion
#region Speech
Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data);
Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName);

9
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -1,7 +1,12 @@
namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile : FileBase
public class BotSharpFile : FileInfo
{
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
}

28
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs
View file

@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models;
public class FileBase
{
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileStorageUrl { get; set; } = string.Empty;
/// <summary>
/// File name without extension
/// </summary>
@ -29,18 +15,4 @@ public class FileBase
[JsonPropertyName("file_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileData { get; set; } = string.Empty;
/// <summary>
/// File content type
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; } = string.Empty;
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
}

38
src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs Normal file
View file

@ -0,0 +1,38 @@
namespace BotSharp.Abstraction.Files.Models;
public class FileInfo
{
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
/// <summary>
/// Internal file storage url
/// </summary>
[JsonPropertyName("file_storage_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileStorageUrl { get; set; } = string.Empty;
/// <summary>
/// File content type
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; } = string.Empty;
/// <summary>
/// File name without extension
/// </summary>
[JsonPropertyName("file_name")]
public string FileName { get; set; } = string.Empty;
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
}

16
src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs Normal file
View file

@ -0,0 +1,16 @@
namespace BotSharp.Abstraction.Files.Models;
public class InputFileModel : FileBase
{
/// <summary>
/// File name with extension
/// </summary>
[JsonPropertyName("file_name")]
public new string FileName { get; set; } = string.Empty;
/// <summary>
/// File data => format: "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
public new string FileData { get; set; } = string.Empty;
}

18
src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs Normal file
View file

@ -0,0 +1,18 @@
namespace BotSharp.Abstraction.Files.Models;
public class InstructFileModel : FileBase
{
/// <summary>
/// File extension without dot
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;
/// <summary>
/// External file url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Abstraction.Files.Models;
public class MessageFileModel : FileBase
public class MessageFileModel : FileInfo
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }

10
src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Models;
public class AiModel
{
public string Id { get; set; } = string.Empty;
public string Model { get; set; } = string.Empty;
public string Name { get; set; } = string.Empty;
public int MaxLength { get; set; }
public int TokenLimit { get; set; }
}

5
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadAudio(string? provider, string? model, BotSharpFile audio)
public async Task<string> ReadAudio(string? provider, string? model, InstructFileModel audio)
{
var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var audioBytes = await DownloadFile(audio);
@ -12,7 +12,8 @@ public partial class FileInstructService
stream.Write(audioBytes, 0, audioBytes.Length);
stream.Position = 0;
var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty);
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var content = await completion.GenerateTextFromAudioAsync(stream, fileName);
stream.Close();
return content;
}

22
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images)
public async Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images)
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
@ -14,7 +14,7 @@ public partial class FileInstructService
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images?.ToList() ?? new List<BotSharpFile>()
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List<BotSharpFile>()
}
});
return message.Content;
@ -30,7 +30,7 @@ public partial class FileInstructService
return message;
}
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image)
public async Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
@ -43,16 +43,17 @@ public partial class FileInstructService
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var message = await completion.GetImageVariation(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image)
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image)
{
if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData))
{
@ -65,16 +66,17 @@ public partial class FileInstructService
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, text), stream, fileName);
stream.Close();
return message;
}
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask)
public async Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask)
{
if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) ||
(string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData)))
@ -94,10 +96,12 @@ public partial class FileInstructService
maskStream.Write(maskBytes, 0, maskBytes.Length);
maskStream.Position = 0;
var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}";
var message = await completion.GetImageEdits(new Agent()
{
Id = Guid.Empty.ToString()
}, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty);
}, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName);
imageStream.Close();
maskStream.Close();
@ -105,7 +109,7 @@ public partial class FileInstructService
}
#region Private methods
private async Task<byte[]> DownloadFile(BotSharpFile file)
private async Task<byte[]> DownloadFile(InstructFileModel file)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))

4
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
public partial class FileInstructService
{
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files)
{
var content = string.Empty;
@ -50,7 +50,7 @@ public partial class FileInstructService
}
#region Private methods
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{

2
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs
View file

@ -118,7 +118,7 @@ public partial class LocalFileStorageService
return foundMsgs;
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files)
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files)
{
if (files.IsNullOrEmpty()) return false;

2
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs
View file

@ -16,7 +16,7 @@ public partial class LocalFileStorageService
return found;
}
public bool SaveUserAvatar(BotSharpFile file)
public bool SaveUserAvatar(InputFileModel file)
{
if (file == null || string.IsNullOrEmpty(file.FileData)) return false;

7
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -135,9 +135,14 @@ public class UserController : ControllerBase
#region Avatar
[HttpPost("/user/avatar")]
public bool UploadUserAvatar([FromBody] BotSharpFile file)
public bool UploadUserAvatar([FromBody] UserAvatarModel input)
{
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var file = new InputFileModel
{
FileName = input.FileName,
FileData = input.FileData,
};
return fileStorage.SaveUserAvatar(file);
}

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs
View file

@ -3,5 +3,5 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations;
public class InputMessageFiles
{
public List<MessageState> States { get; set; } = new();
public List<BotSharpFile> Files { get; set; } = new();
public List<InputFileModel> Files { get; set; } = new();
}

12
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs
View file

@ -23,7 +23,7 @@ public class MultiModalRequest : InstructBaseRequest
public string Text { get; set; } = string.Empty;
[JsonPropertyName("files")]
public List<BotSharpFile> Files { get; set; } = new();
public List<InstructFileModel> Files { get; set; } = new();
}
public class ImageGenerationRequest : InstructBaseRequest
@ -35,7 +35,7 @@ public class ImageGenerationRequest : InstructBaseRequest
public class ImageVariationRequest : InstructBaseRequest
{
[JsonPropertyName("file")]
public BotSharpFile File { get; set; }
public InstructFileModel File { get; set; }
}
public class ImageEditRequest : InstructBaseRequest
@ -44,7 +44,7 @@ public class ImageEditRequest : InstructBaseRequest
public string Text { get; set; } = string.Empty;
[JsonPropertyName("file")]
public BotSharpFile File { get; set; }
public InstructFileModel File { get; set; }
}
public class ImageMaskEditRequest : InstructBaseRequest
@ -53,14 +53,14 @@ public class ImageMaskEditRequest : InstructBaseRequest
public string Text { get; set; } = string.Empty;
[JsonPropertyName("file")]
public BotSharpFile File { get; set; }
public InstructFileModel File { get; set; }
[JsonPropertyName("mask")]
public BotSharpFile Mask { get; set; }
public InstructFileModel Mask { get; set; }
}
public class AudioCompletionRequest : InstructBaseRequest
{
[JsonPropertyName("file")]
public BotSharpFile File { get; set; }
public InstructFileModel File { get; set; }
}

18
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs Normal file
View file

@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Users;
public class UserAvatarModel
{
/// <summary>
/// File name with extension
/// </summary>
[JsonPropertyName("file_name")]
public string FileName { get; set; } = string.Empty;
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
public string FileData { get; set; } = string.Empty;
}

4
src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs
View file

@ -99,9 +99,9 @@ public class EditImageFn : IFunctionCallback
{
if (image == null) return;
var files = new List<BotSharpFile>()
var files = new List<InputFileModel>()
{
new BotSharpFile
new InputFileModel
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}"

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs
View file

@ -77,7 +77,7 @@ public class GenerateImageFn : IFunctionCallback
{
if (images.IsNullOrEmpty()) return;
var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new BotSharpFile
var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel
{
FileName = $"{Guid.NewGuid()}.png",
FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}"

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs
View file

@ -115,7 +115,7 @@ public partial class TencentCosService
return foundMsgs;
}
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files)
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files)
{
if (files.IsNullOrEmpty()) return false;

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs
View file

@ -16,7 +16,7 @@ public partial class TencentCosService
return found;
}
public bool SaveUserAvatar(BotSharpFile file)
public bool SaveUserAvatar(InputFileModel file)
{
if (file == null || string.IsNullOrEmpty(file.FileData)) return false;