From 115477f3eaecdb859586029a3ba158d4ceb42f81 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 13 Feb 2025 12:06:15 -0600 Subject: [PATCH] add agent id as ref --- .../Files/IFileInstructService.cs | 12 +++---- .../Instruct/FileInstructService.Image.cs | 20 ++++++------ .../Instruct/FileInstructService.Pdf.cs | 5 ++- .../BotSharpLoggerExtensions.cs | 2 +- .../Hooks/GlobalStatsConversationHook.cs | 12 +++---- .../Controllers/InstructModeController.cs | 32 +++++++++---------- .../Instructs/InstructBaseRequest.cs | 3 ++ 7 files changed, 42 insertions(+), 44 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index f8f4c605..9ec3c8ed 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); - Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, InstructFileModel image); - Task EditImage(string? provider, string? model, string text, InstructFileModel image); - Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask); + Task ReadImages(string? provider, string? model, string text, IEnumerable images, string? agentId = null); + Task GenerateImage(string? provider, string? model, string text, string? agentId = null); + Task VaryImage(string? provider, string? model, InstructFileModel image, string? agentId = null); + Task EditImage(string? provider, string? model, string text, InstructFileModel image, string? agentId = null); + Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask, string? agentId = null); #endregion #region Pdf @@ -17,7 +17,7 @@ public interface IFileInstructService /// /// Pdf files /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files, string? agentId = null); #endregion #region Audio diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index cd65c16a..4de657c3 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,12 +4,12 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images, string? agentId = null) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { - Id = Guid.Empty.ToString(), + Id = agentId ?? Guid.Empty.ToString(), }, new List { new RoleDialogModel(AgentRole.User, text) @@ -20,17 +20,17 @@ public partial class FileInstructService return message.Content; } - public async Task GenerateImage(string? provider, string? model, string text) + public async Task GenerateImage(string? provider, string? model, string text, string? agentId = null) { var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); var message = await completion.GetImageGeneration(new Agent() { - Id = Guid.Empty.ToString(), + Id = agentId ?? Guid.Empty.ToString(), }, new RoleDialogModel(AgentRole.User, text)); return message; } - public async Task VaryImage(string? provider, string? model, InstructFileModel image) + public async Task VaryImage(string? provider, string? model, InstructFileModel image, string? agentId = null) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -46,14 +46,14 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { - Id = Guid.Empty.ToString() + Id = agentId ?? Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, InstructFileModel image) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, string? agentId = null) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -69,14 +69,14 @@ public partial class FileInstructService var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = Guid.Empty.ToString() + Id = agentId ?? Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask, string? agentId = null) { if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) @@ -100,7 +100,7 @@ public partial class FileInstructService var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { - Id = Guid.Empty.ToString() + Id = agentId ?? Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 29dfdaf9..e1362455 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files, string? agentId = null) { var content = string.Empty; @@ -14,7 +14,6 @@ public partial class FileInstructService } var guid = Guid.NewGuid().ToString(); - var sessionDir = _fileStorage.BuildDirectory(SESSION_FOLDER, guid); DeleteIfExistDirectory(sessionDir, true); @@ -28,7 +27,7 @@ public partial class FileInstructService model: model, modelId: modelId ?? "gpt-4", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { - Id = Guid.Empty.ToString(), + Id = agentId ?? Guid.Empty.ToString(), }, new List { new RoleDialogModel(AgentRole.User, prompt) diff --git a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs index 3ea2cd82..6ccd70ac 100644 --- a/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs +++ b/src/Infrastructure/BotSharp.Logger/BotSharpLoggerExtensions.cs @@ -13,8 +13,8 @@ public static class BotSharpLoggerExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); - services.AddScoped(); return services; } } diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs index 32c4b7d7..2cc2c9fb 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs @@ -4,7 +4,7 @@ using BotSharp.Abstraction.Statistics.Services; namespace BotSharp.Logger.Hooks; -public class GlobalStatsConversationHook : ConversationHookBase +public class GlobalStatsConversationHook : IContentGeneratingHook { private readonly IServiceProvider _services; @@ -14,14 +14,10 @@ public class GlobalStatsConversationHook : ConversationHookBase _services = services; } - public override async Task OnMessageReceived(RoleDialogModel message) - { - UpdateAgentCall(message); - } - - public override async Task OnPostbackMessageReceived(RoleDialogModel message, PostbackMessageModel replyMsg) + public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) { UpdateAgentCall(message); + await Task.CompletedTask; } private void UpdateAgentCall(RoleDialogModel message) @@ -33,7 +29,7 @@ public class GlobalStatsConversationHook : ConversationHookBase { Metric = StatsMetric.AgentCall, Dimension = "agent", - DimRefVal = message.CurrentAgentId, + DimRefVal = message.CurrentAgentId ?? string.Empty, RecordTime = DateTime.UtcNow, IntervalType = StatsInterval.Day, Data = [ diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 61862781..433d691a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -88,7 +88,7 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); + var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files, input.AgentId); return content; } catch (Exception ex) @@ -101,7 +101,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/multi-modal/upload")] public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null) + [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) { var state = _services.GetRequiredService(); states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -115,7 +115,7 @@ public class InstructModeController : ControllerBase new InstructFileModel { FileData = data } }; var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadImages(provider, model, text, files); + var content = await fileInstruct.ReadImages(provider, model, text, files, agentId); viewModel.Content = content; return viewModel; } @@ -140,7 +140,7 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text); + var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text, input.AgentId); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -171,7 +171,7 @@ public class InstructModeController : ControllerBase } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File, input.AgentId); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); @@ -188,7 +188,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-variation/upload")] public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null) + [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) { var state = _services.GetRequiredService(); states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -203,7 +203,7 @@ public class InstructModeController : ControllerBase var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var message = await completion.GetImageVariation(new Agent() { - Id = Guid.Empty.ToString() + Id = agentId ?? Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName); imageViewModel.Content = message.Content; @@ -235,7 +235,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File, input.AgentId); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -251,7 +251,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-edit/upload")] public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null) + [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -267,7 +267,7 @@ public class InstructModeController : ControllerBase var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var message = await completion.GetImageEdits(new Agent() { - Id = Guid.Empty.ToString() + Id = agentId ?? Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), stream, file.FileName); imageViewModel.Content = message.Content; @@ -301,7 +301,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask, input.AgentId); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -317,7 +317,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-mask-edit/upload")] public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null) + [FromForm] string? model = null, [FromForm] List? states = null, [FromForm] string? agentId = null) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -337,7 +337,7 @@ public class InstructModeController : ControllerBase var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); var message = await completion.GetImageEdits(new Agent() { - Id = Guid.Empty.ToString() + Id = agentId ?? Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName); imageViewModel.Content = message.Content; @@ -368,7 +368,7 @@ public class InstructModeController : ControllerBase try { var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files, input.AgentId); viewModel.Content = content; return viewModel; } @@ -383,7 +383,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/pdf-completion/upload")] public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null) + [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null, [FromForm] string? agentId = null) { var state = _services.GetRequiredService(); states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -398,7 +398,7 @@ public class InstructModeController : ControllerBase }; var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files); + var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files, agentId); viewModel.Content = content; return viewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 39a6439d..88f011be 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -13,6 +13,9 @@ public class InstructBaseRequest [JsonPropertyName("model_id")] public virtual string? ModelId { get; set; } = null; + [JsonPropertyName("agent_id")] + public virtual string? AgentId { get; set; } + [JsonPropertyName("states")] public List States { get; set; } = new(); }