diff --git a/Directory.Packages.props b/Directory.Packages.props index aa76cf61..f49939eb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,7 +1,7 @@ 8.0.0 - 2.3.0 + 2.3.0 true @@ -13,7 +13,7 @@ - + @@ -51,7 +51,7 @@ - + @@ -108,7 +108,9 @@ + + @@ -127,7 +129,6 @@ - diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 1abf7994..c28a6cda 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -20,7 +20,7 @@ public class PageActionArgs public bool OpenNewTab { get; set; } = true; [JsonPropertyName("open_blank_page")] public bool OpenBlankPage { get; set; } = true; - + [JsonPropertyName("enable_response_callback")] public bool EnableResponseCallback { get; set; } = false; /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs index 14932118..cb3e6c4f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs @@ -7,9 +7,20 @@ public class WebPageResponseData public string ResponseData { get; set; } = null!; public bool ResponseInMemory { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public string Method { get; set; } + public List Cookies { get; set; } + public int ResponseCode { get; set; } public override string ToString() { return $"{Url} {ResponseData.Length}"; } } +public class WebPageCookieData +{ + public string Name { get; set; } = null!; + public string Value { get; set; } = null!; + public string Domain { get; set; } = null!; + public string Path { get; set; } = null!; + public float Expires { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index f876c449..aa983ad6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -6,6 +6,7 @@ public class WebBrowsingSettings public bool Headless { get; set; } // Default timeout in milliseconds public float DefaultTimeout { get; set; } = 30000; + public float DefaultNavigationTimeout { get; set; } = 30000; public bool IsEnableScreenshot { get; set; } // Default wait time in seconds after page is opened public int DefaultWaitTime { get; set; } = 5; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 15f7e89e..4eb44ae4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -9,7 +9,7 @@ public interface IConversationService string ConversationId { get; } Task NewConversation(Conversation conversation); void SetConversationId(string conversationId, List states, bool isReadOnly = false); - Task GetConversation(string id); + Task GetConversation(string id, bool isLoadStates = false); Task> GetConversations(ConversationFilter filter); Task UpdateConversationTitle(string id, string title); Task UpdateConversationTitleAlias(string id, string titleAlias); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs index 23388807..ca55e385 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs @@ -6,6 +6,7 @@ public class TokenStatsModel public string Model { get; set; } public string Prompt { get; set; } public int PromptCount { get; set; } + public int CachedPromptCount { get; set; } public int CompletionCount { get; set; } public AgentLlmConfig LlmConfig { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs index e2a56817..bd71009b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs @@ -25,6 +25,10 @@ public class FunctionDef [JsonPropertyName("parameters")] public FunctionParametersDef Parameters { get; set; } = new FunctionParametersDef(); + [JsonPropertyName("output")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Output { get; set; } + public override string ToString() { return $"{Name}: {Description}"; diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index 9db7c440..038699ac 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -16,4 +16,5 @@ public class StateConst public const string SUB_CONVERSATION_ID = "sub_conversation_id"; public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id"; + public const string WEB_DRIVER_TASK_ID = "web_driver_task_id"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs index f19cca5c..9209684c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs @@ -8,16 +8,16 @@ public class InstructHookBase : IInstructHook public virtual async Task BeforeCompletion(Agent agent, RoleDialogModel message) { - return; + await Task.CompletedTask; } public virtual async Task AfterCompletion(Agent agent, InstructResult result) { - return; + await Task.CompletedTask; } public virtual async Task OnResponseGenerated(InstructResponseModel response) { - return; + await Task.CompletedTask; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/ExecuteTemplateArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/ExecuteTemplateArgs.cs new file mode 100644 index 00000000..e3d2405e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/ExecuteTemplateArgs.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Instructs.Models; + +public class ExecuteTemplateArgs +{ + [JsonPropertyName("template_name")] + public string? TemplateName { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs index d1f51348..aaa4092f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs @@ -2,5 +2,11 @@ namespace BotSharp.Abstraction.Instructs.Settings; public class InstructionSettings { - public bool EnableLog { get; set; } + public InstructionLogSetting Logging { get; set; } = new(); } + +public class InstructionLogSetting +{ + public bool Enabled { get; set; } = true; + public List ExcludedAgentIds { get; set; } = []; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs index de843023..36f17274 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -5,6 +5,7 @@ namespace BotSharp.Abstraction.Loggers.Models; public class InstructionLogModel { [JsonPropertyName("id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Id { get; set; } = default!; [JsonPropertyName("agent_id")] diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs deleted file mode 100644 index 85fc84f2..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.IO; - -namespace BotSharp.Abstraction.MLTasks; - -public interface IAudioCompletion -{ - string Provider { get; } - - string Model { get; } - - Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); - Task GenerateAudioFromTextAsync(string text); - - void SetModelName(string model); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioSynthesis.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioSynthesis.cs new file mode 100644 index 00000000..049b6ddf --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioSynthesis.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.MLTasks; + +/// +/// Text to speech synthesis +/// +public interface IAudioSynthesis +{ + string Provider { get; } + + string Model { get; } + + void SetModelName(string model); + + Task GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioTranscription.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioTranscription.cs new file mode 100644 index 00000000..f56119e2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioTranscription.cs @@ -0,0 +1,17 @@ +using System.IO; + +namespace BotSharp.Abstraction.MLTasks; + +/// +/// Audio transcription service +/// +public interface IAudioTranscription +{ + string Provider { get; } + + string Model { get; } + + Task TranscriptTextAsync(Stream audio, string audioFileName, string? text = null); + + void SetModelName(string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 2a06ece6..e28f61ae 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,7 +6,7 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false); List GetProviderModels(string provider); List GetLlmConfigs(LlmConfigOptions? options = null); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index a40415c7..2d98b254 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -3,14 +3,14 @@ namespace BotSharp.Abstraction.MLTasks.Settings; public class LlmModelSetting { /// - /// Model Id, like "gpt-3.5" and "gpt-4". + /// Model Id, like "gpt-4", "gpt-4o", "o1". /// - public string? Id { get; set; } + public string Id { get; set; } = null!; /// /// Deployment model name /// - public string Name { get; set; } + public string Name { get; set; } = null!; /// /// Model version @@ -28,8 +28,8 @@ public class LlmModelSetting /// public string? Group { get; set; } - public string ApiKey { get; set; } - public string Endpoint { get; set; } + public string ApiKey { get; set; } = null!; + public string? Endpoint { get; set; } public LlmModelType Type { get; set; } = LlmModelType.Chat; /// @@ -62,12 +62,22 @@ public class LlmModelSetting /// public int Dimension { get; set; } + public LlmCost AdditionalCost { get; set; } = new(); + public override string ToString() { return $"[{Type}] {Name} {Endpoint}"; } } +public class LlmCost +{ + public float CachedPromptCost { get; set; } = 0f; + public float AudioPromptCost { get; set; } = 0f; + public float ReasoningCompletionCost { get; } = 0f; + public float AudioCompletionCost { get; } = 0f; +} + public enum LlmModelType { Text = 1, diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs index a65a8985..3a57791e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs @@ -4,7 +4,13 @@ public class ModelTurnDetection { public int PrefixPadding { get; set; } = 300; - public int SilenceDuration { get; set; } = 800; + public int SilenceDuration { get; set; } = 500; - public float Threshold { get; set; } = 0.8f; + public float Threshold { get; set; } = 0.5f; +} + +public class AudioTranscription +{ + public string Model { get; set; } = "gpt-4o-mini-transcribe"; + public string? Language { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index 30b6cfb8..a1827ff0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models; public class RealtimeModelSettings { + public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; + public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs index d05c49c6..e5eed361 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs @@ -27,6 +27,8 @@ public class ConversationFilter public List? Tags { get; set; } + public bool IsLoadLatestStates { get; set; } + public static ConversationFilter Empty() { return new ConversationFilter(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 332ca340..b6b58a6b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -124,7 +124,7 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); void UpdateConversationStatus(string conversationId, string status) => throw new NotImplementedException(); - Conversation GetConversation(string conversationId) + Conversation GetConversation(string conversationId, bool isLoadStates = false) => throw new NotImplementedException(); PagedItems GetConversations(ConversationFilter filter) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 55378b01..3ad194ee 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -74,7 +74,7 @@ public class RealtimeHub : IRealtimeHub if (!model.Contains("-realtime-")) { var llmProviderService = _services.GetRequiredService(); - model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name; + model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name; } _completer.SetModelName(model); @@ -92,6 +92,7 @@ public class RealtimeHub : IRealtimeHub } routing.Context.SetDialogs(dialogs); + routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); @@ -115,13 +116,14 @@ public class RealtimeHub : IRealtimeHub } else { - // Push dialogs into model context + // Append dialogs into model context + var history = "[CONVERSATION HISTORY]\r\n"; foreach (var message in dialogs) { - await _completer.InsertConversationItem(message); + history += $"{message.Role}: {message.Content}\r\n"; } - await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement."); + await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement."); } }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => @@ -188,6 +190,7 @@ public class RealtimeHub : IRealtimeHub // append input audio transcript to conversation dialogs.Add(message); storage.Append(_conn.ConversationId, message); + routing.Context.SetMessageId(_conn.ConversationId, message.MessageId); foreach (var hook in hookProvider.HooksOrderByPriority) { diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 00e1f30a..0f02e417 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -46,18 +46,22 @@ + + + + @@ -90,6 +94,7 @@ + @@ -187,6 +192,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest @@ -213,4 +224,11 @@ + + + + true + + + diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 9fc97a72..20e6a7fd 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -71,7 +71,7 @@ public partial class ConversationService : IConversationService return db.UpdateConversationMessage(conversationId, request); } - public async Task GetConversation(string id) + public async Task GetConversation(string id, bool isLoadStates = false) { var db = _services.GetRequiredService(); var conversation = db.GetConversation(id); @@ -80,6 +80,11 @@ public partial class ConversationService : IConversationService public async Task> GetConversations(ConversationFilter filter) { + if (filter == null) + { + filter = ConversationFilter.Empty(); + } + var db = _services.GetRequiredService(); var conversations = db.GetConversations(filter); return conversations; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 5d19e2d0..37c8c760 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -69,23 +69,34 @@ public class ConversationStateService : IConversationStateService return this; } + var defaultRound = -1; var preValue = string.Empty; var currentValue = value.ToString(); - var hooks = _services.GetServices(); - var curActiveRounds = activeRounds > 0 ? activeRounds : -1; - int? preActiveRounds = null; + var curActive = true; + StateKeyValue? pair = null; + StateValue? prevLeafNode = null; + var curActiveRounds = activeRounds > 0 ? activeRounds : defaultRound; - if (ContainsState(name) && _curStates.TryGetValue(name, out var pair)) + if (ContainsState(name) && _curStates.TryGetValue(name, out pair)) { - var leafNode = pair?.Values?.LastOrDefault(); - preActiveRounds = leafNode?.ActiveRounds; - preValue = leafNode?.Data ?? string.Empty; + prevLeafNode = pair?.Values?.LastOrDefault(); + preValue = prevLeafNode?.Data ?? string.Empty; } _logger.LogInformation($"[STATE] {name} = {value}"); var routingCtx = _services.GetRequiredService(); - if (!ContainsState(name) || preValue != currentValue || preActiveRounds != curActiveRounds) + var isNoChange = ContainsState(name) + && preValue == currentValue + && prevLeafNode?.ActiveRounds == curActiveRounds + && curActiveRounds == defaultRound + && prevLeafNode?.Source == source + && prevLeafNode?.DataType == valueType + && prevLeafNode?.Active == curActive + && pair?.Readonly == readOnly; + + var hooks = _services.GetServices(); + if (!ContainsState(name) || preValue != currentValue || prevLeafNode?.ActiveRounds != curActiveRounds) { foreach (var hook in hooks) { @@ -95,7 +106,7 @@ public class ConversationStateService : IConversationStateService MessageId = routingCtx.MessageId, Name = name, BeforeValue = preValue, - BeforeActiveRounds = preActiveRounds, + BeforeActiveRounds = prevLeafNode?.ActiveRounds, AfterValue = currentValue, AfterActiveRounds = curActiveRounds, DataType = valueType, @@ -116,7 +127,7 @@ public class ConversationStateService : IConversationStateService { Data = currentValue, MessageId = routingCtx.MessageId, - Active = true, + Active = curActive, ActiveRounds = curActiveRounds, DataType = valueType, Source = source, @@ -128,6 +139,10 @@ public class ConversationStateService : IConversationStateService newPair.Values = new List { newValue }; _curStates[name] = newPair; } + else if (isNoChange) + { + // do nothing + } else { _curStates[name].Values.Add(newValue); @@ -415,14 +430,14 @@ public class ConversationStateService : IConversationStateService { var values = _curStates.Values.ToList(); var copy = JsonSerializer.Deserialize>(JsonSerializer.Serialize(values)); - return new ConversationState(copy ?? new()); + return new ConversationState(copy ?? []); } public void SetCurrentState(ConversationState state) { var values = _curStates.Values.ToList(); var copy = JsonSerializer.Deserialize>(JsonSerializer.Serialize(values)); - _curStates = new ConversationState(copy ?? new()); + _curStates = new ConversationState(copy ?? []); } public void ResetCurrentState() diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 0698c0cb..5a0614ed 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -41,9 +41,11 @@ public class TokenStatistics : ITokenStatistics var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(stats.Provider, _model); - var deltaPromptCost = stats.PromptCount / 1000f * settings.PromptCost; + var deltaPromptCost = (stats.PromptCount - stats.CachedPromptCount) / 1000f * settings.PromptCost; + var deltaCachedPromptCost = stats.CachedPromptCount / 1000f * (settings.AdditionalCost?.CachedPromptCost ?? 0f); var deltaCompletionCost = stats.CompletionCount / 1000f * settings.CompletionCost; - var deltaTotal = deltaPromptCost + deltaCompletionCost; + + var deltaTotal = deltaPromptCost + deltaCachedPromptCost + deltaCompletionCost; _promptCost += deltaPromptCost; _completionCost += deltaCompletionCost; @@ -53,6 +55,8 @@ public class TokenStatistics : ITokenStatistics stat.SetState("prompt_total", stats.PromptCount + inputCount, isNeedVersion: false, source: StateSource.Application); var outputCount = int.Parse(stat.GetState("completion_total", "0")); stat.SetState("completion_total", stats.CompletionCount + outputCount, isNeedVersion: false, source: StateSource.Application); + var cachedCount = int.Parse(stat.GetState("cached_prompt_total", "0")); + stat.SetState("cached_prompt_total", stats.CachedPromptCount + cachedCount, isNeedVersion: false, source: StateSource.Application); // Total cost var total_cost = float.Parse(stat.GetState("llm_total_cost", "0")); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index e2aa844c..0b8f25cd 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -6,14 +6,14 @@ public partial class FileInstructService { public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) { - var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); var audioBytes = await DownloadFile(audio); using var stream = new MemoryStream(); stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; - var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text); + var content = await completion.TranscriptTextAsync(stream, fileName, text); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 9f2132a1..af62e479 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -27,7 +27,7 @@ public partial class FileInstructService var innerAgentId = agentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", - model: model, modelId: modelId ?? "gpt-4", multiModal: true); + model: model, modelId: modelId ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() { Id = innerAgentId, diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 41e0becc..0e049b20 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -93,7 +93,7 @@ public partial class FileInstructService } var providerName = options.Provider ?? "openai"; - var modelId = options?.ModelId ?? "gpt-4"; + var modelId = options?.ModelId ?? "gpt-4o"; var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); var model = llmProviderService.GetProviderModel(provider: provider, id: modelId); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index 71039f78..1db0d036 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -32,6 +32,10 @@ public partial class LocalFileStorageService public BinaryData GetSpeechFile(string conversationId, string fileName) { var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + if (!File.Exists(path)) + { + return BinaryData.Empty; + } using var fs = new FileStream(path, FileMode.Open, FileAccess.Read); return BinaryData.FromStream(fs); } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index f399c50e..386cce9e 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -30,7 +30,7 @@ public class CompletionProvider } else if (settings.Type == LlmModelType.Audio) { - return GetAudioCompletion(services, provider: provider, model: model); + return GetAudioTranscriber(services, provider: provider, model: model); } else { @@ -126,20 +126,39 @@ public class CompletionProvider return completer; } - public static IAudioCompletion GetAudioCompletion( + public static IAudioTranscription GetAudioTranscriber( IServiceProvider services, - string provider, - string model) + string? provider = null, + string? model = null) { - var completions = services.GetServices(); - var completer = completions.FirstOrDefault(x => x.Provider == provider); + var completions = services.GetServices(); + var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai")); if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve audio-completion provider by {provider}"); + logger.LogError($"Can't resolve audio-transcriber provider by {provider}"); + return default!; } - completer.SetModelName(model); + completer.SetModelName(model ?? "gpt-4o-mini-transcribe"); + return completer; + } + + public static IAudioSynthesis GetAudioSynthesizer( + IServiceProvider services, + string? provider = null, + string? model = null) + { + var completions = services.GetServices(); + var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai")); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve audio-synthesizer provider by {provider}"); + return default!; + } + + completer.SetModelName(model ?? "gpt-4o-mini-tts"); return completer; } @@ -172,7 +191,7 @@ public class CompletionProvider string? model = null, string? modelId = null, bool? multiModal = null, - bool? realTime = null, + bool realTime = false, bool imageGenerate = false, AgentLlmConfig? agentConfig = null) { diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 6d62c227..3baf4fb4 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false) { var models = GetProviderModels(provider) .Where(x => x.Id == id); @@ -54,10 +54,7 @@ public class LlmProviderService : ILlmProviderService models = models.Where(x => x.MultiModal == multiModal); } - if (realTime.HasValue) - { - models = models.Where(x => x.RealTime == realTime); - } + models = models.Where(x => x.RealTime == realTime); models = models.Where(x => x.ImageGeneration == imageGenerate); diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs b/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs new file mode 100644 index 00000000..8b4763f8 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs @@ -0,0 +1,92 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Instructs.Models; + +namespace BotSharp.Core.Instructs.Functions; + +public class ExecuteTemplateFn : IFunctionCallback +{ + public string Name => "util-instruct-execute_template"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public ExecuteTemplateFn( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + if (string.IsNullOrEmpty(args.TemplateName)) + { + message.Content = $"Invalid template name."; + return false; + } + + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + var template = agent.Templates.FirstOrDefault(x => x.Name.IsEqualTo(args.TemplateName)); + + if (template == null) + { + message.Content = $"Cannot find template ({args.TemplateName}) in agent {agent.Name}"; + return false; + } + + var response = await GetAiResponse(agent, args.TemplateName); + message.Content = response; + return true; + } + + private async Task GetAiResponse(Agent agent, string templateName) + { + try + { + var agentService = _services.GetRequiredService(); + var text = agentService.RenderedTemplate(agent, templateName); + + var completion = CompletionProvider.GetChatCompletion(_services, provider: agent.LlmConfig?.Provider, model: agent.LlmConfig?.Model); + var response = await completion.GetChatCompletions(new Agent() + { + Id = agent.Id + }, + new List + { + new(AgentRole.User, text) + }); + + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agent.Id) + { + continue; + } + + await hook.OnResponseGenerated(new InstructResponseModel + { + AgentId = agent.Id, + TemplateName = templateName, + Provider = completion.Provider, + Model = completion.Model, + UserMessage = text, + CompletionText = response.Content + }); + } + + return response.Content; + } + catch (Exception ex) + { + var error = $"Error when getting agent {agent.Name} instruction response."; + _logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}"); + return error; + } + + } +} diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Hooks/InstructUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Instructs/Hooks/InstructUtilityHook.cs new file mode 100644 index 00000000..e0c6d686 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Instructs/Hooks/InstructUtilityHook.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Core.Instructs.Hooks; + +public class InstructUtilityHook : IAgentUtilityHook +{ + private static string PREFIX = "util-instruct-"; + private static string EXECUTE_TEMPLATE = $"{PREFIX}execute_template"; + + public void AddUtilities(List utilities) + { + utilities.Add(new AgentUtility + { + Name = "instruct.template", + Functions = [new($"{EXECUTE_TEMPLATE}")], + Templates = [new($"{EXECUTE_TEMPLATE}.fn")] + }); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs index c44cb874..7bde20e8 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Instructs.Settings; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; +using BotSharp.Core.Instructs.Hooks; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Instructs; @@ -18,6 +19,8 @@ public class InsturctionPlugin : IBotSharpPlugin var settingService = provider.GetRequiredService(); return settingService.Bind("Instruction"); }); + + services.AddScoped(); } public bool AttachMenu(List menu) diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 587de372..492dcde3 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -80,7 +80,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository public bool DeleteConversations(IEnumerable conversationIds) => throw new NotImplementedException(); - public Conversation GetConversation(string conversationId) + public Conversation GetConversation(string conversationId, bool isLoadStates = false) => throw new NotImplementedException(); public PagedItems GetConversations(ConversationFilter filter) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 647282a4..b8ff420e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Users.Models; +using System; using System.IO; namespace BotSharp.Core.Repository; @@ -346,7 +347,7 @@ public partial class FileRepository } } - public Conversation GetConversation(string conversationId) + public Conversation GetConversation(string conversationId, bool isLoadStates = false) { var convDir = FindConversationDirectory(conversationId); if (string.IsNullOrEmpty(convDir)) return null; @@ -361,18 +362,20 @@ public partial class FileRepository record.Dialogs = CollectDialogElements(dialogFile); } - var stateFile = Path.Combine(convDir, STATE_FILE); - if (record != null) + if (isLoadStates) { - var states = CollectConversationStates(stateFile); - var curStates = new Dictionary(); - states.ForEach(x => + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + if (record != null && File.Exists(latestStateFile)) { - curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; - }); - record.States = curStates; + var stateJson = File.ReadAllText(latestStateFile); + var states = JsonSerializer.Deserialize>(stateJson, _options) ?? []; + record.States = states.ToDictionary(x => x.Key, x => + { + var elem = x.Value.RootElement.GetProperty("data"); + return elem.ValueKind != JsonValueKind.Null ? elem.ToString() : null; + }); + } } - return record; } @@ -508,6 +511,21 @@ public partial class FileRepository if (!matched) continue; + if (filter.IsLoadLatestStates) + { + var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) + { + var stateJson = File.ReadAllText(latestStateFile); + var states = JsonSerializer.Deserialize>(stateJson, _options) ?? []; + record.States = states.ToDictionary(x => x.Key, x => + { + var elem = x.Value.RootElement.GetProperty("data"); + return elem.ValueKind != JsonValueKind.Null ? elem.ToString() : null; + }); + } + } + records.Add(record); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs index e6fedc05..de7a556b 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs @@ -8,13 +8,11 @@ public class RoutingUtilityHook : IAgentUtilityHook public void AddUtilities(List utilities) { - var utility = new AgentUtility + utilities.Add(new AgentUtility { Name = "routing.tools", Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")], Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")] - }; - - utilities.Add(utility); + }); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index bec44bc4..7bfd9352 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -101,8 +101,8 @@ public partial class RoutingService Context.SetDialogs(dialogs); // Send to Next LLM - var agentId = routing.Context.GetCurrentAgentId(); - await InvokeAgent(agentId, dialogs); + var curAgentId = routing.Context.GetCurrentAgentId(); + await InvokeAgent(curAgentId, dialogs); } } else diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index fb074dfc..ee403ff8 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Templating; + namespace BotSharp.Core.Routing; public partial class RoutingService @@ -6,12 +8,20 @@ public partial class RoutingService public async Task InvokeFunction(string name, RoleDialogModel message) { var function = _services.GetServices().FirstOrDefault(x => x.Name == name); + + var isFillDummyContent = false; + var dummyFuncResponse = string.Empty; if (function == null) { - message.StopCompletion = true; - message.Content = $"Can't find function implementation of {name}."; - _logger.LogError(message.Content); - return false; + dummyFuncResponse = await GetDummyFunctionOutput(name, message); + isFillDummyContent = !string.IsNullOrEmpty(dummyFuncResponse); + if (!isFillDummyContent) + { + message.StopCompletion = true; + message.Content = $"Can't find function implementation of {name}."; + _logger.LogError(message.Content); + return false; + } } // Clone message @@ -25,7 +35,15 @@ public partial class RoutingService var progressService = _services.GetService(); // Before executing functions - clonedMessage.Indication = await function.GetIndication(message); + if (!isFillDummyContent) + { + clonedMessage.Indication = await function.GetIndication(message); + } + else + { + clonedMessage.Indication = "Running"; + } + if (progressService?.OnFunctionExecuting != null) { await progressService.OnFunctionExecuting(clonedMessage); @@ -40,7 +58,15 @@ public partial class RoutingService try { - result = await function.Execute(clonedMessage); + if (!isFillDummyContent) + { + result = await function.Execute(clonedMessage); + } + else + { + clonedMessage.Content = dummyFuncResponse; + result = true; + } // After functions have been executed foreach (var hook in hooks) @@ -87,4 +113,32 @@ public partial class RoutingService return result; } + + private async Task GetDummyFunctionOutput(string functionName, RoleDialogModel message) + { + if (string.IsNullOrEmpty(message.CurrentAgentId)) + { + return null; + } + + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + var found = agent?.Functions?.FirstOrDefault(x => x.Name == functionName); + if (string.IsNullOrWhiteSpace(found?.Output)) + { + return null; + } + + var render = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + var dict = new Dictionary(); + foreach (var item in state.GetStates()) + { + dict[item.Key] = item.Value; + } + + var text = render.Render(found.Output, dict); + return text; + } } diff --git a/src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets b/src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets new file mode 100644 index 00000000..ffb6c063 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-instruct-execute_template.json b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-instruct-execute_template.json new file mode 100644 index 00000000..52780e61 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-instruct-execute_template.json @@ -0,0 +1,14 @@ +{ + "name": "util-instruct-execute_template", + "description": "Select a specific template that can handle the user's request.", + "parameters": { + "type": "object", + "properties": { + "template_name": { + "type": "string", + "description": "The template name that is selected for handling the request." + } + }, + "required": [ "template_name" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-instruct-execute_template.fn.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-instruct-execute_template.fn.liquid new file mode 100644 index 00000000..aa33961d --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-instruct-execute_template.fn.liquid @@ -0,0 +1,3 @@ +please call function util-routing-execute_template if user wants to use a template to fulfill a specific task. +Please ensure each template is executed only once. +Please output the template response directly without changing anthything. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs index 30e1d066..8de3db70 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -26,7 +26,14 @@ public class InstructionLogHook : InstructHookBase public override async Task OnResponseGenerated(InstructResponseModel response) { var settings = _services.GetRequiredService(); - if (!settings.EnableLog || response == null) return; + if (response == null + || string.IsNullOrWhiteSpace(response.AgentId) + || settings == null + || !settings.Logging.Enabled + || settings.Logging.ExcludedAgentIds.Contains(response.AgentId)) + { + return; + } var db = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -49,6 +56,7 @@ public class InstructionLogHook : InstructHookBase UserId = user?.Id } }); - return; + + await base.OnResponseGenerated(response); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index f032f504..dc0f2b0f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -138,7 +138,7 @@ public class ConversationController : ControllerBase } [HttpGet("/conversation/{conversationId}")] - public async Task GetConversation([FromRoute] string conversationId) + public async Task GetConversation([FromRoute] string conversationId, [FromQuery] bool isLoadStates = false) { var service = _services.GetRequiredService(); var userService = _services.GetRequiredService(); @@ -151,7 +151,8 @@ public class ConversationController : ControllerBase var filter = new ConversationFilter { Id = conversationId, - UserId = !isAdmin ? user.Id : null + UserId = !isAdmin ? user.Id : null, + IsLoadLatestStates = isLoadStates }; var conversations = await service.GetConversations(filter); if (conversations.Items.IsNullOrEmpty()) @@ -161,7 +162,6 @@ public class ConversationController : ControllerBase var result = ConversationViewModel.FromSession(conversations.Items.First()); var state = _services.GetRequiredService(); - result.States = state.Load(conversationId, isReadOnly: true); user = await userService.GetUser(result.User.Id); result.User = UserViewModel.FromUser(user); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 6a1c3705..3db47703 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -499,8 +499,8 @@ public class InstructModeController : ControllerBase file.CopyTo(stream); stream.Position = 0; - var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); - var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text); + var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); + var content = await completion.TranscriptTextAsync(stream, file.FileName, text); viewModel.Content = content; stream.Close(); return viewModel; @@ -520,8 +520,8 @@ public class InstructModeController : ControllerBase var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1"); - var binaryData = await completion.GenerateAudioFromTextAsync(input.Text); + var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model); + var binaryData = await completion.GenerateAudioAsync(input.Text); var stream = binaryData.ToStream(); stream.Position = 0; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs index 7d6fc056..8773e35b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs @@ -22,7 +22,7 @@ public class RealtimeController : ControllerBase [HttpGet("/agent/{agentId}/realtime/session")] public async Task CreateSession(string agentId) { - var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4"); + var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4o"); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(agentId); diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs index a660a525..686974b9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs @@ -31,7 +31,7 @@ public class ConversationViewModel public string? TaskId { get; set; } public string Status { get; set; } - public Dictionary States { get; set; } + public Dictionary States { get; set; } = []; public List Tags { get; set; } = new(); @@ -55,7 +55,8 @@ public class ConversationViewModel Channel = sess.Channel, Status = sess.Status, TaskId = sess.TaskId, - Tags = sess.Tags ?? new(), + Tags = sess.Tags ?? [], + States = sess.States ?? [], CreatedTime = sess.CreatedTime, UpdatedTime = sess.UpdatedTime }; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index d79ca6c2..1bc9c331 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -16,7 +16,7 @@ public class AudioHandlerPlugin : IBotSharpPlugin return settingService.Bind("AudioHandler"); }); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 9c0ab4f4..f20a675a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -91,7 +91,7 @@ public class HandleAudioRequestFn : IFunctionCallback using var stream = new MemoryStream(bytes); stream.Position = 0; - var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName); + var result = await audioCompletion.TranscriptTextAsync(stream, fileName); transcripts.Add(result); stream.Close(); } @@ -104,9 +104,9 @@ public class HandleAudioRequestFn : IFunctionCallback return string.Join("\r\n\r\n", transcripts); } - private IAudioCompletion PrepareModel() + private IAudioTranscription PrepareModel() { - return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1"); + return CompletionProvider.GetAudioTranscriber(_serviceProvider); } private bool ParseAudioFileType(string fileName) diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 00d1da58..aa35d17b 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// /// Native Whisper provider for speech to text conversion /// -public class NativeWhisperProvider : IAudioCompletion +public class NativeWhisperProvider : IAudioTranscription { private static WhisperProcessor _whisperProcessor; @@ -29,7 +29,7 @@ public class NativeWhisperProvider : IAudioCompletion _logger = logger; } - public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + public async Task TranscriptTextAsync(Stream audio, string audioFileName, string? text = null) { var textResult = new List(); @@ -49,7 +49,7 @@ public class NativeWhisperProvider : IAudioCompletion return audioOutput.ToString(); } - public async Task GenerateAudioFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3") { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index eba22bfa..8a2c1c53 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -31,6 +31,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index 36f0c3ee..082daacf 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -4,7 +4,7 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + public async Task TranscriptTextAsync(Stream audio, string audioFileName, string? text = null) { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index 44d49f3c..57c1f97e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -4,27 +4,27 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateAudioFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3") { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var (voice, options) = PrepareGenerationOptions(); - var result = await audioClient.GenerateSpeechAsync(text, voice, options); + var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format); + var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options); return result.Value; } - private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format) { var state = _services.GetRequiredService(); - var voice = GetVoice(state.GetState("speech_generate_voice")); - var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speechVoice = GetVoice(voice ?? "alloy"); + var responseFormat = GetSpeechFormat(format ?? "mp3"); var speed = GetSpeed(state.GetState("speech_generate_speed")); var options = new SpeechGenerationOptions { - ResponseFormat = format, - SpeedRatio = speed + ResponseFormat = responseFormat, + SpeedRatio = speed, }; return (voice, options); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs index 2948703e..595e3f9a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; -public partial class AudioCompletionProvider : IAudioCompletion +public partial class AudioCompletionProvider : IAudioTranscription { private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 99b734e6..7d848eb0 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -106,6 +106,7 @@ public class ChatHubConversationHook : ConversationHookBase if (!AllowSendingMessage()) return; var conv = _services.GetRequiredService(); + var state = _services.GetRequiredService(); var json = JsonSerializer.Serialize(new ChatResponseModel() { ConversationId = conv.ConversationId, @@ -114,6 +115,7 @@ public class ChatHubConversationHook : ConversationHookBase Function = message.FunctionName, RichContent = message.SecondaryRichContent ?? message.RichContent, Data = message.Data, + States = state.GetStates(), Sender = new UserViewModel() { FirstName = "AI", diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs index 7666e525..fcd2be3a 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs @@ -67,7 +67,7 @@ public class HandleEmailReaderFn : IFunctionCallback var llmProviderService = _services.GetRequiredService(); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4"); + var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4o"); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); var convService = _services.GetRequiredService(); var conversationId = convService.ConversationId; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 710c2c80..ff19fbf1 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -100,7 +100,7 @@ public class ReadImageFn : IFunctionCallback { var llmProviderService = _services.GetRequiredService(); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4o", multiModal: true); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 2aeb64a1..5a2d2d9b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -78,7 +78,7 @@ public class ReadPdfFn : IFunctionCallback { var llmProviderService = _services.GetRequiredService(); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4o", multiModal: true); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); var response = await completion.GetChatCompletions(agent, dialogs); return response.Content; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs index f274eb0c..8ad69be5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs @@ -12,6 +12,7 @@ public class FunctionDefMongoElement public string? VisibilityExpression { get; set; } public string? Impact { get; set; } public FunctionParametersDefMongoElement Parameters { get; set; } = new(); + public string? Output { get; set; } public static FunctionDefMongoElement ToMongoElement(FunctionDef function) { @@ -27,7 +28,8 @@ public class FunctionDefMongoElement Type = function.Parameters.Type, Properties = JsonSerializer.Serialize(function.Parameters.Properties), Required = function.Parameters.Required, - } + }, + Output = function.Output }; } @@ -45,7 +47,8 @@ public class FunctionDefMongoElement Type = function.Parameters.Type, Properties = JsonSerializer.Deserialize(function.Parameters.Properties.IfNullOrEmptyAs("{}")), Required = function.Parameters.Required, - } + }, + Output = function.Output }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 069845ef..0d3a670b 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -35,7 +35,7 @@ public partial class MongoRepository UpdateAgentProfiles(agent.Id, agent.Profiles); break; case AgentField.Label: - UpdateAgentLabels(agent.Id, agent.Profiles); + UpdateAgentLabels(agent.Id, agent.Labels); break; case AgentField.RoutingRule: UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index ebae075b..f0e89473 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -299,26 +299,25 @@ public partial class MongoRepository _dc.Conversations.UpdateOne(filter, update); } - public Conversation GetConversation(string conversationId) + public Conversation GetConversation(string conversationId, bool isLoadStates = false) { if (string.IsNullOrEmpty(conversationId)) return null; var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); var conv = _dc.Conversations.Find(filterConv).FirstOrDefault(); var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault(); - var states = _dc.ConversationStates.Find(filterState).FirstOrDefault(); if (conv == null) return null; var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List(); - var curStates = new Dictionary(); - states.States.ForEach(x => + var curStates = conv.LatestStates?.ToDictionary(x => x.Key, x => { - curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; - }); + var jsonDoc = JsonDocument.Parse(x.Value.ToJson()); + var data = jsonDoc.RootElement.GetProperty("data"); + return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; + }) ?? []; return new Conversation { @@ -456,19 +455,34 @@ public partial class MongoRepository var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList(); var count = _dc.Conversations.CountDocuments(filterDef); - var conversations = conversationDocs.Select(x => new Conversation + var conversations = conversationDocs.Select(x => { - Id = x.Id.ToString(), - AgentId = x.AgentId.ToString(), - UserId = x.UserId.ToString(), - TaskId = x.TaskId, - Title = x.Title, - Channel = x.Channel, - Status = x.Status, - DialogCount = x.DialogCount, - Tags = x.Tags ?? new(), - CreatedTime = x.CreatedTime, - UpdatedTime = x.UpdatedTime + var states = new Dictionary(); + if (filter.IsLoadLatestStates) + { + states = x.LatestStates.ToDictionary(p => p.Key, p => + { + var jsonDoc = JsonDocument.Parse(p.Value.ToJson()); + var data = jsonDoc.RootElement.GetProperty("data"); + return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; + }); + } + + return new Conversation + { + Id = x.Id.ToString(), + AgentId = x.AgentId.ToString(), + UserId = x.UserId.ToString(), + TaskId = x.TaskId, + Title = x.Title, + Channel = x.Channel, + Status = x.Status, + DialogCount = x.DialogCount, + Tags = x.Tags ?? [], + States = states, + CreatedTime = x.CreatedTime, + UpdatedTime = x.UpdatedTime + }; }).ToList(); return new PagedItems diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index 2f7f79c6..cdd9c308 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -48,6 +48,9 @@ public class RealtimeSessionBody [JsonPropertyName("turn_detection")] public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new(); + + [JsonPropertyName("input_audio_noise_reduction")] + public InputAudioNoiseReduction InputAudioNoiseReduction { get; set; } = new(); } public class RealtimeSessionTurnDetection @@ -58,28 +61,39 @@ public class RealtimeSessionTurnDetection /// /// Milliseconds /// - [JsonPropertyName("prefix_padding_ms")] + /*[JsonPropertyName("prefix_padding_ms")] public int PrefixPadding { get; set; } = 300; [JsonPropertyName("silence_duration_ms")] public int SilenceDuration { get; set; } = 500; [JsonPropertyName("threshold")] - public float Threshold { get; set; } = 0.5f; + public float Threshold { get; set; } = 0.5f;*/ [JsonPropertyName("type")] - public string Type { get; set; } = "server_vad"; + public string Type { get; set; } = "semantic_vad"; + + [JsonPropertyName("eagerness")] + public string eagerness { get;set; } = "auto"; } public class InputAudioTranscription { [JsonPropertyName("model")] - public string Model { get; set; } = null!; + public string Model { get; set; } = "gpt-4o-transcribe"; [JsonPropertyName("language")] - public string Language { get; set; } = "en"; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Language { get; set; } [JsonPropertyName("prompt")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Prompt { get; set; } +} + +public class InputAudioNoiseReduction +{ + [JsonPropertyName("type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Type { get; set; } = "far_field"; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index c1adbbe7..fe3dae99 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -33,7 +33,8 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddRefitClient() diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs deleted file mode 100644 index 338affd9..00000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs +++ /dev/null @@ -1,23 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio; - -public partial class AudioCompletionProvider : IAudioCompletion -{ - private readonly IServiceProvider _services; - - public string Provider => "openai"; - public string Model => _model; - - private string _model; - - public AudioCompletionProvider(IServiceProvider service) - { - _services = service; - } - - public void SetModelName(string model) - { - _model = model; - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs similarity index 72% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs index a0c0225a..feb846f5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs @@ -2,29 +2,45 @@ using OpenAI.Audio; namespace BotSharp.Plugin.OpenAI.Providers.Audio; -public partial class AudioCompletionProvider +public class AudioSynthesisProvider : IAudioSynthesis { - public async Task GenerateAudioFromTextAsync(string text) + private readonly IServiceProvider _services; + public string Provider => "openai"; + public string Model => _model; + + private string _model; + + public AudioSynthesisProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } + + public async Task GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null) { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var (voice, options) = PrepareGenerationOptions(); - var result = await audioClient.GenerateSpeechAsync(text, voice, options); + var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format); + var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options); return result.Value; } - private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format) { var state = _services.GetRequiredService(); - var voice = GetVoice(state.GetState("speech_generate_voice")); - var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speechVoice = GetVoice(voice ?? "alloy"); + var responseFormat = GetSpeechFormat(format ?? "mp3"); var speed = GetSpeed(state.GetState("speech_generate_speed")); var options = new SpeechGenerationOptions { - ResponseFormat = format, - SpeedRatio = speed + ResponseFormat = responseFormat, + SpeedRatio = speed, }; return (voice, options); @@ -32,10 +48,8 @@ public partial class AudioCompletionProvider private GeneratedSpeechVoice GetVoice(string input) { - var value = !string.IsNullOrEmpty(input) ? input : "alloy"; - GeneratedSpeechVoice voice; - switch (value) + switch (input) { case "echo": voice = GeneratedSpeechVoice.Echo; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs similarity index 83% rename from src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs index f6213eb7..079df13f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs @@ -2,9 +2,26 @@ using OpenAI.Audio; namespace BotSharp.Plugin.OpenAI.Providers.Audio; -public partial class AudioCompletionProvider +public class AudioTranscriptionProvider : IAudioTranscription { - public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + private readonly IServiceProvider _services; + + public string Provider => "openai"; + public string Model => _model; + + private string _model; + + public AudioTranscriptionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } + + public async Task TranscriptTextAsync(Stream audio, string audioFileName, string? text = null) { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 031fab9c..7d6bdcea 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -84,6 +84,7 @@ public class ChatCompletionProvider : IChatCompletion Provider = Provider, Model = _model, PromptCount = response.Value?.Usage?.InputTokenCount ?? 0, + CachedPromptCount = response.Value?.Usage?.InputTokenDetails?.CachedTokenCount ?? 0, CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0 }); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 2510c852..fe5b32fe 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -96,14 +96,24 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { // Triggering model inference - await SendEventToModel(new + if (!string.IsNullOrEmpty(instructions)) { - type = "response.create", - response = new + await SendEventToModel(new { - instructions - } - }); + type = "response.create", + response = new + { + instructions + } + }); + } + else + { + await SendEventToModel(new + { + type = "response.create" + }); + } } public async Task CancelModelResponse() @@ -317,7 +327,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - var realitmeModelSettings = _services.GetRequiredService(); + var realtimeModelSettings = _services.GetRequiredService(); var sessionUpdate = new { @@ -328,23 +338,27 @@ public class RealTimeCompletionProvider : IRealTimeCompletion OutputAudioFormat = "g711_ulaw", InputAudioTranscription = new InputAudioTranscription { - Model = "whisper-1", - Language = "en", + Model = realtimeModelSettings.InputAudioTranscription.Model, + Language = realtimeModelSettings.InputAudioTranscription.Language, Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024) }, - Voice = "alloy", + Voice = realtimeModelSettings.Voice, Instructions = instruction, ToolChoice = "auto", Tools = functions, Modalities = [ "text", "audio" ], - Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f), - MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens, + Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f), + MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens, TurnDetection = new RealtimeSessionTurnDetection { - InterruptResponse = interruptResponse, - Threshold = realitmeModelSettings.TurnDetection.Threshold, - PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding, - SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration + InterruptResponse = interruptResponse/*, + Threshold = realtimeModelSettings.TurnDetection.Threshold, + PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding, + SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/ + }, + InputAudioNoiseReduction = new InputAudioNoiseReduction + { + Type = "near_field" } } }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 8fe2ecce..775a9384 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -10,6 +10,19 @@ + + + + + + + + + PreserveNewest + + + PreserveNewest + PreserveNewest @@ -33,8 +46,4 @@ - - - - diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs new file mode 100644 index 00000000..92d3abe4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs @@ -0,0 +1,65 @@ +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioOutboundController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly ILogger _logger; + + public TwilioOutboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _context = context; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/voice/init-outbound-call")] + public async Task InitiateOutboundCall(ConversationalVoiceRequest request) + { + var twilio = _services.GetRequiredService(); + + VoiceResponse response = default!; + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api") + { + response = new VoiceResponse(); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailStarting(request); + }); + + var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); + response.Play(new Uri(url)); + } + else + { + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + ActionOnEmptyResult = true, + CallbackPath = $"twilio/voice/receive/1?agent-id={request.AgentId}&conversation-id={request.ConversationId}", + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + + response = twilio.ReturnNoninterruptedInstructions(instruction); + } + + return TwiML(response); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 6960f489..f22d46e6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -35,17 +35,9 @@ public class TwilioStreamController : TwilioController throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } + var twilio = _services.GetRequiredService(); VoiceResponse response = default!; - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api" && - request.InitAudioFile != null) - { - response = new VoiceResponse(); - response.Play(new Uri(request.InitAudioFile)); - return TwiML(response); - } - var instruction = new ConversationalVoiceResponse { ConversationId = request.ConversationId, @@ -67,10 +59,25 @@ public class TwilioStreamController : TwilioController }); request.ConversationId = await InitConversation(request); + instruction.ConversationId = request.ConversationId; - var twilio = _services.GetRequiredService(); + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api") + { + response = new VoiceResponse(); - response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction); + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailStarting(request); + }); + + var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); + response.Play(new Uri(url)); + } + else + { + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); + } await HookEmitter.Emit(_services, async hook => { @@ -91,7 +98,7 @@ public class TwilioStreamController : TwilioController { var conv = new Conversation { - AgentId = request.AgentId ?? _settings.AgentId, + AgentId = request.AgentId, Channel = ConversationChannel.Phone, ChannelId = request.CallSid, Title = $"Incoming phone call from {request.From}", diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 079e13c5..2a1c7cd0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; @@ -7,6 +6,7 @@ using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Twilio.Http; +using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Controllers; @@ -49,27 +49,26 @@ public class TwilioVoiceController : TwilioController throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } - VoiceResponse response = null; + VoiceResponse response = default!; + request.ConversationId = $"twilio_{request.CallSid}"; + var instruction = new ConversationalVoiceResponse { + AgentId = request.AgentId, ConversationId = request.ConversationId, - SpeechPaths = ["twilio/welcome.mp3"], + SpeechPaths = [$"twilio/welcome-{request.AgentId}.mp3"], ActionOnEmptyResult = true }; + await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true }); - request.ConversationId = $"TwilioVoice_{request.CallSid}"; - instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"; - var twilio = _services.GetRequiredService(); if (string.IsNullOrWhiteSpace(request.Intent)) { + instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"; response = twilio.ReturnNoninterruptedInstructions(instruction); } else @@ -80,6 +79,7 @@ public class TwilioVoiceController : TwilioController await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); var callerMessage = new CallerMessage() { + AgentId = request.AgentId, ConversationId = request.ConversationId, SeqNumber = seqNum, Content = request.Intent, @@ -88,15 +88,14 @@ public class TwilioVoiceController : TwilioController }; await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post); + // delay 3 seconds to wait for the first message reply and caller is listening dudu sound + await Task.Delay(1000 * 3); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post); } await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); - }, new HookEmitOption - { - OnlyOnce = true }); return TwiML(response); @@ -114,21 +113,25 @@ public class TwilioVoiceController : TwilioController var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + + // Fetch all accumulated caller message. var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(request.ConversationId, request.SeqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); if (!string.IsNullOrWhiteSpace(text)) { + // Concanate with incoming message messages.Add(text); await sessionManager.StageCallerMessageAsync(request.ConversationId, request.SeqNum, text); } - VoiceResponse response = null; + VoiceResponse response = default!; if (messages.Any()) { var messageContent = string.Join("\r\n", messages); var callerMessage = new CallerMessage() { + AgentId = request.AgentId, ConversationId = request.ConversationId, SeqNumber = request.SeqNum, Content = messageContent, @@ -141,14 +144,11 @@ public class TwilioVoiceController : TwilioController await messageQueue.EnqueueAsync(callerMessage); response = new VoiceResponse(); - response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post); await HookEmitter.Emit(_services, async hook => { await hook.OnReceivedUserMessage(request); - }, new HookEmitOption - { - OnlyOnce = true }); } else @@ -159,21 +159,19 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); - }, new HookEmitOption - { - OnlyOnce = true }); - response = twilio.HangUp(null); + response = twilio.HangUp(string.Empty); } // keep waiting for user response else { var instruction = new ConversationalVoiceResponse { + AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = new List(), - CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", + CallbackPath = $"twilio/voice/receive/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}&attempts={++request.Attempts}", ActionOnEmptyResult = true }; @@ -185,9 +183,6 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnWaitingUserResponse(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true }); response = twilio.ReturnInstructions(instruction); @@ -210,9 +205,10 @@ public class TwilioVoiceController : TwilioController var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); - if (request.SpeechResult != null) + var text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); + if (!string.IsNullOrEmpty(text)) { - await sessionManager.StageCallerMessageAsync(request.ConversationId, nextSeqNum, request.SpeechResult); + await sessionManager.StageCallerMessageAsync(request.ConversationId, nextSeqNum, text); } var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum); @@ -225,112 +221,13 @@ public class TwilioVoiceController : TwilioController { request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!"; await hook.OnAgentHangUp(request); - }, new HookEmitOption - { - OnlyOnce = true }); response = twilio.HangUp($"twilio/error.mp3"); } else if (reply == null) { - var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); - if (indication != null) - { - _logger.LogWarning($"Indication: {indication}"); - var speechPaths = new List(); - int segIndex = 0; - foreach (var text in indication.Split('|')) - { - var seg = text.Trim(); - if (seg.StartsWith('#')) - { - speechPaths.Add($"twilio/{seg.Substring(1)}.mp3"); - } - else - { - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(seg); - - // add hold-on - var holdOnIndex = Random.Shared.Next(1, 10); - if (holdOnIndex < 7) - { - speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3"); - } - - var fileName = $"indication_{request.SeqNum}_{segIndex}.mp3"; - fileStorage.SaveSpeechFile(request.ConversationId, fileName, data); - speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}"); - - // add typing - var typingIndex = Random.Shared.Next(1, 7); - if (typingIndex < 4) - { - speechPaths.Add($"twilio/typing-{typingIndex}.mp3"); - } - segIndex++; - } - } - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - SpeechPaths = speechPaths, - CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", - ActionOnEmptyResult = true - }; - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnIndicationGenerated(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true - }); - - response = twilio.ReturnInstructions(instruction); - - await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum); - } - else - { - var instructions = new List - { - }; - - // add hold-on - var holdOnIndex = Random.Shared.Next(1, 15); - if (holdOnIndex < 9) - { - instructions.Add($"twilio/hold-on-long-{holdOnIndex}.mp3"); - } - - // add typing - var typingIndex = Random.Shared.Next(1, 7); - if (typingIndex < 4) - { - instructions.Add($"twilio/typing-{typingIndex}.mp3"); - } - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - SpeechPaths = instructions, - CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", - ActionOnEmptyResult = true - }; - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnWaitingAgentResponse(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true - }); - - response = twilio.ReturnInstructions(instruction); - } + response = await twilio.WaitingForAiResponse(request); } else { @@ -339,9 +236,6 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentTransferring(request, _settings); - }, new HookEmitOption - { - OnlyOnce = true }); response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); @@ -353,18 +247,16 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); - }, new HookEmitOption - { - OnlyOnce = true }); } else { var instruction = new ConversationalVoiceResponse { + AgentId = request.AgentId, ConversationId = request.ConversationId, SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"], - CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}", + CallbackPath = $"twilio/voice/receive/{nextSeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}", ActionOnEmptyResult = true, Hints = reply.Hints }; @@ -372,9 +264,6 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentResponsing(request, instruction); - }, new HookEmitOption - { - OnlyOnce = true }); response = twilio.ReturnInstructions(instruction); @@ -384,38 +273,6 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - [ValidateRequest] - [HttpPost("twilio/voice/init-outbound-call")] - public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request) - { - VoiceResponse response = default!; - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api" && - request.InitAudioFile != null) - { - response = new VoiceResponse(); - response.Play(new Uri(request.InitAudioFile)); - return TwiML(response); - } - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - ActionOnEmptyResult = true, - CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}", - }; - - if (request.InitAudioFile != null) - { - instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}"; - instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"); - } - - var twilio = _services.GetRequiredService(); - response = twilio.ReturnNoninterruptedInstructions(instruction); - return TwiML(response); - } - [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) @@ -433,8 +290,40 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/hang-up")] public async Task Hangup(ConversationalVoiceRequest request) { + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + var twilio = _services.GetRequiredService(); - var response = twilio.HangUp("twilio/bye.mp3"); + var response = twilio.HangUp(instruction); + return TwiML(response); + } + + [ValidateRequest] + [HttpPost("twilio/voice/transfer-call")] + public async Task TransferCall(ConversationalVoiceRequest request) + { + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + TransferTo = request.TransferTo + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + + var twilio = _services.GetRequiredService(); + var response = twilio.TransferCall(instruction); return TwiML(response); } @@ -445,8 +334,7 @@ public class TwilioVoiceController : TwilioController if (request.CallStatus == "completed") { if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api" && - request.InitAudioFile != null) + request.Direction == "outbound-api") { // voicemail await HookEmitter.Emit(_services, async hook => @@ -481,13 +369,4 @@ public class TwilioVoiceController : TwilioController } return result; } - - private string GenerateStatesParameter(List states) - { - if (states is null || states.Count == 0) - { - return null; - } - return string.Join("&", states.Select(x => $"states={x}")); - } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index c9ed4710..0c9aa83e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -8,4 +8,5 @@ public interface ITwilioCallStatusHook Task OnVoicemailLeft(ConversationalVoiceRequest request); Task OnUserDisconnected(ConversationalVoiceRequest request); Task OnRecordingCompleted(ConversationalVoiceRequest request); + Task OnVoicemailStarting(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 29d4bc7c..6c66924b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -45,15 +45,6 @@ public interface ITwilioSessionHook Task OnWaitingUserResponse(ConversationalVoiceRequest request, ConversationalVoiceResponse response) => Task.CompletedTask; - /// - /// On agent generated indication - /// - /// - /// - /// - Task OnIndicationGenerated(ConversationalVoiceRequest request, ConversationalVoiceResponse response) - => Task.CompletedTask; - /// /// Waiting agent response /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs index 4b9c6a84..c021737c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs @@ -4,17 +4,18 @@ namespace BotSharp.Plugin.Twilio.Models { public class CallerMessage { - public string ConversationId { get; set; } + public string AgentId { get; set; } = null!; + public string ConversationId { get; set; } = null!; public int SeqNumber { get; set; } - public string Content { get; set; } - public string Digits { get; set; } - public string From { get; set; } + public string Content { get; set; } = null!; + public string? Digits { get; set; } + public string From { get; set; } = null!; public Dictionary States { get; set; } = new(); - public KeyValuePair[] RequestHeaders { get; set; } + public KeyValuePair[] RequestHeaders { get; set; } = []; public override string ToString() { - return $"{ConversationId}-{SeqNumber}"; + return $"{ConversationId}-{SeqNumber}: {Content}"; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 25e3ba0d..46a2cb2b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -27,6 +27,9 @@ public class ConversationalVoiceRequest : VoiceRequest [FromForm] public string? CallbackSource { get; set; } + [FromQuery(Name = "transfer-to")] + public string? TransferTo { get; set; } + /// /// machine_start /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs index 7870f961..12094fad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models; public class ConversationalVoiceResponse { + public string AgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; public List SpeechPaths { get; set; } = []; public string CallbackPath { get; set; } @@ -13,4 +14,9 @@ public class ConversationalVoiceResponse public int Timeout { get; set; } = 3; public string Hints { get; set; } + + /// + /// The Phone Number to transfer to + /// + public string? TransferTo { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs index 9fa7d03c..e2b78fd4 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; using Twilio.Rest.Api.V2010.Account; @@ -27,6 +29,7 @@ public class HangupPhoneCallFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs); + var fileStorage = _services.GetRequiredService(); var routing = _services.GetRequiredService(); var conversationId = routing.Context.ConversationId; var states = _services.GetRequiredService(); @@ -34,26 +37,33 @@ public class HangupPhoneCallFn : IFunctionCallback if (string.IsNullOrEmpty(callSid)) { - message.Content = "The call has not been initiated."; + message.Content = "Please hang up the phone directly."; _logger.LogError(message.Content); return false; } - if (args.AnythingElseToHelp) - { - message.Content = "Tell me how I can help."; - } - else - { - var call = CallResource.Update( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"), - pathSid: callSid - ); + var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}"; - message.Content = "The call is ending."; - message.StopCompletion = true; + // Generate initial assistant audio + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.ResponseContent)) + { + var completion = CompletionProvider.GetAudioSynthesizer(_services); + var data = await completion.GenerateAudioAsync(args.ResponseContent); + initAudioFile = "ending.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + + processUrl += $"&init-audio-file={initAudioFile}"; } + var call = CallResource.Update( + url: new Uri(processUrl), + pathSid: callSid + ); + + message.Content = args.Reason; + message.StopCompletion = true; + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/LeaveVoicemailFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/LeaveVoicemailFn.cs new file mode 100644 index 00000000..96aa5580 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/LeaveVoicemailFn.cs @@ -0,0 +1,59 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class LeaveVoicemailFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly TwilioSetting _setting; + + public string Name => "util-twilio-leave_voicemail"; + public string Indication => "leaving a voicemail"; + + public LeaveVoicemailFn( + IServiceProvider services, + ILogger logger, + TwilioSetting setting) + { + _services = services; + _logger = logger; + _setting = setting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var fileStorage = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var states = _services.GetRequiredService(); + var callSid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(callSid)) + { + message.Content = "The call has not been initiated."; + _logger.LogError(message.Content); + return false; + } + + // Generate voice message audio + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.VoicemailMessage)) + { + var completion = CompletionProvider.GetAudioSynthesizer(_services); + var data = await completion.GenerateAudioAsync(args.VoicemailMessage); + initAudioFile = "voicemail.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + } + + message.Content = args.VoicemailMessage; + message.StopCompletion = true; + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 1e10a135..905a2253 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -62,15 +62,15 @@ public class OutboundPhoneCallFn : IFunctionCallback states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId); var processUrl = $"{_twilioSetting.CallbackHost}/twilio"; - var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}"; - var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}"; + var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}"; + var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}"; // Generate initial assistant audio string initAudioFile = null; if (!string.IsNullOrEmpty(args.InitialMessage)) { - var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); - var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage); + var completion = CompletionProvider.GetAudioSynthesizer(_services); + var data = await completion.GenerateAudioAsync(args.InitialMessage); initAudioFile = "intial.mp3"; fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data); @@ -94,7 +94,7 @@ public class OutboundPhoneCallFn : IFunctionCallback processUrl += "/voice/init-outbound-call"; } - processUrl += $"?conversation-id={newConversationId}"; + processUrl += $"?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}"; if (!string.IsNullOrEmpty(initAudioFile)) { processUrl += $"&init-audio-file={initAudioFile}"; @@ -108,8 +108,9 @@ public class OutboundPhoneCallFn : IFunctionCallback statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection machineDetection: _twilioSetting.MachineDetection, + machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout, record: _twilioSetting.RecordingEnabled, - recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}"); + recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}"); var convService = _services.GetRequiredService(); var routing = _services.GetRequiredService(); @@ -127,7 +128,7 @@ public class OutboundPhoneCallFn : IFunctionCallback string entryAgentId, string originConversationId, string newConversationId, - CallResource resource) + CallResource call) { // new scope service for isolated conversation using var scope = _services.CreateScope(); @@ -140,7 +141,7 @@ public class OutboundPhoneCallFn : IFunctionCallback Id = newConversationId, AgentId = entryAgentId, Channel = ConversationChannel.Phone, - ChannelId = resource.Sid, + ChannelId = call.Sid, Title = args.InitialMessage }); @@ -159,7 +160,10 @@ public class OutboundPhoneCallFn : IFunctionCallback convService.SetConversationId(newConversationId, [ new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId), - new MessageState("phone_number", resource.To) + new MessageState("phone_from", call.From), + new MessageState("phone_direction", call.Direction), + new MessageState("phone_number", call.To), + new MessageState("twilio_call_sid", call.Sid) ]); convService.SaveStates(); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs new file mode 100644 index 00000000..a5c64423 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs @@ -0,0 +1,69 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class TransferPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-transfer_phone_call"; + public string Indication => "Transferring the active line"; + + public TransferPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + var sid = states.GetState("twilio_call_sid"); + if (string.IsNullOrEmpty(sid)) + { + _logger.LogError("Twilio call sid is empty."); + message.Content = "There is an error when transferring the phone call."; + return false; + } + + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?agent-id={routing.Context.EntryAgentId}&conversation-id={conversationId}&transfer-to={args.PhoneNumber}"; + + // Generate initial assistant audio + if (!string.IsNullOrEmpty(args.TransitionMessage)) + { + var completion = CompletionProvider.GetAudioSynthesizer(_services); + var data = await completion.GenerateAudioAsync(args.TransitionMessage); + var initAudioFile = "transfer.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + + processUrl += $"&init-audio-file={initAudioFile}"; + } + + // Transfer call + var call = CallResource.Update( + pathSid: sid, + url: new Uri(processUrl)); + + message.Content = args.TransitionMessage; + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index fa0b3179..58999457 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -7,8 +7,10 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call"; private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; - public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; + private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; + private static string LEAVE_VOICEMAIL_FN = $"{PREFIX}leave_voicemail"; public void AddUtilities(List utilities) { @@ -18,13 +20,13 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook Functions = [ new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{TRANSFER_PHONE_CALL_FN}"), new($"{HANGUP_PHONE_CALL_FN}"), - new($"{TEXT_MESSAGE_FN}") + new($"{TEXT_MESSAGE_FN}"), + new($"{LEAVE_VOICEMAIL_FN}") ], Templates = [ - new($"{OUTBOUND_PHONE_CALL_FN}.fn"), - new($"{HANGUP_PHONE_CALL_FN}.fn") ] }; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs new file mode 100644 index 00000000..abd964b4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class ForwardPhoneCallArgs +{ + [JsonPropertyName("phone_number")] + public string PhoneNumber { get; set; } = null!; + + [JsonPropertyName("transition_message")] + public string TransitionMessage { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs index efd8114a..02242ccf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs @@ -4,6 +4,9 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; public class HangupPhoneCallArgs { - [JsonPropertyName("anything_else_to_help")] - public bool AnythingElseToHelp { get; set; } = true; + [JsonPropertyName("reason")] + public string Reason { get; set; } = null!; + + [JsonPropertyName("response_content")] + public string ResponseContent { get; set; } = null!; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LeaveVoicemailArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LeaveVoicemailArgs.cs new file mode 100644 index 00000000..85cc3913 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LeaveVoicemailArgs.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class LeaveVoicemailArgs +{ + [JsonPropertyName("phone_number")] + public string PhoneNumber { get; set; } = null!; + + [JsonPropertyName("voicemail_message")] + public string VoicemailMessage { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index fb992c4a..35bc3f12 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -74,7 +74,7 @@ public class TwilioMessageQueueService : BackgroundService } httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST"; - AssistantMessage reply = null; + AssistantMessage reply = default!; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); @@ -87,7 +87,7 @@ public class TwilioMessageQueueService : BackgroundService // Need to consider Inbound and Outbound call var conversation = await conv.GetConversation(message.ConversationId); - var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId; + var agentId = message.AgentId; var result = await conv.SendMessage(agentId, inputMsg, @@ -105,7 +105,6 @@ public class TwilioMessageQueueService : BackgroundService ); reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp); reply.Hints = GetHints(reply); - reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); } @@ -138,9 +137,9 @@ public class TwilioMessageQueueService : BackgroundService private static async Task GetReplySpeechFileName(string conversationId, AssistantMessage reply, IServiceProvider sp) { - var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var completion = CompletionProvider.GetAudioSynthesizer(sp); var fileStorage = sp.GetRequiredService(); - var data = await completion.GenerateAudioFromTextAsync(reply.Content); + var data = await completion.GenerateAudioAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; fileStorage.SaveSpeechFile(conversationId, fileName, data); return fileName; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 082a0721..da209f31 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,4 +1,7 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Utilities; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -12,11 +15,13 @@ public class TwilioService { private readonly TwilioSetting _settings; private readonly IServiceProvider _services; + public readonly ILogger _logger; - public TwilioService(TwilioSetting settings, IServiceProvider services) + public TwilioService(TwilioSetting settings, IServiceProvider services, ILogger logger) { _settings = settings; _services = services; + _logger = logger; } public string GetAccessToken() @@ -47,29 +52,6 @@ public class TwilioService return token.ToJwt(); } - public VoiceResponse ReturnInstructions(string message) - { - var twilioSetting = _services.GetRequiredService(); - - var response = new VoiceResponse(); - var gather = new Gather() - { - Input = new List() - { - Gather.InputEnum.Speech, - Gather.InputEnum.Dtmf - }, - Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), - Enhanced = true, - SpeechModel = Gather.SpeechModelEnum.PhoneCall, - SpeechTimeout = "auto" - }; - - gather.Say(message); - response.Append(gather); - return response; - } - public VoiceResponse ReturnInstructions(ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); @@ -103,19 +85,13 @@ public class TwilioService public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse) { var response = new VoiceResponse(); - + var conversationId = voiceResponse.ConversationId; if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any()) { foreach (var speechPath in voiceResponse.SpeechPaths) { - if (speechPath.StartsWith(_settings.CallbackHost)) - { - response.Play(new Uri(speechPath)); - } - else - { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); - } + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); } } @@ -149,6 +125,22 @@ public class TwilioService return response; } + public VoiceResponse HangUp(ConversationalVoiceResponse voiceResponse) + { + var response = new VoiceResponse(); + var conversationId = voiceResponse.ConversationId; + if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in voiceResponse.SpeechPaths) + { + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); + } + } + response.Hangup(); + return response; + } + public VoiceResponse DialCsrAgent(string speechPath) { var response = new VoiceResponse(); @@ -160,6 +152,23 @@ public class TwilioService return response; } + public VoiceResponse TransferCall(ConversationalVoiceResponse conversationalVoiceResponse) + { + var response = new VoiceResponse(); + var conversationId = conversationalVoiceResponse.ConversationId; + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + { + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); + } + } + response.Dial(conversationalVoiceResponse.TransferTo, answerOnBridge: true); + + return response; + } + public VoiceResponse HoldOn(int interval, string message = null) { var twilioSetting = _services.GetRequiredService(); @@ -172,7 +181,7 @@ public class TwilioService Gather.InputEnum.Speech, Gather.InputEnum.Dtmf }, - Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), + Action = new Uri($"{_settings.CallbackHost}/twilio/voice/"), ActionOnEmptyResult = true }; @@ -190,26 +199,17 @@ public class TwilioService /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); + var conversationId = conversationalVoiceResponse.ConversationId; if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - if (speechPath.StartsWith("twilio/")) - { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); - } - else if (speechPath.StartsWith(_settings.CallbackHost)) - { - response.Play(new Uri(speechPath)); - } - else - { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}")); - } + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); } } @@ -220,4 +220,99 @@ public class TwilioService return response; } + + public async Task WaitingForAiResponse(ConversationalVoiceRequest request) + { + VoiceResponse response; + var sessionManager = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + + var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum); + if (indication != null) + { + _logger.LogWarning($"Indication ({request.SeqNum}): {indication}"); + var speechPaths = new List(); + foreach (var text in indication.Split('|')) + { + var seg = text.Trim(); + if (seg.StartsWith('#')) + { + speechPaths.Add($"twilio/{seg.Substring(1)}.mp3"); + } + else + { + var hash = Utilities.HashTextMd5(seg); + var fileName = $"indication_{hash}.mp3"; + + var existing = fileStorage.GetSpeechFile(request.ConversationId, fileName); + if (existing == BinaryData.Empty) + { + var completion = CompletionProvider.GetAudioSynthesizer(_services); + var data = await completion.GenerateAudioAsync(seg); + fileStorage.SaveSpeechFile(request.ConversationId, fileName, data); + } + + speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}"); + } + } + + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + SpeechPaths = speechPaths, + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", + ActionOnEmptyResult = true + }; + + response = ReturnInstructions(instruction); + + await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum); + } + else + { + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + SpeechPaths = [], + CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}", + ActionOnEmptyResult = true + }; + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnWaitingAgentResponse(request, instruction); + }); + + response = ReturnInstructions(instruction); + } + + return response; + } + + public string GetSpeechPath(string conversationId, string speechPath) + { + if (speechPath.StartsWith("twilio/")) + { + return $"{_settings.CallbackHost}/{speechPath}"; + } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + return speechPath; + } + else + { + return $"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"; + } + } + + public string GenerateStatesParameter(List states) + { + if (states is null || states.Count == 0) + { + return null; + } + return string.Join("&", states.Select(x => $"states={x}")); + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 2de42ec8..f9b27357 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -20,11 +20,6 @@ public class TwilioSetting public string? MessagingShortCode { get; set; } - /// - /// Default Agent Id to handle inbound phone call - /// - public string? AgentId { get; set; } - /// /// Human agent phone number if AI can't handle the call /// @@ -33,6 +28,7 @@ public class TwilioSetting public int MaxGatherAttempts { get; set; } = 4; public string? MachineDetection { get; set; } + public int MachineDetectionSilenceTimeout { get; set; } = 2500; public bool RecordingEnabled { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json index be5f3ca6..f3568661 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json @@ -9,11 +9,11 @@ "type": "string", "description": "The reason why user wants to end the phone call." }, - "anything_else_to_help": { - "type": "boolean", - "description": "Check if user has anything else to help." + "response_content": { + "type": "string", + "description": "A statement said to the user when politely ending a conversation." } }, - "required": [ "reason", "anything_else_to_help" ] + "required": [ "reason", "response_content" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-leave_voicemail.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-leave_voicemail.json new file mode 100644 index 00000000..53307a83 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-leave_voicemail.json @@ -0,0 +1,19 @@ +{ + "name": "util-twilio-leave_voicemail", + "description": "If the user wants you to leave a voicemail.", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "voicemail_message": { + "type": "string", + "description": "User voicemail with details." + }, + "phone_number": { + "type": "string", + "description": "Phone number to callback." + } + }, + "required": [ "voicemail_message", "phone_number" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json new file mode 100644 index 00000000..7a835386 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json @@ -0,0 +1,19 @@ +{ + "name": "util-twilio-transfer_phone_call", + "description": "When user wants to transfer the phone call", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "transition_message": { + "type": "string", + "description": "Transition message when forwarding." + }, + "phone_number": { + "type": "string", + "description": "Phone number transfer to." + } + }, + "required": [ "transition_message", "phone_number" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid deleted file mode 100644 index 5e89cbd2..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid +++ /dev/null @@ -1,3 +0,0 @@ -{% if channel == 'phone' %} -** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call. -{% endif %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid deleted file mode 100644 index d8fa9131..00000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid +++ /dev/null @@ -1 +0,0 @@ -** Please call util-twilio-outbound_phone_call if user wants to make an outbound call. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 283ad314..b698141d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,5 +1,6 @@ using Azure; using BotSharp.Abstraction.Browsing.Settings; +using Microsoft.Playwright; using System.IO; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -44,6 +45,7 @@ public class PlaywrightInstance : IDisposable public async Task InitContext(string ctxId, BrowserActionArgs args) { + var _webDriver = _services.GetRequiredService(); if (_contexts.ContainsKey(ctxId)) return _contexts[ctxId]; @@ -82,6 +84,8 @@ public class PlaywrightInstance : IDisposable // "--start-maximized" ] }); + _contexts[ctxId].SetDefaultTimeout(_webDriver.DefaultTimeout); + _contexts[ctxId].SetDefaultNavigationTimeout(_webDriver.DefaultNavigationTimeout); } _pages[ctxId] = new List(); @@ -131,7 +135,6 @@ public class PlaywrightInstance : IDisposable { return page; } - page.Request += async (sender, e) => { await HandleFetchRequest(e, message, args); @@ -154,7 +157,7 @@ public class PlaywrightInstance : IDisposable public async Task HandleFetchResponse(IResponse response, MessageInfo message, PageActionArgs args) { - if (response.Status != 204 && + if (response.Status != 204 && response.Status != 302 && response.Headers.ContainsKey("content-type") && (response.Request.ResourceType == "fetch" || response.Request.ResourceType == "xhr") && (args.ExcludeResponseUrls == null || !args.ExcludeResponseUrls.Any(url => response.Url.ToLower().Contains(url))) && @@ -164,11 +167,23 @@ public class PlaywrightInstance : IDisposable try { + var context = await GetContext(message.ContextId); + var cookies = await context.CookiesAsync(new string[] { response.Url }); var result = new WebPageResponseData { Url = response.Url.ToLower(), PostData = response.Request?.PostData ?? string.Empty, - ResponseInMemory = args.ResponseInMemory + ResponseInMemory = args.ResponseInMemory, + Method = response.Request.Method, + ResponseCode = response.Status, + Cookies = cookies.Select(x => new WebPageCookieData + { + Name = x.Name, + Value = x.Value, + Domain = x.Domain, + Path = x.Path, + Expires = x.Expires + }).ToList() }; var html = await response.TextAsync(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs index 139515cc..39be4f95 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -34,7 +34,7 @@ public class ChangeCheckboxFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index 9de29293..22c7127d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -34,7 +34,7 @@ public class ChangeListValueFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs index c8d88e48..9716b0f7 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -34,7 +34,7 @@ public class CheckRadioButtonFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index b0bf22aa..05436bfc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -34,7 +34,7 @@ public class ClickButtonFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs index 5388cb98..19f8e58d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -34,7 +34,7 @@ public class ClickElementFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index 6a390429..1e7afa41 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -28,7 +28,7 @@ public class ExtractDataFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index fe69f9d1..a9303526 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -31,7 +31,7 @@ public class GoToPageFn : IFunctionCallback var result = await _browser.GoToPage(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, new PageActionArgs { @@ -45,7 +45,7 @@ public class GoToPageFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs index fff2c240..9aea9670 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/HttpRequestFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Plugin.WebDriver.Services; + namespace BotSharp.Plugin.WebDriver.Functions; public class HttpRequestFn : IFunctionCallback @@ -20,12 +22,13 @@ public class HttpRequestFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); + var webDriverService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); var result = await _browser.SendHttpRequest(new MessageInfo { AgentId = agent.Id, MessageId = message.MessageId, - ContextId = convService.ConversationId + ContextId = webDriverService.GetMessageContext(message) }, args); message.Content = result.IsSuccess ? diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index d910519a..5085ae24 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -33,7 +33,7 @@ public class InputUserPasswordFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index dab4c36f..2312df4f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -39,7 +39,7 @@ public class InputUserTextFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 0beb80fc..631bd7a5 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -31,7 +31,7 @@ public class OpenBrowserFn : IFunctionCallback var msgInfo = new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }; var result = await _browser.LaunchBrowser(msgInfo, new BrowserActionArgs @@ -58,7 +58,7 @@ public class OpenBrowserFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs index 8fb4049d..5c58a69e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScreenshotFn.cs @@ -24,7 +24,7 @@ public class ScreenshotFn : IFunctionCallback message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); message.Content = "Took screenshot completed. You can take another screenshot if needed."; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs index 5907c0b4..b6ca139a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ScrollPageFn.cs @@ -20,12 +20,13 @@ public class ScrollPageFn : IFunctionCallback var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); + var webDriverService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); message.Data = await _browser.ScrollPage(new MessageInfo { AgentId = agent.Id, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, new PageActionArgs { @@ -35,13 +36,12 @@ public class ScrollPageFn : IFunctionCallback message.Content = "Scrolled. You can scroll more if needed."; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await _browser.ScreenshotAsync(new MessageInfo { AgentId = message.CurrentAgentId, - ContextId = convService.ConversationId, + ContextId = webDriverService.GetMessageContext(message), MessageId = message.MessageId }, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.GetMessageContext.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.GetMessageContext.cs new file mode 100644 index 00000000..a316999f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.GetMessageContext.cs @@ -0,0 +1,20 @@ +using BotSharp.Abstraction.Infrastructures.Enums; + +namespace BotSharp.Plugin.WebDriver.Services +{ + public partial class WebDriverService + { + public string GetMessageContext(RoleDialogModel message) + { + var states = _services.GetService(); + var convService = _services.GetRequiredService(); + var webDriverTaskId = states.GetState(StateConst.WEB_DRIVER_TASK_ID, ""); + var contextId = message.CurrentAgentId; + if (!string.IsNullOrWhiteSpace(webDriverTaskId)) + { + contextId = webDriverTaskId; + } + return contextId; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs index 6b328b2c..8cb9119a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs @@ -35,17 +35,17 @@ public class UtilWebActionOnElementFn : IFunctionCallback var conv = _services.GetRequiredService(); var browser = _services.GetRequiredService(); + var webDriverService = _services.GetRequiredService(); var msg = new MessageInfo { AgentId = message.CurrentAgentId, MessageId = message.MessageId, - ContextId = message.CurrentAgentId, + ContextId = webDriverService.GetMessageContext(message), }; var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs); message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}"; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await browser.ScreenshotAsync(msg, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebCloseBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebCloseBrowserFn.cs index 0d836b73..36badff3 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebCloseBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebCloseBrowserFn.cs @@ -18,20 +18,19 @@ public class UtilWebCloseBrowserFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { var conv = _services.GetRequiredService(); - + var webDriverService = _services.GetRequiredService(); var browser = _services.GetRequiredService(); var msg = new MessageInfo { AgentId = message.CurrentAgentId, MessageId = message.MessageId, - ContextId = message.CurrentAgentId, + ContextId = webDriverService.GetMessageContext(message) }; await browser.CloseBrowser(message.CurrentAgentId); message.Content = $"Browser closed."; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await browser.ScreenshotAsync(msg, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs index 7a5d8f89..0c22d916 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs @@ -29,13 +29,13 @@ public class UtilWebGoToPageFn : IFunctionCallback args.WaitTime = _webDriver.DefaultWaitTime; var conv = _services.GetRequiredService(); - + var webDriverService = _services.GetRequiredService(); var browser = _services.GetRequiredService(); var msg = new MessageInfo { AgentId = message.CurrentAgentId, MessageId = message.MessageId, - ContextId = message.CurrentAgentId, + ContextId = webDriverService.GetMessageContext(message) }; if (!args.KeepBrowserOpen) { @@ -50,7 +50,6 @@ public class UtilWebGoToPageFn : IFunctionCallback message.Content = $"Open web page successfully."; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await browser.ScreenshotAsync(msg, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebLocateElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebLocateElementFn.cs index 8267f4ab..672ad6ad 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebLocateElementFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebLocateElementFn.cs @@ -22,17 +22,17 @@ public class UtilWebLocateElementFn : IFunctionCallback locatorArgs.Highlight = true; var browser = _services.GetRequiredService(); + var webDriverService = _services.GetRequiredService(); var msg = new MessageInfo { AgentId = message.CurrentAgentId, MessageId = message.MessageId, - ContextId = message.CurrentAgentId, + ContextId = webDriverService.GetMessageContext(message) }; var result = await browser.LocateElement(msg, locatorArgs); message.Content = $"Locating element {(result.IsSuccess ? "success" : "failed")}"; - var webDriverService = _services.GetRequiredService(); var path = webDriverService.GetScreenshotFilePath(message.MessageId); message.Data = await browser.ScreenshotAsync(msg, path); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs index 152ca030..2f4f4f08 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs @@ -17,13 +17,7 @@ public class WebDriverPlugin : IBotSharpPlugin { var settings = new WebBrowsingSettings(); config.Bind("WebBrowsing", settings); - - services.AddScoped(provider => - { - var settingService = provider.GetRequiredService(); - return settings; - }); - + services.AddSingleton(x => settings); services.AddScoped(); services.AddSingleton(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json index 4f6b21fe..fc6832e0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json @@ -15,6 +15,10 @@ "open_blank_page": { "type": "boolean", "description": "Open blank page" + }, + "enable_response_callback": { + "type": "boolean", + "description": "Enable response callback" } }, "required": [ "url" ] diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index d06fa30a..9cff5c73 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -11,6 +11,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 2582a8ae..3c2cd567 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -238,7 +238,10 @@ }, "Instruction": { - "EnableLog": true + "Logging": { + "Enabled": true, + "ExcludedAgentIds": [] + } }, "ChatHub": { diff --git a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj index 716f1f4d..91812c25 100644 --- a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj +++ b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj @@ -10,7 +10,11 @@ - + + + + + @@ -97,8 +101,4 @@ - - - -