From 7571512e0d67ee9e8680a92af83b0e0192576998 Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Mon, 10 Mar 2025 15:52:43 -0500 Subject: [PATCH 1/7] Update headless setting for Playwright --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 6 +++++- .../BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 4c65aa32..9f830f2e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -33,9 +33,13 @@ public class PlaywrightInstance : IDisposable public async Task GetContext(string ctxId) { + var headless = true; +#if DEBUG + headless = false; +#endif if (!_contexts.ContainsKey(ctxId)) { - await InitContext(ctxId, new BrowserActionArgs()); + await InitContext(ctxId, new BrowserActionArgs() { Headless = headless }); } return _contexts[ctxId]; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 6a5a1ac1..ed1c7398 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -29,9 +29,13 @@ public class OpenBrowserFn : IFunctionCallback ContextId = convService.ConversationId, MessageId = message.MessageId }; + var headless = true; +#if DEBUG + headless = false; +#endif var result = await _browser.LaunchBrowser(msgInfo, new BrowserActionArgs { - Headless = false + Headless = headless }); result = await _browser.GoToPage(msgInfo, new PageActionArgs { From b8083b2f3893bf86ae01e588a2b6e6edd8a822f3 Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Mon, 10 Mar 2025 16:54:51 -0500 Subject: [PATCH 2/7] Fetch WebDrive headless setting from config --- .../Browsing/Settings/WebBrowsingSettings.cs | 1 + .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 8 +++----- .../Functions/OpenBrowserFn.cs | 13 +++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs index 25149fe0..e1ae9a79 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs @@ -3,4 +3,5 @@ namespace BotSharp.Abstraction.Browsing.Settings; public class WebBrowsingSettings { public string Driver { get; set; } = "Playwright"; + public bool Headless { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 9f830f2e..755d4faf 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,4 +1,5 @@ using Azure; +using BotSharp.Abstraction.Browsing.Settings; using System.IO; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; @@ -33,13 +34,10 @@ public class PlaywrightInstance : IDisposable public async Task GetContext(string ctxId) { - var headless = true; -#if DEBUG - headless = false; -#endif + var _webDriver = _services.GetRequiredService(); if (!_contexts.ContainsKey(ctxId)) { - await InitContext(ctxId, new BrowserActionArgs() { Headless = headless }); + await InitContext(ctxId, new BrowserActionArgs() { Headless = _webDriver.Headless }); } return _contexts[ctxId]; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index ed1c7398..92935233 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Browsing.Settings; + namespace BotSharp.Plugin.WebDriver.Functions; public class OpenBrowserFn : IFunctionCallback @@ -6,12 +8,15 @@ public class OpenBrowserFn : IFunctionCallback private readonly IServiceProvider _services; private readonly IWebBrowser _browser; + private readonly WebBrowsingSettings _webBrowsingSettings; public OpenBrowserFn(IServiceProvider services, - IWebBrowser browser) + IWebBrowser browser, + WebBrowsingSettings webBrowsingSettings) { _services = services; _browser = browser; + _webBrowsingSettings = webBrowsingSettings; } public async Task Execute(RoleDialogModel message) @@ -29,13 +34,9 @@ public class OpenBrowserFn : IFunctionCallback ContextId = convService.ConversationId, MessageId = message.MessageId }; - var headless = true; -#if DEBUG - headless = false; -#endif var result = await _browser.LaunchBrowser(msgInfo, new BrowserActionArgs { - Headless = headless + Headless = _webBrowsingSettings.Headless }); result = await _browser.GoToPage(msgInfo, new PageActionArgs { From 25a9c77cddcd672ebca9640285d187544b75ac43 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 10 Mar 2025 17:03:23 -0500 Subject: [PATCH 3/7] refine agent task --- .../Tasks/Models/AgentTask.cs | 17 +++++++++++++---- .../Services/CrontabService.cs | 2 +- .../Agents/Services/AgentService.CreateAgent.cs | 4 ++-- .../FileRepository/FileRepository.AgentTask.cs | 8 ++++---- .../Tasks/Services/AgentTaskService.cs | 2 +- .../ViewModels/Agents/AgentTaskViewModel.cs | 12 ++++++------ .../Collections/AgentTaskDocument.cs | 8 ++++---- 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs index e4852307..f5e379c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs @@ -2,10 +2,13 @@ namespace BotSharp.Abstraction.Tasks.Models; public class AgentTask { + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string Id { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; public string? Description { get; set; } public bool Enabled { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string Content { get; set; } = string.Empty; [JsonIgnore(Condition = JsonIgnoreCondition.Always)] @@ -19,9 +22,15 @@ public class AgentTask /// public string Status { get; set; } = TaskStatus.New; - public DateTime? LastExecutedDateTime { get; set; } - public DateTime? NextExecutionDateTime { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTime? LastExecutionTime { get; set; } - public DateTime CreatedDateTime { get; set; } - public DateTime UpdatedDateTime { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTime? NextExecutionTime { get; set; } + + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } + + [JsonPropertyName("updated_time")] + public DateTime UpdatedTime { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 5cbf9028..cf4e1846 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -97,7 +97,7 @@ public class CrontabService : ICrontabService, ITaskFeeder Status = TaskStatus.Scheduled, Enabled = !x.Disabled, Description = cron.Description, - LastExecutedDateTime = cron.LastExecutionTime + LastExecutionTime = cron.LastExecutionTime })); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index d4ff408b..5becbec6 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -185,8 +185,8 @@ public partial class AgentService Enabled = parsedTask.Enabled, Content = parsedTask.Content, AgentId = agentId, - CreatedDateTime = parsedTask.CreatedDateTime, - UpdatedDateTime = parsedTask.UpdatedDateTime + CreatedTime = parsedTask.CreatedTime, + UpdatedTime = parsedTask.UpdatedTime }; tasks.Add(task); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs index 6d47c4cc..016e0c36 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -129,8 +129,8 @@ public partial class FileRepository Name = task.Name, Description = task.Description, Enabled = task.Enabled, - CreatedDateTime = DateTime.UtcNow, - UpdatedDateTime = DateTime.UtcNow + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow }; var fileContent = BuildAgentTaskFileContent(metaData, task.Content); @@ -163,8 +163,8 @@ public partial class FileRepository Name = parsedTask.Name, Description = parsedTask.Description, Enabled = parsedTask.Enabled, - CreatedDateTime = parsedTask.CreatedDateTime, - UpdatedDateTime = DateTime.UtcNow + CreatedTime = parsedTask.CreatedTime, + UpdatedTime = DateTime.UtcNow }; var content = parsedTask.Content; diff --git a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs index 5ad0637e..ce5d72b8 100644 --- a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs +++ b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs @@ -31,7 +31,7 @@ public class AgentTaskService : IAgentTaskService return new PagedItems { - Items = items.OrderByDescending(x => x.UpdatedDateTime) + Items = items.OrderByDescending(x => x.UpdatedTime) .Skip(filter.Pager.Offset).Take(filter.Pager.Size), Count = items.Count() }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs index cf051544..d46d134e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs @@ -13,11 +13,11 @@ public class AgentTaskViewModel public string Status { get; set; } = null!; - [JsonPropertyName("created_datetime")] - public DateTime CreatedDateTime { get; set; } + [JsonPropertyName("created_time")] + public DateTime CreatedTime { get; set; } - [JsonPropertyName("updated_datetime")] - public DateTime UpdatedDateTime { get; set; } + [JsonPropertyName("updated_time")] + public DateTime UpdatedTime { get; set; } [JsonPropertyName("agent_id")] public string AgentId { get; set; } = null!; @@ -37,8 +37,8 @@ public class AgentTaskViewModel AgentId = task.AgentId, AgentName = task.Agent?.Name, Status = task.Status, - CreatedDateTime = task.CreatedDateTime, - UpdatedDateTime = task.UpdatedDateTime + CreatedTime = task.CreatedTime, + UpdatedTime = task.UpdatedTime }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs index 7f09318a..8cade792 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs @@ -23,8 +23,8 @@ public class AgentTaskDocument : MongoBase Enabled = model.Enabled, AgentId = model.AgentId, Status = model.Status, - CreatedDateTime = model.CreatedTime, - UpdatedDateTime = model.UpdatedTime + CreatedTime = model.CreatedTime, + UpdatedTime = model.UpdatedTime }; } @@ -38,8 +38,8 @@ public class AgentTaskDocument : MongoBase Enabled = model.Enabled, AgentId = model.AgentId, Status = model.Status, - CreatedTime = model.CreatedDateTime, - UpdatedTime = model.UpdatedDateTime + CreatedTime = model.CreatedTime, + UpdatedTime = model.UpdatedTime }; } } From 0d8e9903cb419e8657f74c23be12d97baf0015df Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 10 Mar 2025 21:56:18 -0500 Subject: [PATCH 4/7] Increase ReceiveMessage buffer to 32K. --- .../Providers/Realtime/RealTimeCompletionProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 1d44bc8d..0d6f76b4 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -131,7 +131,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Action onInputAudioTranscriptionCompleted, Action onUserInterrupted) { - var buffer = new byte[1024 * 16]; + var buffer = new byte[1024 * 32]; WebSocketReceiveResult result; do From db3ed518751d8e3758057853f4e8eaad43ec0b30 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 11 Mar 2025 10:18:54 -0500 Subject: [PATCH 5/7] get template name from state --- .../BotSharp.Logger/Hooks/InstructionLogHook.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs index 0abea5eb..30e1d066 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs @@ -32,6 +32,8 @@ public class InstructionLogHook : InstructHookBase var state = _services.GetRequiredService(); var user = db.GetUserById(_user.Id); + var templateName = response.TemplateName ?? state.GetState("instruct_template_name") ?? null; + db.SaveInstructionLogs(new List { new InstructionLogModel @@ -39,7 +41,7 @@ public class InstructionLogHook : InstructHookBase AgentId = response.AgentId, Provider = response.Provider, Model = response.Model, - TemplateName = response.TemplateName, + TemplateName = !string.IsNullOrWhiteSpace(templateName) ? templateName : null, UserMessage = response.UserMessage, SystemInstruction = response.SystemInstruction, CompletionText = response.CompletionText, From 23baf7008868c9118a50be4e95f8256fafc1a0ed Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 11 Mar 2025 16:48:44 -0400 Subject: [PATCH 6/7] Update to M.E.AI 9.3.0-preview.1.25161.3 --- Directory.Packages.props | 2 +- .../MicrosoftExtensionsAIChatCompletionProvider.cs | 11 +++++------ .../MicrosoftExtensionsAITextCompletionProvider.cs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 4683fc9d..aa76cf61 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -54,7 +54,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 37d3877c..6ee77772 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -27,7 +27,6 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio private readonly IChatClient _client; private readonly ILogger _logger; private readonly IServiceProvider _services; - private List renderedInstructions = []; private string? _model; /// @@ -46,7 +45,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio /// public string Provider => "microsoft.extensions.ai"; - public string Model => _model; + public string Model => _model ?? ""; /// public void SetModelName(string model) => _model = model; @@ -56,7 +55,7 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio { // Before chat completion hook var hooks = _services.GetServices().ToArray(); - renderedInstructions = []; + List renderedInstructions = []; await Task.WhenAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations))); // Configure options @@ -145,13 +144,13 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio var completion = await _client.GetResponseAsync(messages); - RoleDialogModel result = new(AgentRole.Assistant, string.Concat(completion.Message.Contents.OfType())) + RoleDialogModel result = new(AgentRole.Assistant, completion.Text) { CurrentAgentId = agent.Id, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + //RenderedInstruction = renderedInstructions, }; - if (completion.Message.Contents.OfType().FirstOrDefault() is { } fcc) + if (completion.Messages.SelectMany(m => m.Contents).OfType().FirstOrDefault() is { } fcc) { result.Role = AgentRole.Function; result.MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs index ed0e94d9..ef3b35c7 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAITextCompletionProvider.cs @@ -51,7 +51,7 @@ public sealed class MicrosoftExtensionsAITextCompletionProvider : ITextCompletio _tokenStatistics.StartTimer(); var completion = await _chatClient.GetResponseAsync(text); - var result = string.Concat(completion.Message.Contents.OfType()); + var result = completion.Text; _tokenStatistics.StopTimer(); // After chat completion hook From 9fe9b123f13039b9a15e6f96fc0e8c061bcba20d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 11 Mar 2025 20:57:49 -0500 Subject: [PATCH 7/7] HandleUserDtmfReceived --- .../BotSharp.Core.Realtime/Services/RealtimeHub.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 68636bd1..e8548c4d 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -233,7 +233,8 @@ public class RealtimeHub : IRealtimeHub } await _completer.InsertConversationItem(message); - await _completer.TriggerModelInference("Reply based on the user input"); + var instruction = await _completer.UpdateSession(_conn); + await _completer.TriggerModelInference($"{instruction}\r\n\r\nReply based on the user input: {message.Content}"); } private async Task HandleUserDisconnected()