From 7b75f0c89f28c1e3c84d7b16fa123e1bc8c849c7 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Mon, 26 May 2025 11:47:09 +0200 Subject: [PATCH] Implement retry attempt capturing (#6674) * Update default initializations and input parameters to `null` Replaced `default!` with explicit `null` for input parameters and properties throughout various classes. Adjusted constructors' default values for consistency and readability. This change ensures better clarity and alignment with nullable reference types. * Add Polly-based resilience integration for retry tracking Introduce Polly diagnostics to log retry events in the execution context. Updated resilience strategy interfaces and implementations to support Polly's context and retry event tracking. * Refactor resilience and retry handling, add flaky endpoint. Removed custom Polly-based diagnostic listeners and observers in favor of a transient status code utility class. Introduced a mock "flaky" endpoint for testing failure scenarios and updated configuration for resilience strategies. Minor namespace fixes * Add retry attempt recording to resilience feature Introduce `IRetryAttemptRecorder` and its implementations to enable recording of retry attempts during activity execution. Updated `ResilientActivityInvoker` to persist retry attempts and modified `ResilienceFeature` to support configurable retry attempt recorders. * Add retry attempt tracking and retrieval functionality Introduced mechanisms to track and fetch retry attempts, including new interfaces, reader implementations, API endpoints, and related models. These enhancements improve resilience tracking and data access for activity execution across workflows. * Add GetOutcome method to RetryAttempt model Introduce a GetOutcome method to encapsulate logic for determining the retry attempt's outcome. It prioritizes the Result, falls back to the Exception message, or defaults to "Unknown" if neither is available. This improves clarity and reusability of the outcome evaluation. * Add scoped registration for _retryAttemptReader This change ensures that _retryAttemptReader is registered in the DI container as a scoped service. * Refactor retry mechanism to support detailed retry metadata Introduced a `CollectRetryDetails` method to `IResilientActivity` for enhanced retry data collection. Updated `RetryAttemptRecord` to include a `Details` dictionary for capturing metadata, replacing previous `Result` and `Exception` fields. These changes simplify the retry recording process and improve extensibility for tracking retry details across activities. * Add support for capturing background activity properties Introduced functionality to capture and persist background activity properties during workflow execution. This includes defining a key for properties, capturing them in middleware, and storing them in the workflow execution context. These changes ensure properties are handled consistently alongside other activity data. * Add support for storing and propagating activity execution properties Introduced a `Properties` dictionary to track additional metadata in activity execution records and stats, enabling richer diagnostics and tracing. Refactored resilience logic to improve retry handling and propagate retry-related flags in workflows. Enhanced database queries to map serialized properties for execution summaries. * Add retry propagation for background activity execution Introduced a mechanism to propagate the retry-attempted flag across activity execution contexts. Added a new notification `BackgroundActivityExecutionCompleted` and updated related middleware to send this notification. Enhanced resilience features to handle and propagate retry state effectively. * Refactor default parameters and values to use 'null'. Replaced 'default' with 'null' for optional parameters and values in `AddExecutionLogEntry`, improving clarity and ensuring semantic consistency with nullable types. No functional changes were introduced. * Refactor flaky endpoint and enhance resilience support. Replaced the "Flaky" endpoint with a more robust "SimulateResponseEndpoint" under a new module. Introduced a status code lookup utility and improved resilience strategies with configurable backoff types. Updated serialization to support enum conversions and enhanced caching behavior for response simulation. * Update activity execution models with nullable properties Replaced `default!` initializations with `null!` to ensure correct handling of nullable string properties in `ActivityExecutionRecord`. Added a new `Properties` dictionary to `ActivityExecutionRecordSummary` to store additional activity execution data. This enhances model flexibility and data extensibility. * Add support for recording resilience strategy in context Introduced a new method to store resilience strategy details in the activity execution context for enhanced diagnostics. Updated `ResilientActivityInvoker` to serialize and set the resilience strategy using this method, leveraging `JsonSerializer`. * Remove redundant PropertyNamingPolicy assignment The PropertyNamingPolicy was set to the default value (CamelCase), making the assignment unnecessary. This change simplifies the code while maintaining existing functionality. * Set JSON property naming policy to camelCase Updated JSON serialization settings to use camelCase naming for property names. This improves consistency with standard JSON naming conventions and ensures compatibility with camelCase-based APIs. * Remove unused Endpoints folder reference from project file The Endpoints folder reference in the project file was unnecessary and has been removed. This cleanup helps maintain a tidy and accurate project structure. * Remove unused RetryAttemptFilter and add Polly packages Removed the obsolete RetryAttemptFilter class as it was no longer in use. Added Polly and Polly.Extensions packages to the project to support resilience and fault-handling strategies. This update aligns with keeping dependencies relevant and reducing unused artifacts. * Add resilience integration test for FlowSendHttpRequest (#6692) * Refactor and fix resilience test cases for clarity and accuracy Simplified imports, adjusted code structure, and corrected attempt indexing logic in resilience tests. These changes improve readability, maintainability, and ensure accurate validation of retry attempts in test scenarios. --- Directory.Packages.props | 2 + Elsa.sln | 25 +++-- src/apps/Elsa.Server.Web/appsettings.json | 12 +-- .../Elsa.Api.Client/Elsa.Api.Client.csproj | 4 - .../Extensions/ActivityExtensions.cs | 2 +- .../DependencyInjectionExtensions.cs | 3 +- .../Models/ActivityExecutionRecord.cs | 10 +- .../Models/ActivityExecutionRecordSummary.cs | 8 +- .../Models/ActivityExecutionStats.cs | 2 +- .../Contracts/IIncidentStrategiesApi.cs | 2 +- .../Resilience/Contracts/IRetryAttemptsApi.cs | 11 +++ .../Models/ResilienceStrategyConfig.cs | 2 +- .../Models/ResilienceStrategyConfigMode.cs | 2 +- .../Resilience/Models/RetryAttempt.cs | 13 +++ .../Contracts/IWorkflowInstancesApi.cs | 4 +- .../Models/ExceptionState.cs | 2 +- src/modules/Elsa.Common/Models/Page.cs | 1 + .../Runtime/ActivityExecutionLogStore.cs | 67 ++++++++++++- .../Activities/SendHttpRequestBase.cs | 16 ++++ .../Extensions/HttpStatusCodeExtensions.cs | 21 +++++ .../Middleware/HttpWorkflowsMiddleware.cs | 2 - .../Resilience/HttpResilienceStrategy.cs | 35 ++++--- .../JavaScript/HttpJavaScriptHandler.cs | 1 - .../Services/DefaultRouteTableUpdater.cs | 1 - .../Elsa.Http/Services/RouteMatcher.cs | 1 - .../Contracts/IResilienceStrategy.cs | 6 +- .../Contracts/IResilientActivity.cs | 6 +- .../Contracts/IRetryAttemptReader.cs | 9 ++ .../Contracts/IRetryAttemptRecorder.cs | 8 ++ .../Elsa.Resilience.Core.csproj | 5 + .../Entities/RetryAttemptRecord.cs | 13 +++ .../Extensions/ActivityExecutionExtensions.cs | 31 ++++++ .../Models/RecordRetryAttemptsContext.cs | 6 ++ .../Models/RetryAttempt.cs | 8 ++ .../ResilienceStrategySerializer.cs | 4 +- .../Services/ResilientActivityInvoker.cs | 94 ++++++++++++++++++- .../Services/RetryTelemetryListener.cs | 27 ++++++ .../Services/VoidRetryAttemptReader.cs | 14 +++ .../Services/VoidRetryAttemptRecorder.cs | 14 +++ .../Elsa.Resilience/Elsa.Resilience.csproj | 5 + .../ResilienceStrategies/List/Endpoint.cs | 2 +- .../Endpoints/Retries/List/Endpoint.cs | 34 +++++++ .../Endpoints/SimulateResponse/Endpoint.cs | 55 +++++++++++ .../Endpoints/SimulateResponse/Models.cs | 3 + .../StatusCodeMessageLookup.cs | 70 ++++++++++++++ .../Features/ResilienceFeature.cs | 40 +++++++- .../Handlers/PropagateRetryAttemptedFlag.cs | 20 ++++ ...ivityExecutionContextRetryAttemptReader.cs | 22 +++++ ...ityExecutionContextRetryAttemptRecorder.cs | 14 +++ .../ActivityExecutions/Report/Endpoint.cs | 2 +- ...kflowExecutionContext.ExecutionLogEntry.cs | 6 +- .../ActivityExecutionRecordSummary.cs | 7 ++ .../BackgroundActivityInvokerMiddleware.cs | 23 ++++- .../Models/ActivityExecutionStats.cs | 5 + .../BackgroundActivityExecutionCompleted.cs | 5 + .../Services/ActivityExecutionService.cs | 30 +++--- .../Services/BackgroundActivityInvoker.cs | 12 ++- .../Elsa.Resilience.IntegrationTests.csproj | 15 +++ .../FlowSendHttpRequestResilienceTests.cs | 81 ++++++++++++++++ .../SequentialStatusHandler.cs | 19 ++++ .../TestRetryAttemptRecorder.cs | 16 ++++ .../Usings.cs | 1 + 62 files changed, 897 insertions(+), 84 deletions(-) rename src/clients/Elsa.Api.Client/Resources/{ResilienceStrategies => Resilience}/Contracts/IIncidentStrategiesApi.cs (92%) create mode 100644 src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IRetryAttemptsApi.cs rename src/clients/Elsa.Api.Client/Resources/{ResilienceStrategies => Resilience}/Models/ResilienceStrategyConfig.cs (91%) rename src/clients/Elsa.Api.Client/Resources/{ResilienceStrategies => Resilience}/Models/ResilienceStrategyConfigMode.cs (54%) create mode 100644 src/clients/Elsa.Api.Client/Resources/Resilience/Models/RetryAttempt.cs create mode 100644 src/modules/Elsa.Http/Extensions/HttpStatusCodeExtensions.cs create mode 100644 src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptReader.cs create mode 100644 src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptRecorder.cs create mode 100644 src/modules/Elsa.Resilience.Core/Entities/RetryAttemptRecord.cs create mode 100644 src/modules/Elsa.Resilience.Core/Extensions/ActivityExecutionExtensions.cs create mode 100644 src/modules/Elsa.Resilience.Core/Models/RecordRetryAttemptsContext.cs create mode 100644 src/modules/Elsa.Resilience.Core/Models/RetryAttempt.cs create mode 100644 src/modules/Elsa.Resilience.Core/Services/RetryTelemetryListener.cs create mode 100644 src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptReader.cs create mode 100644 src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptRecorder.cs create mode 100644 src/modules/Elsa.Resilience/Endpoints/Retries/List/Endpoint.cs create mode 100644 src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Endpoint.cs create mode 100644 src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Models.cs create mode 100644 src/modules/Elsa.Resilience/Endpoints/SimulateResponse/StatusCodeMessageLookup.cs create mode 100644 src/modules/Elsa.Resilience/Handlers/PropagateRetryAttemptedFlag.cs create mode 100644 src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptReader.cs create mode 100644 src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptRecorder.cs create mode 100644 src/modules/Elsa.Workflows.Runtime/Notifications/BackgroundActivityExecutionCompleted.cs create mode 100644 test/integration/Elsa.Resilience.IntegrationTests/Elsa.Resilience.IntegrationTests.csproj create mode 100644 test/integration/Elsa.Resilience.IntegrationTests/FlowSendHttpRequestResilienceTests.cs create mode 100644 test/integration/Elsa.Resilience.IntegrationTests/SequentialStatusHandler.cs create mode 100644 test/integration/Elsa.Resilience.IntegrationTests/TestRetryAttemptRecorder.cs create mode 100644 test/integration/Elsa.Resilience.IntegrationTests/Usings.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index c277948ca..fd43a70aa 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -86,6 +86,8 @@ + + diff --git a/Elsa.sln b/Elsa.sln index 98441e50b..373517434 100644 --- a/Elsa.sln +++ b/Elsa.sln @@ -241,9 +241,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Activities.Integration EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.JavaScript.IntegrationTests", "test\integration\Elsa.JavaScript.IntegrationTests\Elsa.JavaScript.IntegrationTests.csproj", "{4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Resilience.IntegrationTests", "test\integration\Elsa.Resilience.IntegrationTests\Elsa.Resilience.IntegrationTests.csproj", "{832675FA-C597-4554-AE6B-18F189198A1F}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "apps", "apps", "{D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1}" - ProjectSection(SolutionItems) = preProject - src\apps\Directory.Build.props = src\apps\Directory.Build.props + ProjectSection(SolutionItems) = preProject + src\apps\Directory.Build.props = src\apps\Directory.Build.props EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Server.Web", "src\apps\Elsa.Server.Web\Elsa.Server.Web.csproj", "{5ADDDFB1-E59B-4097-97B7-8C24E2D60463}" @@ -696,10 +698,14 @@ Global {99B171E6-0248-4402-836D-98947CD63772}.Release|Any CPU.ActiveCfg = Release|Any CPU {99B171E6-0248-4402-836D-98947CD63772}.Release|Any CPU.Build.0 = Release|Any CPU {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Release|Any CPU.Build.0 = Release|Any CPU - {5ADDDFB1-E59B-4097-97B7-8C24E2D60463}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D}.Release|Any CPU.Build.0 = Release|Any CPU + {832675FA-C597-4554-AE6B-18F189198A1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {832675FA-C597-4554-AE6B-18F189198A1F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {832675FA-C597-4554-AE6B-18F189198A1F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {832675FA-C597-4554-AE6B-18F189198A1F}.Release|Any CPU.Build.0 = Release|Any CPU + {5ADDDFB1-E59B-4097-97B7-8C24E2D60463}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5ADDDFB1-E59B-4097-97B7-8C24E2D60463}.Debug|Any CPU.Build.0 = Debug|Any CPU {5ADDDFB1-E59B-4097-97B7-8C24E2D60463}.Release|Any CPU.ActiveCfg = Release|Any CPU {5ADDDFB1-E59B-4097-97B7-8C24E2D60463}.Release|Any CPU.Build.0 = Release|Any CPU @@ -929,9 +935,10 @@ Global {94A61AD7-2A2B-40DB-81F3-C59D596958A8} = {C6658DE0-2B2F-47F0-BB61-2CA66D435C09} {4B598AF7-BD7D-4544-A274-2CDDD98F4167} = {C6658DE0-2B2F-47F0-BB61-2CA66D435C09} {99B171E6-0248-4402-836D-98947CD63772} = {1B8D5897-902E-4632-8698-E89CAF3DDF54} - {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D} = {1B8D5897-902E-4632-8698-E89CAF3DDF54} - {D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1} = {61017E64-6D00-49CB-9E81-5002DC8F7D5F} - {5ADDDFB1-E59B-4097-97B7-8C24E2D60463} = {D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1} + {4332A6BC-434A-4AF5-A075-F1BBCDD28F5D} = {1B8D5897-902E-4632-8698-E89CAF3DDF54} + {832675FA-C597-4554-AE6B-18F189198A1F} = {1B8D5897-902E-4632-8698-E89CAF3DDF54} + {D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1} = {61017E64-6D00-49CB-9E81-5002DC8F7D5F} + {5ADDDFB1-E59B-4097-97B7-8C24E2D60463} = {D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1} {97C7E531-9D5F-43FD-AA19-BF24DA13B612} = {5BA4A8FA-F7F4-45B3-AEC8-8886D35AAC79} {65F2AD97-3ECF-4BA6-8AAA-1E5882FDCE68} = {D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1} {690B0274-291F-4D9E-BA76-54EFF7D3E4BC} = {D92BEAB2-60D6-4BB4-885A-6BA681C6CCF1} diff --git a/src/apps/Elsa.Server.Web/appsettings.json b/src/apps/Elsa.Server.Web/appsettings.json index 306c906a5..b239fcece 100644 --- a/src/apps/Elsa.Server.Web/appsettings.json +++ b/src/apps/Elsa.Server.Web/appsettings.json @@ -277,16 +277,16 @@ { "$type": "HttpResilienceStrategy", "Id": "1", - "DisplayName": "Retry 3 times", - "RetryCount": 3, - "BackoffFactor": 2.0 + "DisplayName": "Retry 3 times, exponential", + "MaxRetryAttempts": 3, + "BackoffType": "Exponential" }, { "$type": "HttpResilienceStrategy", "Id": "2", - "DisplayName": "Retry 10 times", - "RetryCount": 10, - "BackoffFactor": 2.0 + "DisplayName": "Retry 10 times, linear", + "MaxRetryAttempts": 10, + "BackoffType": "Linear" } ] } diff --git a/src/clients/Elsa.Api.Client/Elsa.Api.Client.csproj b/src/clients/Elsa.Api.Client/Elsa.Api.Client.csproj index ef83a5aee..f8246f150 100644 --- a/src/clients/Elsa.Api.Client/Elsa.Api.Client.csproj +++ b/src/clients/Elsa.Api.Client/Elsa.Api.Client.csproj @@ -17,8 +17,4 @@ - - - - diff --git a/src/clients/Elsa.Api.Client/Extensions/ActivityExtensions.cs b/src/clients/Elsa.Api.Client/Extensions/ActivityExtensions.cs index c16488325..f34b3a67b 100644 --- a/src/clients/Elsa.Api.Client/Extensions/ActivityExtensions.cs +++ b/src/clients/Elsa.Api.Client/Extensions/ActivityExtensions.cs @@ -1,5 +1,5 @@ using System.Text.Json.Nodes; -using Elsa.Api.Client.Resources.ResilienceStrategies.Models; +using Elsa.Api.Client.Resources.Resilience.Models; using Elsa.Api.Client.Resources.WorkflowDefinitions.Models; using Elsa.Api.Client.Shared.Models; diff --git a/src/clients/Elsa.Api.Client/Extensions/DependencyInjectionExtensions.cs b/src/clients/Elsa.Api.Client/Extensions/DependencyInjectionExtensions.cs index 3446d0637..4bc70d80c 100644 --- a/src/clients/Elsa.Api.Client/Extensions/DependencyInjectionExtensions.cs +++ b/src/clients/Elsa.Api.Client/Extensions/DependencyInjectionExtensions.cs @@ -8,7 +8,7 @@ using Elsa.Api.Client.Resources.Features.Contracts; using Elsa.Api.Client.Resources.Identity.Contracts; using Elsa.Api.Client.Resources.IncidentStrategies.Contracts; using Elsa.Api.Client.Resources.LogPersistenceStrategies; -using Elsa.Api.Client.Resources.ResilienceStrategies.Contracts; +using Elsa.Api.Client.Resources.Resilience.Contracts; using Elsa.Api.Client.Resources.Scripting.Contracts; using Elsa.Api.Client.Resources.StorageDrivers.Contracts; using Elsa.Api.Client.Resources.Tasks.Contracts; @@ -78,6 +78,7 @@ public static class DependencyInjectionExtensions services.AddApi(builderOptions); services.AddApi(builderOptions); services.AddApi(builderOptions); + services.AddApi(builderOptions); services.AddApi(builderOptions); services.AddApi(builderOptions); services.AddApi(builderOptions); diff --git a/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecord.cs b/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecord.cs index 0b1f928b7..c43313dd0 100644 --- a/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecord.cs +++ b/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecord.cs @@ -13,22 +13,22 @@ public class ActivityExecutionRecord : Entity /// /// Gets or sets the workflow instance ID. /// - public string WorkflowInstanceId { get; set; } = default!; + public string WorkflowInstanceId { get; set; } = null!; /// /// Gets or sets the activity ID. /// - public string ActivityId { get; set; } = default!; + public string ActivityId { get; set; } = null!; /// /// Gets or sets the activity node ID. /// - public string ActivityNodeId { get; set; } = default!; + public string ActivityNodeId { get; set; } = null!; /// /// The type of the activity. /// - public string ActivityType { get; set; } = default!; + public string ActivityType { get; set; } = null!; /// /// The version of the activity type. @@ -38,7 +38,7 @@ public class ActivityExecutionRecord : Entity /// /// The name of the activity. /// - public string? ActivityName { get; set; } = default!; + public string? ActivityName { get; set; } = null!; /// /// The state of the activity at the time this record is created or last updated. diff --git a/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecordSummary.cs b/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecordSummary.cs index b8734bc43..56409a15c 100644 --- a/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecordSummary.cs +++ b/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionRecordSummary.cs @@ -47,6 +47,11 @@ public class ActivityExecutionRecordSummary : Entity /// public bool HasBookmarks { get; set; } + /// + /// Gets or sets a collection of properties for the activity execution. + /// + public IDictionary? Properties { get; set; } + /// /// Gets or sets the status of the activity. /// @@ -74,7 +79,8 @@ public class ActivityExecutionRecordSummary : Entity StartedAt = record.StartedAt, HasBookmarks = record.HasBookmarks, Status = record.Status, - CompletedAt = record.CompletedAt + CompletedAt = record.CompletedAt, + Properties = record.Properties }; } } \ No newline at end of file diff --git a/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionStats.cs b/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionStats.cs index 306078897..61d5e212d 100644 --- a/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionStats.cs +++ b/src/clients/Elsa.Api.Client/Resources/ActivityExecutions/Models/ActivityExecutionStats.cs @@ -43,5 +43,5 @@ public class ActivityExecutionStats /// /// Gets or sets the total count of faults aggregated from the activity execution and its descendants. /// - public int AggregateFaultCount { get; set; } + public IDictionary? Properties { get; set; } } \ No newline at end of file diff --git a/src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Contracts/IIncidentStrategiesApi.cs b/src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IIncidentStrategiesApi.cs similarity index 92% rename from src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Contracts/IIncidentStrategiesApi.cs rename to src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IIncidentStrategiesApi.cs index 089226afc..713b34833 100644 --- a/src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Contracts/IIncidentStrategiesApi.cs +++ b/src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IIncidentStrategiesApi.cs @@ -2,7 +2,7 @@ using System.Text.Json.Nodes; using Elsa.Api.Client.Shared.Models; using Refit; -namespace Elsa.Api.Client.Resources.ResilienceStrategies.Contracts; +namespace Elsa.Api.Client.Resources.Resilience.Contracts; /// /// Provides operations for managing and retrieving resilience strategies within the application. diff --git a/src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IRetryAttemptsApi.cs b/src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IRetryAttemptsApi.cs new file mode 100644 index 000000000..fc25331d0 --- /dev/null +++ b/src/clients/Elsa.Api.Client/Resources/Resilience/Contracts/IRetryAttemptsApi.cs @@ -0,0 +1,11 @@ +using Elsa.Api.Client.Resources.Resilience.Models; +using Elsa.Api.Client.Shared.Models; +using Refit; + +namespace Elsa.Api.Client.Resources.Resilience.Contracts; + +public interface IRetryAttemptsApi +{ + [Get("/resilience/retries/{activityInstanceId}")] + Task> ListAsync(string activityInstanceId, int? skip = null, int? take = null, CancellationToken cancellationToken = default); +} \ No newline at end of file diff --git a/src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Models/ResilienceStrategyConfig.cs b/src/clients/Elsa.Api.Client/Resources/Resilience/Models/ResilienceStrategyConfig.cs similarity index 91% rename from src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Models/ResilienceStrategyConfig.cs rename to src/clients/Elsa.Api.Client/Resources/Resilience/Models/ResilienceStrategyConfig.cs index 136f3e2bd..2b56bd4f4 100644 --- a/src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Models/ResilienceStrategyConfig.cs +++ b/src/clients/Elsa.Api.Client/Resources/Resilience/Models/ResilienceStrategyConfig.cs @@ -3,7 +3,7 @@ using System.Text.Json.Nodes; using Elsa.Api.Client.Resources.Scripting.Models; using Elsa.Api.Client.Serialization; -namespace Elsa.Api.Client.Resources.ResilienceStrategies.Models; +namespace Elsa.Api.Client.Resources.Resilience.Models; public class ResilienceStrategyConfig { diff --git a/src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Models/ResilienceStrategyConfigMode.cs b/src/clients/Elsa.Api.Client/Resources/Resilience/Models/ResilienceStrategyConfigMode.cs similarity index 54% rename from src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Models/ResilienceStrategyConfigMode.cs rename to src/clients/Elsa.Api.Client/Resources/Resilience/Models/ResilienceStrategyConfigMode.cs index 489730c14..1c55276e2 100644 --- a/src/clients/Elsa.Api.Client/Resources/ResilienceStrategies/Models/ResilienceStrategyConfigMode.cs +++ b/src/clients/Elsa.Api.Client/Resources/Resilience/Models/ResilienceStrategyConfigMode.cs @@ -1,4 +1,4 @@ -namespace Elsa.Api.Client.Resources.ResilienceStrategies.Models; +namespace Elsa.Api.Client.Resources.Resilience.Models; public enum ResilienceStrategyConfigMode { diff --git a/src/clients/Elsa.Api.Client/Resources/Resilience/Models/RetryAttempt.cs b/src/clients/Elsa.Api.Client/Resources/Resilience/Models/RetryAttempt.cs new file mode 100644 index 000000000..f76518386 --- /dev/null +++ b/src/clients/Elsa.Api.Client/Resources/Resilience/Models/RetryAttempt.cs @@ -0,0 +1,13 @@ +using Elsa.Api.Client.Shared.Models; + +namespace Elsa.Api.Client.Resources.Resilience.Models; + +public class RetryAttemptRecord : Entity +{ + public string ActivityInstanceId { get; set; } = null!; + public string ActivityId { get; set; } = null!; + public string WorkflowInstanceId { get; set; } = null!; + public int AttemptNumber { get; set; } + public TimeSpan RetryDelay { get; set; } + public IDictionary Details { get; set; } +} \ No newline at end of file diff --git a/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Contracts/IWorkflowInstancesApi.cs b/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Contracts/IWorkflowInstancesApi.cs index a61f6f3fb..b3e071b59 100644 --- a/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Contracts/IWorkflowInstancesApi.cs +++ b/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Contracts/IWorkflowInstancesApi.cs @@ -34,7 +34,7 @@ public interface IWorkflowInstancesApi /// The number of records to return. /// The cancellation token. [Get("/workflow-instances/{workflowInstanceId}/journal")] - Task> GetJournalAsync(string workflowInstanceId, int? skip = default, int? take = default, CancellationToken cancellationToken = default); + Task> GetJournalAsync(string workflowInstanceId, int? skip = null, int? take = null, CancellationToken cancellationToken = default); /// /// Returns a page of journal records for the specified workflow instance. @@ -45,7 +45,7 @@ public interface IWorkflowInstancesApi /// The number of records to return. /// The cancellation token. [Post("/workflow-instances/{workflowInstanceId}/journal")] - Task> GetFilteredJournalAsync(string workflowInstanceId, GetFilteredJournalRequest? filter, int? skip = default, int? take = default, CancellationToken cancellationToken = default); + Task> GetFilteredJournalAsync(string workflowInstanceId, GetFilteredJournalRequest? filter, int? skip = null, int? take = null, CancellationToken cancellationToken = default); /// /// Returns the execution state of the specified workflow instance. diff --git a/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Models/ExceptionState.cs b/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Models/ExceptionState.cs index 9f4d36092..27f15b69b 100644 --- a/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Models/ExceptionState.cs +++ b/src/clients/Elsa.Api.Client/Resources/WorkflowInstances/Models/ExceptionState.cs @@ -11,7 +11,7 @@ public record ExceptionState(string Type, string Message, string? StackTrace, Ex /// Initializes a new instance of the class. /// [JsonConstructor] - public ExceptionState() : this(default!, default!, default, default) + public ExceptionState() : this(null!, null!, null, null) { } diff --git a/src/modules/Elsa.Common/Models/Page.cs b/src/modules/Elsa.Common/Models/Page.cs index fd9043d0e..cb59b6ab3 100644 --- a/src/modules/Elsa.Common/Models/Page.cs +++ b/src/modules/Elsa.Common/Models/Page.cs @@ -21,4 +21,5 @@ public static class Page /// The type of the items. /// A new instance of the class. public static Page Of(ICollection items, long totalCount) => new(items, totalCount); + public static Page Empty() => new([], 0); } \ No newline at end of file diff --git a/src/modules/Elsa.EntityFrameworkCore/Modules/Runtime/ActivityExecutionLogStore.cs b/src/modules/Elsa.EntityFrameworkCore/Modules/Runtime/ActivityExecutionLogStore.cs index 092781327..b3aca57f3 100644 --- a/src/modules/Elsa.EntityFrameworkCore/Modules/Runtime/ActivityExecutionLogStore.cs +++ b/src/modules/Elsa.EntityFrameworkCore/Modules/Runtime/ActivityExecutionLogStore.cs @@ -1,7 +1,9 @@ using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; using System.Text.Json; using Elsa.Common; using Elsa.Common.Codecs; +using Elsa.Common.Entities; using Elsa.Extensions; using Elsa.Workflows; using Elsa.Workflows.Management.Options; @@ -12,6 +14,7 @@ using Elsa.Workflows.Runtime.Filters; using Elsa.Workflows.Runtime.OrderDefinitions; using Elsa.Workflows.State; using JetBrains.Annotations; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using Open.Linq.AsyncExtensions; @@ -60,13 +63,15 @@ public class EFCoreActivityExecutionStore( /// public async Task> FindManySummariesAsync(ActivityExecutionRecordFilter filter, ActivityExecutionRecordOrder order, CancellationToken cancellationToken = default) { - return await store.QueryAsync(query => Filter(query, filter), ActivityExecutionRecordSummary.FromRecordExpression(), cancellationToken).ToList().AsEnumerable(); + var shadowRecords = await store.QueryAsync(query => Filter(query, filter), FromRecordExpression(), cancellationToken).ToList(); + return Map(shadowRecords); } /// public async Task> FindManySummariesAsync(ActivityExecutionRecordFilter filter, CancellationToken cancellationToken = default) { - return await store.QueryAsync(query => Filter(query, filter), ActivityExecutionRecordSummary.FromRecordExpression(), cancellationToken).ToList().AsEnumerable(); + var shadowRecords = await store.QueryAsync(query => Filter(query, filter), FromRecordExpression(), cancellationToken).ToList(); + return Map(shadowRecords); } /// @@ -142,4 +147,62 @@ public class EFCoreActivityExecutionStore( } private static IQueryable Filter(IQueryable queryable, ActivityExecutionRecordFilter filter) => filter.Apply(queryable); + + private static Expression> FromRecordExpression() + { + return record => new() + { + Id = record.Id, + WorkflowInstanceId = record.WorkflowInstanceId, + ActivityId = record.ActivityId, + ActivityNodeId = record.ActivityNodeId, + ActivityType = record.ActivityType, + ActivityTypeVersion = record.ActivityTypeVersion, + ActivityName = record.ActivityName, + StartedAt = record.StartedAt, + HasBookmarks = record.HasBookmarks, + Status = record.Status, + AggregateFaultCount = record.AggregateFaultCount, + SerializedProperties = EF.Property(record, "SerializedProperties"), + CompletedAt = record.CompletedAt + }; + } + + private IEnumerable Map(IEnumerable source) => source.Select(Map); + + private ActivityExecutionRecordSummary Map(ShadowActivityExecutionRecordSummary source) + { + return new() + { + Id = source.Id, + WorkflowInstanceId = source.WorkflowInstanceId, + ActivityId = source.ActivityId, + ActivityNodeId = source.ActivityNodeId, + ActivityType = source.ActivityType, + ActivityTypeVersion = source.ActivityTypeVersion, + ActivityName = source.ActivityName, + StartedAt = source.StartedAt, + HasBookmarks = source.HasBookmarks, + Status = source.Status, + AggregateFaultCount = source.AggregateFaultCount, + CompletedAt = source.CompletedAt, + Properties = source.SerializedProperties is null ? null : payloadSerializer.Deserialize>(source.SerializedProperties) + }; + } + + private class ShadowActivityExecutionRecordSummary : Entity + { + public string WorkflowInstanceId { get; set; } = null!; + public string ActivityId { get; set; } = null!; + public string ActivityNodeId { get; set; } = null!; + public string ActivityType { get; set; } = null!; + public int ActivityTypeVersion { get; set; } + public string? ActivityName { get; set; } + public DateTimeOffset StartedAt { get; set; } + public bool HasBookmarks { get; set; } + public ActivityStatus Status { get; set; } + public string? SerializedProperties { get; set; } + public int AggregateFaultCount { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + } } \ No newline at end of file diff --git a/src/modules/Elsa.Http/Activities/SendHttpRequestBase.cs b/src/modules/Elsa.Http/Activities/SendHttpRequestBase.cs index d922d1965..898c76365 100644 --- a/src/modules/Elsa.Http/Activities/SendHttpRequestBase.cs +++ b/src/modules/Elsa.Http/Activities/SendHttpRequestBase.cs @@ -4,6 +4,7 @@ using Elsa.Extensions; using Elsa.Http.ContentWriters; using Elsa.Http.UIHints; using Elsa.Resilience; +using Elsa.Resilience.Models; using Elsa.Workflows; using Elsa.Workflows.Attributes; using Elsa.Workflows.UIHints; @@ -124,6 +125,21 @@ public abstract class SendHttpRequestBase(string? source = null, int? line = nul await TrySendAsync(context); } + public IDictionary CollectRetryDetails(ActivityExecutionContext context, RetryAttempt attempt) + { + if (attempt.Result is not HttpResponseMessage response) + return new Dictionary(); + + return new Dictionary + { + ["StatusCode"] = response.StatusCode.ToString(), + ["ReasonPhrase"] = response.ReasonPhrase, + ["Content-Type"] = response.Content.Headers.ContentType?.MediaType ?? "application/octet-stream", + ["Date"] = response.Headers.Date.ToString(), + ["Retry-After"] = response.Headers.RetryAfter?.ToString() + }; + } + /// /// Handles the response. /// diff --git a/src/modules/Elsa.Http/Extensions/HttpStatusCodeExtensions.cs b/src/modules/Elsa.Http/Extensions/HttpStatusCodeExtensions.cs new file mode 100644 index 000000000..bbdfceccd --- /dev/null +++ b/src/modules/Elsa.Http/Extensions/HttpStatusCodeExtensions.cs @@ -0,0 +1,21 @@ +using System.Net; + +namespace Elsa.Http.Extensions; + +public static class HttpStatusCodeExtensions +{ + public static bool IsTransientStatusCode(this HttpStatusCode statusCode) + { + return statusCode switch + { + HttpStatusCode.RequestTimeout => true, + HttpStatusCode.TooManyRequests => true, + HttpStatusCode.InternalServerError => true, + HttpStatusCode.BadGateway => true, + HttpStatusCode.ServiceUnavailable => true, + HttpStatusCode.GatewayTimeout => true, + HttpStatusCode.Conflict => true, + _ => false + }; + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Http/Middleware/HttpWorkflowsMiddleware.cs b/src/modules/Elsa.Http/Middleware/HttpWorkflowsMiddleware.cs index fd621b9b7..668d18863 100644 --- a/src/modules/Elsa.Http/Middleware/HttpWorkflowsMiddleware.cs +++ b/src/modules/Elsa.Http/Middleware/HttpWorkflowsMiddleware.cs @@ -10,11 +10,9 @@ using System.Net; using System.Net.Mime; using System.Text.Json; using Elsa.Workflows.Activities; -using Elsa.Workflows.Helpers; using Elsa.Workflows.Runtime.Entities; using FastEndpoints; using System.Diagnostics.CodeAnalysis; -using Elsa.Common.Multitenancy; using Elsa.Workflows; using Elsa.Workflows.Management; using Elsa.Workflows.Management.Entities; diff --git a/src/modules/Elsa.Http/Resilience/HttpResilienceStrategy.cs b/src/modules/Elsa.Http/Resilience/HttpResilienceStrategy.cs index c346a5773..e466b5ab2 100644 --- a/src/modules/Elsa.Http/Resilience/HttpResilienceStrategy.cs +++ b/src/modules/Elsa.Http/Resilience/HttpResilienceStrategy.cs @@ -1,3 +1,4 @@ +using Elsa.Http.Extensions; using Elsa.Resilience; using Polly; using Polly.Retry; @@ -9,18 +10,30 @@ public class HttpResilienceStrategy : IResilienceStrategy { public string Id { get; set; } = null!; public string DisplayName { get; set; } = null!; - public int RetryCount { get; set; } = 3; - public double BackoffFactor { get; set; } = 2.0; + public int MaxRetryAttempts { get; set; } = 3; + public bool UseJitter { get; set; } + public TimeSpan Delay { get; set; } = TimeSpan.FromSeconds(1); + public DelayBackoffType BackoffType { get; set; } = DelayBackoffType.Exponential; - public async Task ExecuteAsync(Func> action) + public Task ConfigurePipeline(ResiliencePipelineBuilder pipelineBuilder, ResilienceContext context) { - AsyncRetryPolicy policy = Policy - .Handle() - .WaitAndRetryAsync( - RetryCount, - retryAttempt => TimeSpan.FromSeconds(Math.Pow(BackoffFactor, retryAttempt)) - ); - - return await policy.ExecuteAsync(action); + if (typeof(T) != typeof(HttpResponseMessage)) + throw new NotSupportedException($"{nameof(HttpResilienceStrategy)} only supports HttpResponseMessage."); + + var options = new RetryStrategyOptions + { + ShouldHandle = new PredicateBuilder() + .Handle() + .Handle() + .HandleResult(response => ((HttpResponseMessage)(object)response!).StatusCode.IsTransientStatusCode()), + MaxRetryAttempts = MaxRetryAttempts, + Delay = Delay, + UseJitter = UseJitter, + BackoffType = BackoffType, + Name = DisplayName + }; + + pipelineBuilder.AddRetry(options); + return Task.CompletedTask; } } \ No newline at end of file diff --git a/src/modules/Elsa.Http/Scripting/JavaScript/HttpJavaScriptHandler.cs b/src/modules/Elsa.Http/Scripting/JavaScript/HttpJavaScriptHandler.cs index ed51cd4d2..e3c611b2c 100644 --- a/src/modules/Elsa.Http/Scripting/JavaScript/HttpJavaScriptHandler.cs +++ b/src/modules/Elsa.Http/Scripting/JavaScript/HttpJavaScriptHandler.cs @@ -5,7 +5,6 @@ using Elsa.JavaScript.TypeDefinitions.Builders; using Elsa.JavaScript.TypeDefinitions.Contracts; using Elsa.JavaScript.TypeDefinitions.Models; using Elsa.Mediator.Contracts; -using Elsa.Workflows; using JetBrains.Annotations; namespace Elsa.Http.Scripting.JavaScript; diff --git a/src/modules/Elsa.Http/Services/DefaultRouteTableUpdater.cs b/src/modules/Elsa.Http/Services/DefaultRouteTableUpdater.cs index 715e31b56..bca546879 100644 --- a/src/modules/Elsa.Http/Services/DefaultRouteTableUpdater.cs +++ b/src/modules/Elsa.Http/Services/DefaultRouteTableUpdater.cs @@ -2,7 +2,6 @@ using Elsa.Extensions; using Elsa.Http.Bookmarks; using Elsa.Http.Contexts; using Elsa.Workflows; -using Elsa.Workflows.Helpers; using Elsa.Workflows.Models; using Elsa.Workflows.Runtime; using Elsa.Workflows.Runtime.Entities; diff --git a/src/modules/Elsa.Http/Services/RouteMatcher.cs b/src/modules/Elsa.Http/Services/RouteMatcher.cs index 4d9de3ae7..f0681272c 100644 --- a/src/modules/Elsa.Http/Services/RouteMatcher.cs +++ b/src/modules/Elsa.Http/Services/RouteMatcher.cs @@ -1,6 +1,5 @@ // Taken & adapted from https://github.com/markvincze/Stubbery/blob/main/src/Stubbery/RequestMatching/RouteMatcher.cs -using Elsa.Extensions; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing.Template; diff --git a/src/modules/Elsa.Resilience.Core/Contracts/IResilienceStrategy.cs b/src/modules/Elsa.Resilience.Core/Contracts/IResilienceStrategy.cs index 599f146bb..80ed32fff 100644 --- a/src/modules/Elsa.Resilience.Core/Contracts/IResilienceStrategy.cs +++ b/src/modules/Elsa.Resilience.Core/Contracts/IResilienceStrategy.cs @@ -1,8 +1,10 @@ -namespace Elsa.Resilience; +using Polly; + +namespace Elsa.Resilience; public interface IResilienceStrategy { string Id { get; set; } string DisplayName { get; set; } - Task ExecuteAsync(Func> action); + Task ConfigurePipeline(ResiliencePipelineBuilder pipelineBuilder, ResilienceContext context); } \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Contracts/IResilientActivity.cs b/src/modules/Elsa.Resilience.Core/Contracts/IResilientActivity.cs index d8b07e376..bc004baf5 100644 --- a/src/modules/Elsa.Resilience.Core/Contracts/IResilientActivity.cs +++ b/src/modules/Elsa.Resilience.Core/Contracts/IResilientActivity.cs @@ -1,5 +1,9 @@ +using Elsa.Resilience.Models; using Elsa.Workflows; namespace Elsa.Resilience; -public interface IResilientActivity : IActivity; \ No newline at end of file +public interface IResilientActivity : IActivity +{ + IDictionary CollectRetryDetails(ActivityExecutionContext context, RetryAttempt attempt); +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptReader.cs b/src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptReader.cs new file mode 100644 index 000000000..020ff3275 --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptReader.cs @@ -0,0 +1,9 @@ +using Elsa.Common.Models; +using Elsa.Resilience.Entities; + +namespace Elsa.Resilience; + +public interface IRetryAttemptReader +{ + Task> ReadAttemptsAsync(string activityInstanceId, PageArgs? pageArgs = null, CancellationToken cancellationToken = default); +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptRecorder.cs b/src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptRecorder.cs new file mode 100644 index 000000000..6a23b2b9b --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Contracts/IRetryAttemptRecorder.cs @@ -0,0 +1,8 @@ +using Elsa.Resilience.Models; + +namespace Elsa.Resilience; + +public interface IRetryAttemptRecorder +{ + Task RecordAsync(RecordRetryAttemptsContext context); +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Elsa.Resilience.Core.csproj b/src/modules/Elsa.Resilience.Core/Elsa.Resilience.Core.csproj index d36195aae..2dcaf7965 100644 --- a/src/modules/Elsa.Resilience.Core/Elsa.Resilience.Core.csproj +++ b/src/modules/Elsa.Resilience.Core/Elsa.Resilience.Core.csproj @@ -12,4 +12,9 @@ + + + + + \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Entities/RetryAttemptRecord.cs b/src/modules/Elsa.Resilience.Core/Entities/RetryAttemptRecord.cs new file mode 100644 index 000000000..c4247927a --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Entities/RetryAttemptRecord.cs @@ -0,0 +1,13 @@ +using Elsa.Common.Entities; + +namespace Elsa.Resilience.Entities; + +public class RetryAttemptRecord : Entity +{ + public string ActivityInstanceId { get; set; } = null!; + public string ActivityId { get; set; } = null!; + public string WorkflowInstanceId { get; set; } = null!; + public int AttemptNumber { get; set; } + public TimeSpan RetryDelay { get; set; } + public IDictionary Details { get; set; } = new Dictionary(); +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Extensions/ActivityExecutionExtensions.cs b/src/modules/Elsa.Resilience.Core/Extensions/ActivityExecutionExtensions.cs new file mode 100644 index 000000000..a12ccf001 --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Extensions/ActivityExecutionExtensions.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Nodes; +using Elsa.Workflows; + +namespace Elsa.Resilience.Extensions; + +public static class ActivityExecutionExtensions +{ + private const string RetriesAttemptedFlag = "HasRetryAttempts"; + private const string ResilienceStrategy = "ResilienceStrategy"; + + public static void SetRetriesAttemptedFlag(this ActivityExecutionContext context) + { + var current = context; + + while (current != null) + { + current.SetProperty(RetriesAttemptedFlag, true); + current = current.ParentActivityExecutionContext; + } + } + + public static bool GetRetriesAttemptedFlag(this ActivityExecutionContext context) + { + return context.GetProperty(RetriesAttemptedFlag, () => false); + } + + public static void SetResilienceStrategy(this ActivityExecutionContext context, JsonNode model) + { + context.SetProperty(ResilienceStrategy, model); + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Models/RecordRetryAttemptsContext.cs b/src/modules/Elsa.Resilience.Core/Models/RecordRetryAttemptsContext.cs new file mode 100644 index 000000000..1feb6f0cb --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Models/RecordRetryAttemptsContext.cs @@ -0,0 +1,6 @@ +using Elsa.Resilience.Entities; +using Elsa.Workflows; + +namespace Elsa.Resilience.Models; + +public record RecordRetryAttemptsContext(ActivityExecutionContext ActivityExecutionContext, ICollection Attempts, CancellationToken CancellationToken); \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Models/RetryAttempt.cs b/src/modules/Elsa.Resilience.Core/Models/RetryAttempt.cs new file mode 100644 index 000000000..717da5406 --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Models/RetryAttempt.cs @@ -0,0 +1,8 @@ +using Elsa.Workflows; + +namespace Elsa.Resilience.Models; + +public record RetryAttempt(ActivityExecutionContext ActivityExecutionContext, int AttemptNumber, TimeSpan RetryDelay, object? Result, Exception? Exception) +{ + internal static readonly object RetriesKey = new(); +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Serialization/ResilienceStrategySerializer.cs b/src/modules/Elsa.Resilience.Core/Serialization/ResilienceStrategySerializer.cs index 451a791fa..6bf798961 100644 --- a/src/modules/Elsa.Resilience.Core/Serialization/ResilienceStrategySerializer.cs +++ b/src/modules/Elsa.Resilience.Core/Serialization/ResilienceStrategySerializer.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; -using Elsa.Common.Converters; +using Elsa.Extensions; using Elsa.Resilience.Options; using Microsoft.Extensions.Options; @@ -36,7 +36,7 @@ public class ResilienceStrategySerializer PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true, NumberHandling = JsonNumberHandling.AllowReadingFromString - }; + }.WithConverters(new JsonStringEnumConverter()); _serializerOptions = serializerOptions; } diff --git a/src/modules/Elsa.Resilience.Core/Services/ResilientActivityInvoker.cs b/src/modules/Elsa.Resilience.Core/Services/ResilientActivityInvoker.cs index cc1dcaade..8bf2c250f 100644 --- a/src/modules/Elsa.Resilience.Core/Services/ResilientActivityInvoker.cs +++ b/src/modules/Elsa.Resilience.Core/Services/ResilientActivityInvoker.cs @@ -1,18 +1,86 @@ +using System.Text.Json; using Elsa.Expressions.Helpers; +using Elsa.Resilience.Entities; +using Elsa.Resilience.Extensions; using Elsa.Resilience.Models; +using Elsa.Resilience.Serialization; using Elsa.Workflows; +using Elsa.Workflows.State; +using Polly; +using Polly.Telemetry; namespace Elsa.Resilience; -public class ResilientActivityInvoker(IResilienceStrategyConfigEvaluator configEvaluator) : IResilientActivityInvoker +public class ResilientActivityInvoker( + IResilienceStrategyConfigEvaluator resilienceStrategyConfigEvaluator, + IRetryAttemptRecorder retryAttemptRecorder, + IIdentityGenerator identityGenerator, + ResilienceStrategySerializer resilienceStrategySerializer) : IResilientActivityInvoker { private const string ResilienceStrategyIdPropKey = "resilienceStrategy"; public async Task InvokeAsync(IResilientActivity activity, ActivityExecutionContext context, Func> action, CancellationToken cancellationToken = default) { + // Get the resilience strategy. var strategyConfig = GetStrategyConfig(activity); - var strategy = await configEvaluator.EvaluateAsync(strategyConfig, context.ExpressionExecutionContext, cancellationToken); - return strategy == null ? await action() : await strategy.ExecuteAsync(action); + var resilienceStrategy = await resilienceStrategyConfigEvaluator.EvaluateAsync(strategyConfig, context.ExpressionExecutionContext, cancellationToken); + + // If no resilience strategy is configured, execute the action as-is. + if (resilienceStrategy == null) + return await action(); + + // Record the applied strategy as part of the activity execution context for diagnostics. + var resilienceStrategyModel = JsonSerializer.SerializeToNode(resilienceStrategy, resilienceStrategySerializer.SerializerOptions)!; + context.SetResilienceStrategy(resilienceStrategyModel); + + // Create a resilience pipeline builder. + var builder = CreateResiliencePipelineBuilder(); + var retries = new List(); + context.TransientProperties[RetryAttempt.RetriesKey] = retries; + + // Create a resilience context. + var resilienceContext = ResilienceContextPool.Shared.Get(cancellationToken); + resilienceContext.Properties.Set(new(nameof(ActivityExecutionContext)), context); + + try + { + // Configure the resilience pipeline. + await resilienceStrategy.ConfigurePipeline(builder, resilienceContext); + var pipeline = builder.Build(); + + // Execute the action within the resilience pipeline. + var result = await pipeline.ExecuteAsync(async _ => await action(), resilienceContext); + + // Record the retry attempts. + await RecordRetryAttempts(activity, context, retries, cancellationToken); + + return result; + } + finally + { + ResilienceContextPool.Shared.Return(resilienceContext); + } + + } + + private async Task RecordRetryAttempts(IResilientActivity activity, ActivityExecutionContext context, ICollection attempts, CancellationToken cancellationToken = default) + { + if (attempts.Count > 0) + { + var records = Map(context, activity, attempts); + var recordContext = new RecordRetryAttemptsContext(context, records, cancellationToken); + await retryAttemptRecorder.RecordAsync(recordContext); + + // Propagate a flag that retries have occurred. This information can then be used to show the retry attempts in the workflow designer. + context.SetRetriesAttemptedFlag(); + } + } + + private ResiliencePipelineBuilder CreateResiliencePipelineBuilder() + { + var telemetryOptions = new TelemetryOptions(); + telemetryOptions.TelemetryListeners.Add(new RetryTelemetryListener()); + return new ResiliencePipelineBuilder().ConfigureTelemetry(telemetryOptions); } private ResilienceStrategyConfig? GetStrategyConfig(IResilientActivity resilientActivity) @@ -21,4 +89,24 @@ public class ResilientActivityInvoker(IResilienceStrategyConfigEvaluator configE ? null : value.ConvertTo(); } + + private ICollection Map(ActivityExecutionContext activityExecutionContext, IResilientActivity resilientActivity, ICollection attempts) + { + return attempts.Select(x => Map(activityExecutionContext, resilientActivity, x)).ToList(); + } + + private RetryAttemptRecord Map(ActivityExecutionContext activityExecutionContext, IResilientActivity resilientActivity, RetryAttempt attempt) + { + var details = resilientActivity.CollectRetryDetails(activityExecutionContext, attempt).Where(x => x.Value != null).ToDictionary(x => x.Key, x => x.Value!); + return new() + { + Id = identityGenerator.GenerateId(), + ActivityInstanceId = activityExecutionContext.Id, + ActivityId = activityExecutionContext.Activity.Id, + WorkflowInstanceId = activityExecutionContext.WorkflowExecutionContext.Id, + AttemptNumber = attempt.AttemptNumber, + RetryDelay = attempt.RetryDelay, + Details = details + }; + } } \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Services/RetryTelemetryListener.cs b/src/modules/Elsa.Resilience.Core/Services/RetryTelemetryListener.cs new file mode 100644 index 000000000..0614c6aa2 --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Services/RetryTelemetryListener.cs @@ -0,0 +1,27 @@ +using Elsa.Resilience.Models; +using Elsa.Workflows; +using Polly.Retry; +using Polly.Telemetry; + +namespace Elsa.Resilience; + +public class RetryTelemetryListener : TelemetryListener +{ + public override void Write(in TelemetryEventArguments args) + { + if (args.Event.EventName != "OnRetry" || args.Arguments is not OnRetryArguments retryArgs) + return; + + var ctx = args.Context; + + if (!ctx.Properties.TryGetValue(new(nameof(ActivityExecutionContext)), out var activityExecutionContext)) + return; + + var attempt = retryArgs.AttemptNumber; + var delay = retryArgs.RetryDelay; + var outcome = retryArgs.Outcome; + var record = new RetryAttempt(activityExecutionContext, attempt, delay, outcome.Result, outcome.Exception); + var records = (List)activityExecutionContext.TransientProperties[RetryAttempt.RetriesKey]; + records.Add(record); + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptReader.cs b/src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptReader.cs new file mode 100644 index 000000000..91f33a2b2 --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptReader.cs @@ -0,0 +1,14 @@ +using Elsa.Common.Models; +using Elsa.Resilience.Entities; + +namespace Elsa.Resilience; + +public class VoidRetryAttemptReader : IRetryAttemptReader +{ + public static VoidRetryAttemptReader Instance { get; } = new(); + + public Task> ReadAttemptsAsync(string activityInstanceId, PageArgs? pageArgs = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(Page.Empty()); + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptRecorder.cs b/src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptRecorder.cs new file mode 100644 index 000000000..a65103c91 --- /dev/null +++ b/src/modules/Elsa.Resilience.Core/Services/VoidRetryAttemptRecorder.cs @@ -0,0 +1,14 @@ +using Elsa.Resilience.Models; + +namespace Elsa.Resilience; + +public class VoidRetryAttemptRecorder : IRetryAttemptRecorder +{ + public static VoidRetryAttemptRecorder Instance { get; } = new(); + + public Task RecordAsync(RecordRetryAttemptsContext context) + { + // Send records into the void. + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Elsa.Resilience.csproj b/src/modules/Elsa.Resilience/Elsa.Resilience.csproj index 6a2f5cf12..1442fd235 100644 --- a/src/modules/Elsa.Resilience/Elsa.Resilience.csproj +++ b/src/modules/Elsa.Resilience/Elsa.Resilience.csproj @@ -10,6 +10,11 @@ + + + + + \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Endpoints/ResilienceStrategies/List/Endpoint.cs b/src/modules/Elsa.Resilience/Endpoints/ResilienceStrategies/List/Endpoint.cs index 5398e10e0..87f06d7a0 100644 --- a/src/modules/Elsa.Resilience/Endpoints/ResilienceStrategies/List/Endpoint.cs +++ b/src/modules/Elsa.Resilience/Endpoints/ResilienceStrategies/List/Endpoint.cs @@ -10,7 +10,7 @@ public class Endpoint(IResilienceStrategyCatalog catalog, ResilienceStrategySeri public override void Configure() { Get("/resilience/strategies"); - ConfigurePermissions("read:*", "read:resilience-strategies"); + ConfigurePermissions("read:*", "read:resilience", "read:resilience:strategies"); } public override async Task HandleAsync(CancellationToken ct) diff --git a/src/modules/Elsa.Resilience/Endpoints/Retries/List/Endpoint.cs b/src/modules/Elsa.Resilience/Endpoints/Retries/List/Endpoint.cs new file mode 100644 index 000000000..98de4938a --- /dev/null +++ b/src/modules/Elsa.Resilience/Endpoints/Retries/List/Endpoint.cs @@ -0,0 +1,34 @@ +using Elsa.Abstractions; +using Elsa.Common.Models; +using Elsa.Models; +using Elsa.Resilience.Entities; + +namespace Elsa.Resilience.Endpoints.Retries.List; + +public class Endpoint(IRetryAttemptReader reader) : ElsaEndpointWithoutRequest +{ + public override void Configure() + { + Get("/resilience/retries/{activityInstanceId}"); + ConfigurePermissions("read:*", "read:resilience", "read:resilience:retries"); + } + + public override async Task HandleAsync(CancellationToken ct) + { + var skip = Query("skip", false); + var take = Query("take", false); + var pageArgs = skip == null && take == null ? null : PageArgs.FromRange(skip, take); + var activityInstanceId = Route("activityInstanceId"); + + if (string.IsNullOrWhiteSpace(activityInstanceId)) + { + AddError("ActivityInstanceId is required."); + await SendErrorsAsync(cancellation: ct); + return; + } + + var page = await reader.ReadAttemptsAsync(activityInstanceId, pageArgs, ct); + var response = new PagedListResponse(page); + await SendOkAsync(response, ct); + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Endpoint.cs b/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Endpoint.cs new file mode 100644 index 000000000..17f9fd0b3 --- /dev/null +++ b/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Endpoint.cs @@ -0,0 +1,55 @@ +using System.Text.Json; +using FastEndpoints; +using Microsoft.Extensions.Caching.Memory; +using static Elsa.Resilience.Endpoints.SimulateResponse.StatusCodeMessageLookup; + +namespace Elsa.Resilience.Endpoints.SimulateResponse; + +public class SimulateResponseEndpoint(IMemoryCache memoryCache) : EndpointWithoutRequest +{ + private static readonly TimeSpan SlidingExpirationTimeSpan = TimeSpan.FromMinutes(15); + + public override void Configure() + { + Get("/simulate-response"); + AllowAnonymous(); + } + + public override async Task HandleAsync(CancellationToken ct) + { + var sessionId = HttpContext.Request.Query["sessionId"].FirstOrDefault() ?? "default"; + var codes = GetCodes(); + var cacheKey = $"status-simulation-session-{sessionId}"; + var nextIndex = memoryCache.GetOrCreate(cacheKey, entry => + { + entry.SlidingExpiration = SlidingExpirationTimeSpan; + return 0; + }); + + var currentCode = nextIndex < codes.Length ? codes[nextIndex] : codes[^1]; + var message = StatusMessages.TryGetValue(currentCode, out var reason) + ? reason + : $"Status Code {currentCode}"; + + if (nextIndex + 1 >= codes.Length) + { + memoryCache.Remove(cacheKey); + } + else + { + memoryCache.Set(cacheKey, nextIndex + 1, new MemoryCacheEntryOptions + { + SlidingExpiration = SlidingExpirationTimeSpan + }); + } + + await SendAsync(new(message), currentCode, ct); + } + + private int[] GetCodes() + { + var codesParam = HttpContext.Request.Query["codes"].FirstOrDefault(); + int[] defaultCodes = [429, 503, 200]; + return string.IsNullOrWhiteSpace(codesParam) ? defaultCodes : JsonSerializer.Deserialize(codesParam)!; + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Models.cs b/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Models.cs new file mode 100644 index 000000000..66ecce0ec --- /dev/null +++ b/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/Models.cs @@ -0,0 +1,3 @@ +namespace Elsa.Resilience.Endpoints.SimulateResponse; + +public record SimulatedResponse(string Message); \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/StatusCodeMessageLookup.cs b/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/StatusCodeMessageLookup.cs new file mode 100644 index 000000000..f83d1e7f4 --- /dev/null +++ b/src/modules/Elsa.Resilience/Endpoints/SimulateResponse/StatusCodeMessageLookup.cs @@ -0,0 +1,70 @@ +namespace Elsa.Resilience.Endpoints.SimulateResponse; + +public static class StatusCodeMessageLookup +{ + // Mapping of common HTTP status codes to their standard reason phrases/messages + public static readonly Dictionary StatusMessages = new() + { + [100] = "Continue", + [101] = "Switching Protocols", + [102] = "Processing", + [200] = "OK", + [201] = "Created", + [202] = "Accepted", + [203] = "Non-Authoritative Information", + [204] = "No Content", + [205] = "Reset Content", + [206] = "Partial Content", + [207] = "Multi-Status", + [208] = "Already Reported", + [226] = "IM Used", + [300] = "Multiple Choices", + [301] = "Moved Permanently", + [302] = "Found", + [303] = "See Other", + [304] = "Not Modified", + [305] = "Use Proxy", + [307] = "Temporary Redirect", + [308] = "Permanent Redirect", + [400] = "Bad Request", + [401] = "Unauthorized", + [402] = "Payment Required", + [403] = "Forbidden", + [404] = "Not Found", + [405] = "Method Not Allowed", + [406] = "Not Acceptable", + [407] = "Proxy Authentication Required", + [408] = "Request Timeout", + [409] = "Conflict", + [410] = "Gone", + [411] = "Length Required", + [412] = "Precondition Failed", + [413] = "Payload Too Large", + [414] = "URI Too Long", + [415] = "Unsupported Media Type", + [416] = "Range Not Satisfiable", + [417] = "Expectation Failed", + [418] = "I'm a teapot", + [421] = "Misdirected Request", + [422] = "Unprocessable Entity", + [423] = "Locked", + [424] = "Failed Dependency", + [425] = "Too Early", + [426] = "Upgrade Required", + [428] = "Precondition Required", + [429] = "Too Many Requests", + [431] = "Request Header Fields Too Large", + [451] = "Unavailable For Legal Reasons", + [500] = "Internal Server Error", + [501] = "Not Implemented", + [502] = "Bad Gateway", + [503] = "Service Unavailable", + [504] = "Gateway Timeout", + [505] = "HTTP Version Not Supported", + [506] = "Variant Also Negotiates", + [507] = "Insufficient Storage", + [508] = "Loop Detected", + [510] = "Not Extended", + [511] = "Network Authentication Required", + }; +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Features/ResilienceFeature.cs b/src/modules/Elsa.Resilience/Features/ResilienceFeature.cs index 740d42e9d..ee1ef144b 100644 --- a/src/modules/Elsa.Resilience/Features/ResilienceFeature.cs +++ b/src/modules/Elsa.Resilience/Features/ResilienceFeature.cs @@ -3,6 +3,7 @@ using Elsa.Features.Abstractions; using Elsa.Features.Services; using Elsa.Resilience.Modifiers; using Elsa.Resilience.Options; +using Elsa.Resilience.Recorders; using Elsa.Resilience.Serialization; using Elsa.Resilience.StrategySources; using Elsa.Workflows; @@ -12,6 +13,9 @@ namespace Elsa.Resilience.Features; public class ResilienceFeature(IModule module) : FeatureBase(module) { + private Func _retryAttemptRecorder = sp => sp.GetRequiredService(); + private Func _retryAttemptReader = sp => sp.GetRequiredService(); + public ResilienceFeature AddResilienceStrategyType() where T : IResilienceStrategy { return AddResilienceStrategyType(typeof(T)); @@ -23,6 +27,33 @@ public class ResilienceFeature(IModule module) : FeatureBase(module) return this; } + public ResilienceFeature WithActivityExecutionContextRetryAttemptRecorder() + { + return WithRetryAttemptRecorder() + .WithRetryAttemptReader(); + } + + public ResilienceFeature WithVoidRetryAttemptRecorder() + { + return WithRetryAttemptRecorder() + .WithRetryAttemptReader(); + } + + public ResilienceFeature WithRetryAttemptRecorder() => WithRetryAttemptRecorder(sp => (IRetryAttemptRecorder)ActivatorUtilities.CreateInstance(sp)!); + public ResilienceFeature WithRetryAttemptReader() => WithRetryAttemptReader(sp => (IRetryAttemptReader)ActivatorUtilities.CreateInstance(sp)!); + + public ResilienceFeature WithRetryAttemptRecorder(Func recorder) + { + _retryAttemptRecorder = recorder; + return this; + } + + public ResilienceFeature WithRetryAttemptReader(Func reader) + { + _retryAttemptReader = reader; + return this; + } + public override void Configure() { Module.AddFastEndpointsAssembly(); @@ -38,6 +69,13 @@ public class ResilienceFeature(IModule module) : FeatureBase(module) .AddScoped() .AddScoped() .AddScoped() - .AddScoped(); + .AddScoped() + .AddSingleton(VoidRetryAttemptRecorder.Instance) + .AddSingleton(VoidRetryAttemptReader.Instance) + .AddScoped() + .AddScoped() + .AddScoped(_retryAttemptRecorder) + .AddScoped(_retryAttemptReader) + .AddHandlersFrom(); } } \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Handlers/PropagateRetryAttemptedFlag.cs b/src/modules/Elsa.Resilience/Handlers/PropagateRetryAttemptedFlag.cs new file mode 100644 index 000000000..d56d6ffc6 --- /dev/null +++ b/src/modules/Elsa.Resilience/Handlers/PropagateRetryAttemptedFlag.cs @@ -0,0 +1,20 @@ +using Elsa.Mediator.Contracts; +using Elsa.Resilience.Extensions; +using Elsa.Workflows.Runtime.Notifications; +using JetBrains.Annotations; + +namespace Elsa.Resilience.Handlers; + +[UsedImplicitly] +public class PropagateRetryAttemptedFlag : INotificationHandler +{ + public Task HandleAsync(BackgroundActivityExecutionCompleted notification, CancellationToken cancellationToken) + { + var hasRetries = notification.ActivityExecutionContext.GetRetriesAttemptedFlag(); + + if (hasRetries) + notification.ActivityExecutionContext.SetRetriesAttemptedFlag(); // Propagates the flag to all ancestors. + + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptReader.cs b/src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptReader.cs new file mode 100644 index 000000000..2f9f95779 --- /dev/null +++ b/src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptReader.cs @@ -0,0 +1,22 @@ +using Elsa.Common.Models; +using Elsa.Extensions; +using Elsa.Resilience.Entities; +using Elsa.Workflows.Runtime; + +namespace Elsa.Resilience.Recorders; + +public class ActivityExecutionContextRetryAttemptReader(IActivityExecutionStore activityExecutionStore) : IRetryAttemptReader +{ + public async Task> ReadAttemptsAsync(string activityInstanceId, PageArgs? pageArgs = null, CancellationToken cancellationToken = default) + { + var record = await activityExecutionStore.FindAsync(new() + { + Id = activityInstanceId + }, cancellationToken); + + if (record?.Properties == null || !record.Properties.TryGetValue("RetryAttempts", out var value) || value is not ICollection retryAttempts) + return Page.Empty(); + + return retryAttempts.Paginate(pageArgs); + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptRecorder.cs b/src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptRecorder.cs new file mode 100644 index 000000000..640d1a66f --- /dev/null +++ b/src/modules/Elsa.Resilience/Recorders/ActivityExecutionContextRetryAttemptRecorder.cs @@ -0,0 +1,14 @@ +using Elsa.Resilience.Models; +using JetBrains.Annotations; + +namespace Elsa.Resilience.Recorders; + +[UsedImplicitly] +public class ActivityExecutionContextRetryAttemptRecorder : IRetryAttemptRecorder +{ + public Task RecordAsync(RecordRetryAttemptsContext context) + { + context.ActivityExecutionContext.Properties["RetryAttempts"] = context.Attempts; + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/src/modules/Elsa.Workflows.Api/Endpoints/ActivityExecutions/Report/Endpoint.cs b/src/modules/Elsa.Workflows.Api/Endpoints/ActivityExecutions/Report/Endpoint.cs index 42e49fcfb..e8804937f 100644 --- a/src/modules/Elsa.Workflows.Api/Endpoints/ActivityExecutions/Report/Endpoint.cs +++ b/src/modules/Elsa.Workflows.Api/Endpoints/ActivityExecutions/Report/Endpoint.cs @@ -32,7 +32,7 @@ internal class Report : ElsaEndpoint { var stats = (await _activityExecutionStatsService.GetStatsAsync(request.WorkflowInstanceId, request.ActivityNodeIds, cancellationToken)).ToList(); - return new Response + return new() { Stats = stats }; diff --git a/src/modules/Elsa.Workflows.Core/Contexts/WorkflowExecutionContext.ExecutionLogEntry.cs b/src/modules/Elsa.Workflows.Core/Contexts/WorkflowExecutionContext.ExecutionLogEntry.cs index 137f372fd..2f70551ea 100644 --- a/src/modules/Elsa.Workflows.Core/Contexts/WorkflowExecutionContext.ExecutionLogEntry.cs +++ b/src/modules/Elsa.Workflows.Core/Contexts/WorkflowExecutionContext.ExecutionLogEntry.cs @@ -12,17 +12,17 @@ public partial class WorkflowExecutionContext /// The message of the event. /// Any contextual data related to this event. /// Returns the created . - public WorkflowExecutionLogEntry AddExecutionLogEntry(string eventName, string? message = default, object? payload = default) + public WorkflowExecutionLogEntry AddExecutionLogEntry(string eventName, string? message = null, object? payload = null) { var logEntry = new WorkflowExecutionLogEntry( Id, - default, + null, Workflow.Id, Workflow.Type, Workflow.Identity.Version, Workflow.Name, Workflow.Identity.Id, - default, + null, SystemClock.UtcNow, ExecutionLogSequence++, eventName, diff --git a/src/modules/Elsa.Workflows.Runtime/Entities/ActivityExecutionRecordSummary.cs b/src/modules/Elsa.Workflows.Runtime/Entities/ActivityExecutionRecordSummary.cs index 963ba31ed..1634207aa 100644 --- a/src/modules/Elsa.Workflows.Runtime/Entities/ActivityExecutionRecordSummary.cs +++ b/src/modules/Elsa.Workflows.Runtime/Entities/ActivityExecutionRecordSummary.cs @@ -52,6 +52,11 @@ public class ActivityExecutionRecordSummary : Entity /// Gets or sets the status of the activity. /// public ActivityStatus Status { get; set; } + + /// + /// Gets or sets a dictionary of key-value pairs representing additional properties for the activity execution record summary. + /// + public IDictionary? Properties { get; set; } /// /// Gets or sets the aggregated count of faults encountered during the execution of the activity instance and its descendants. @@ -81,6 +86,7 @@ public class ActivityExecutionRecordSummary : Entity HasBookmarks = record.HasBookmarks, Status = record.Status, AggregateFaultCount = record.AggregateFaultCount, + Properties = record.Properties?.Count > 0 ? record.Properties : null, CompletedAt = record.CompletedAt, }; } @@ -103,6 +109,7 @@ public class ActivityExecutionRecordSummary : Entity HasBookmarks = record.HasBookmarks, Status = record.Status, AggregateFaultCount = record.AggregateFaultCount, + Properties = record.Properties, CompletedAt = record.CompletedAt }; } diff --git a/src/modules/Elsa.Workflows.Runtime/Middleware/Activities/BackgroundActivityInvokerMiddleware.cs b/src/modules/Elsa.Workflows.Runtime/Middleware/Activities/BackgroundActivityInvokerMiddleware.cs index db98a7ac3..e744870a3 100644 --- a/src/modules/Elsa.Workflows.Runtime/Middleware/Activities/BackgroundActivityInvokerMiddleware.cs +++ b/src/modules/Elsa.Workflows.Runtime/Middleware/Activities/BackgroundActivityInvokerMiddleware.cs @@ -1,10 +1,12 @@ using System.Text.Json; using Elsa.Extensions; +using Elsa.Mediator.Contracts; using Elsa.Workflows.CommitStates; using Elsa.Workflows.Middleware.Activities; using Elsa.Workflows.Models; using Elsa.Workflows.Options; using Elsa.Workflows.Pipelines.ActivityExecution; +using Elsa.Workflows.Runtime.Notifications; using Elsa.Workflows.Runtime.Stimuli; using JetBrains.Annotations; using Microsoft.Extensions.Logging; @@ -20,7 +22,8 @@ public class BackgroundActivityInvokerMiddleware( ILogger logger, IIdentityGenerator identityGenerator, IBackgroundActivityScheduler backgroundActivityScheduler, - ICommitStrategyRegistry commitStrategyRegistry) + ICommitStrategyRegistry commitStrategyRegistry, + IMediator mediator) : DefaultActivityInvokerMiddleware(next, commitStrategyRegistry, logger) { internal static string GetBackgroundActivityOutputKey(string activityNodeId) => $"__BackgroundActivityOutput:{activityNodeId}"; @@ -29,6 +32,7 @@ public class BackgroundActivityInvokerMiddleware( internal static string GetBackgroundActivityJournalDataKey(string activityNodeId) => $"__BackgroundActivityJournalData:{activityNodeId}"; internal static string GetBackgroundActivityScheduledActivitiesKey(string activityNodeId) => $"__BackgroundActivityScheduledActivities:{activityNodeId}"; internal static string GetBackgroundActivityBookmarksKey(string activityNodeId) => $"__BackgroundActivityBookmarks:{activityNodeId}"; + internal static string GetBackgroundActivityPropertiesKey(string activityNodeId) => $"__BackgroundActivityProperties:{activityNodeId}"; internal const string BackgroundActivityBookmarkName = "BackgroundActivity"; /// @@ -49,9 +53,11 @@ public class BackgroundActivityInvokerMiddleware( CaptureOutputIfAny(context); CaptureJournalData(context); CaptureBookmarkData(context); + CapturePropertiesIfAny(context); await CompleteBackgroundActivityOutcomesAsync(context); await CompleteBackgroundActivityAsync(context); await CompleteBackgroundActivityScheduledActivitiesAsync(context); + await mediator.SendAsync(new BackgroundActivityExecutionCompleted(context), context.CancellationToken); } } } @@ -155,6 +161,21 @@ public class BackgroundActivityInvokerMiddleware( context.WorkflowExecutionContext.Properties.Remove(bookmarksKey); } + + private void CapturePropertiesIfAny(ActivityExecutionContext context) + { + var activity = context.Activity; + var propertiesKey = GetBackgroundActivityPropertiesKey(activity.NodeId); + var capturedProperties = context.WorkflowExecutionContext.GetProperty>(propertiesKey); + + context.WorkflowExecutionContext.Properties.Remove(propertiesKey); + + if (capturedProperties == null) + return; + + foreach (var property in capturedProperties) + context.Properties[property.Key] = property.Value; + } private async Task CompleteBackgroundActivityOutcomesAsync(ActivityExecutionContext context) { diff --git a/src/modules/Elsa.Workflows.Runtime/Models/ActivityExecutionStats.cs b/src/modules/Elsa.Workflows.Runtime/Models/ActivityExecutionStats.cs index 859efc83f..2307bbdcd 100644 --- a/src/modules/Elsa.Workflows.Runtime/Models/ActivityExecutionStats.cs +++ b/src/modules/Elsa.Workflows.Runtime/Models/ActivityExecutionStats.cs @@ -44,4 +44,9 @@ public class ActivityExecutionStats /// Gets or sets the total count of faults aggregated from the activity execution and its descendants. /// public int AggregateFaultCount { get; set; } + + /// + /// Gets or sets additional properties related to the activity execution. + /// + public IDictionary? Properties { get; set; } } \ No newline at end of file diff --git a/src/modules/Elsa.Workflows.Runtime/Notifications/BackgroundActivityExecutionCompleted.cs b/src/modules/Elsa.Workflows.Runtime/Notifications/BackgroundActivityExecutionCompleted.cs new file mode 100644 index 000000000..80d72eb8a --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/Notifications/BackgroundActivityExecutionCompleted.cs @@ -0,0 +1,5 @@ +using Elsa.Mediator.Contracts; + +namespace Elsa.Workflows.Runtime.Notifications; + +public record BackgroundActivityExecutionCompleted(ActivityExecutionContext ActivityExecutionContext) : INotification; \ No newline at end of file diff --git a/src/modules/Elsa.Workflows.Runtime/Services/ActivityExecutionService.cs b/src/modules/Elsa.Workflows.Runtime/Services/ActivityExecutionService.cs index 2a57bbfab..0235c7d57 100644 --- a/src/modules/Elsa.Workflows.Runtime/Services/ActivityExecutionService.cs +++ b/src/modules/Elsa.Workflows.Runtime/Services/ActivityExecutionService.cs @@ -16,7 +16,7 @@ public class ActivityExecutionStatsService : IActivityExecutionStatsService { _store = store; } - + /// public async Task> GetStatsAsync(string workflowInstanceId, IEnumerable activityNodeIds, CancellationToken cancellationToken = default) { @@ -28,18 +28,24 @@ public class ActivityExecutionStatsService : IActivityExecutionStatsService var order = new ActivityExecutionRecordOrder(x => x.StartedAt, OrderDirection.Ascending); var records = (await _store.FindManySummariesAsync(filter, order, cancellationToken)).ToList(); var groupedRecords = records.GroupBy(x => x.ActivityNodeId).ToList(); - var stats = groupedRecords.Select(grouping => new ActivityExecutionStats + var stats = groupedRecords.Select(grouping => { - ActivityNodeId = grouping.Key, - ActivityId = grouping.First().ActivityId, - StartedCount = grouping.Count(), - CompletedCount = grouping.Count(x => x.CompletedAt != null), - UncompletedCount = grouping.Count(x => x.CompletedAt == null), - IsBlocked = grouping.Any(x => x.HasBookmarks), - IsFaulted = grouping.Any(x => x.Status == ActivityStatus.Faulted), - AggregateFaultCount = grouping.Last().AggregateFaultCount + var first = grouping.First(); + var last = grouping.Last(); + return new ActivityExecutionStats + { + ActivityNodeId = grouping.Key, + ActivityId = first.ActivityId, + StartedCount = grouping.Count(), + CompletedCount = grouping.Count(x => x.CompletedAt != null), + UncompletedCount = grouping.Count(x => x.CompletedAt == null), + IsBlocked = grouping.Any(x => x.HasBookmarks), + IsFaulted = grouping.Any(x => x.Status == ActivityStatus.Faulted), + AggregateFaultCount = last.AggregateFaultCount, + Properties = last.Properties + }; }).ToList(); - + return stats; } @@ -47,7 +53,7 @@ public class ActivityExecutionStatsService : IActivityExecutionStatsService public async Task GetStatsAsync(string workflowInstanceId, string activityNodeId, CancellationToken cancellationToken = default) { var stats = (await GetStatsAsync(workflowInstanceId, [activityNodeId], cancellationToken)).FirstOrDefault(); - + return stats ?? new ActivityExecutionStats { ActivityNodeId = activityNodeId diff --git a/src/modules/Elsa.Workflows.Runtime/Services/BackgroundActivityInvoker.cs b/src/modules/Elsa.Workflows.Runtime/Services/BackgroundActivityInvoker.cs index 13ff7f8a0..49c37e7b2 100644 --- a/src/modules/Elsa.Workflows.Runtime/Services/BackgroundActivityInvoker.cs +++ b/src/modules/Elsa.Workflows.Runtime/Services/BackgroundActivityInvoker.cs @@ -62,25 +62,27 @@ public class BackgroundActivityInvoker( var journalDataKey = BackgroundActivityInvokerMiddleware.GetBackgroundActivityJournalDataKey(activityNodeId); var bookmarksKey = BackgroundActivityInvokerMiddleware.GetBackgroundActivityBookmarksKey(activityNodeId); var scheduledActivitiesKey = BackgroundActivityInvokerMiddleware.GetBackgroundActivityScheduledActivitiesKey(activityNodeId); + var propsKey = BackgroundActivityInvokerMiddleware.GetBackgroundActivityPropertiesKey(activityNodeId); var outcomes = activityExecutionContext.GetBackgroundOutcomes()?.ToList(); var completed = activityExecutionContext.GetBackgroundCompleted(); var scheduledActivities = activityExecutionContext.GetBackgroundScheduledActivities().ToList(); var workflowInstanceId = scheduledBackgroundActivity.WorkflowInstanceId; var outputValues = await activityPropertyLogPersistenceEvaluator.GetPersistableOutputAsync(activityExecutionContext); - var properties = new Dictionary + var bookmarkProps = new Dictionary { [scheduledActivitiesKey] = JsonSerializer.Serialize(scheduledActivities), [inputKey] = outputValues, [journalDataKey] = activityExecutionContext.JournalData, - [bookmarksKey] = activityExecutionContext.Bookmarks.ToList() + [bookmarksKey] = activityExecutionContext.Bookmarks.ToList(), + [propsKey] = activityExecutionContext.Properties }; - if (outcomes != null) properties[outcomesKey] = outcomes; - if (completed != null) properties[completedKey] = completed; + if (outcomes != null) bookmarkProps[outcomesKey] = outcomes; + if (completed != null) bookmarkProps[completedKey] = completed; var resumeBookmarkOptions = new ResumeBookmarkOptions { - Properties = properties, + Properties = bookmarkProps, }; var enqueuedBookmark = new NewBookmarkQueueItem { diff --git a/test/integration/Elsa.Resilience.IntegrationTests/Elsa.Resilience.IntegrationTests.csproj b/test/integration/Elsa.Resilience.IntegrationTests/Elsa.Resilience.IntegrationTests.csproj new file mode 100644 index 000000000..95419bcf8 --- /dev/null +++ b/test/integration/Elsa.Resilience.IntegrationTests/Elsa.Resilience.IntegrationTests.csproj @@ -0,0 +1,15 @@ + + + + net9.0 + + + + + + + + + + + diff --git a/test/integration/Elsa.Resilience.IntegrationTests/FlowSendHttpRequestResilienceTests.cs b/test/integration/Elsa.Resilience.IntegrationTests/FlowSendHttpRequestResilienceTests.cs new file mode 100644 index 000000000..421809094 --- /dev/null +++ b/test/integration/Elsa.Resilience.IntegrationTests/FlowSendHttpRequestResilienceTests.cs @@ -0,0 +1,81 @@ +using System.Net; +using Elsa.Extensions; +using Elsa.Http; +using Elsa.Resilience.Extensions; +using Elsa.Resilience.Models; +using Elsa.Testing.Shared; +using Elsa.Workflows; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; + +namespace Elsa.Resilience.IntegrationTests; + +public class FlowSendHttpRequestResilienceTests +{ + private readonly IServiceProvider _services; + private readonly TestRetryAttemptRecorder _recorder = new(); + + public FlowSendHttpRequestResilienceTests(ITestOutputHelper output) + { + var handler = new SequentialStatusHandler([HttpStatusCode.TooManyRequests, HttpStatusCode.ServiceUnavailable, HttpStatusCode.OK]); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Resilience:Strategies:0:$type"] = "HttpResilienceStrategy", + ["Resilience:Strategies:0:Id"] = "test", + ["Resilience:Strategies:0:DisplayName"] = "Test", + ["Resilience:Strategies:0:MaxRetryAttempts"] = "5", + ["Resilience:Strategies:0:Delay"] = "00:00:00.001" + }) + .Build(); + + _services = new TestApplicationBuilder(output) + .ConfigureServices(s => + { + s.AddSingleton(configuration); + }) + .ConfigureElsa(module => + { + module.UseHttp(http => + { + http.HttpClientBuilder = builder => builder.ConfigurePrimaryHttpMessageHandler(() => handler); + }); + + module.UseResilience(resilience => + { + resilience.WithRetryAttemptRecorder(_ => _recorder); + }); + }) + .Build(); + } + + [Fact(DisplayName = "FlowSendHttpRequest retries using selected resilience strategy")] + public async Task InvokesResilienceStrategy() + { + var activity = new FlowSendHttpRequest + { + Url = new(new Uri("http://localhost/test")), + Method = new("GET"), + ExpectedStatusCodes = new([200]), + CustomProperties = + { + ["resilienceStrategy"] = new ResilienceStrategyConfig + { + Mode = ResilienceStrategyConfigMode.Identifier, + StrategyId = "test" + } + } + }; + + var result = await _services.RunActivityAsync(activity); + var statusCode = result.GetActivityOutput(activity, nameof(SendHttpRequestBase.StatusCode)); + + Assert.Equal(200, statusCode); + Assert.Equal(2, _recorder.Attempts.Count); + Assert.Collection(_recorder.Attempts, + first => Assert.Equal(0, first.AttemptNumber), + second => Assert.Equal(1, second.AttemptNumber)); + } +} diff --git a/test/integration/Elsa.Resilience.IntegrationTests/SequentialStatusHandler.cs b/test/integration/Elsa.Resilience.IntegrationTests/SequentialStatusHandler.cs new file mode 100644 index 000000000..c12392b85 --- /dev/null +++ b/test/integration/Elsa.Resilience.IntegrationTests/SequentialStatusHandler.cs @@ -0,0 +1,19 @@ +using System.Net; +using System.Net.Http.Headers; + +namespace Elsa.Resilience.IntegrationTests; + +public class SequentialStatusHandler(IEnumerable codes) : HttpMessageHandler +{ + private readonly Queue _codes = new(codes); + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var code = _codes.Count > 0 ? _codes.Dequeue() : _codes.Last(); + var response = new HttpResponseMessage(code) + { + Content = new StringContent(string.Empty) + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain"); + return Task.FromResult(response); + } +} diff --git a/test/integration/Elsa.Resilience.IntegrationTests/TestRetryAttemptRecorder.cs b/test/integration/Elsa.Resilience.IntegrationTests/TestRetryAttemptRecorder.cs new file mode 100644 index 000000000..819c888bd --- /dev/null +++ b/test/integration/Elsa.Resilience.IntegrationTests/TestRetryAttemptRecorder.cs @@ -0,0 +1,16 @@ +using Elsa.Resilience.Entities; +using Elsa.Resilience.Models; + +namespace Elsa.Resilience.IntegrationTests; + +public class TestRetryAttemptRecorder : IRetryAttemptRecorder +{ + public IList Attempts { get; } = new List(); + + public Task RecordAsync(RecordRetryAttemptsContext context) + { + foreach (var record in context.Attempts) + Attempts.Add(record); + return Task.CompletedTask; + } +} diff --git a/test/integration/Elsa.Resilience.IntegrationTests/Usings.cs b/test/integration/Elsa.Resilience.IntegrationTests/Usings.cs new file mode 100644 index 000000000..c802f4480 --- /dev/null +++ b/test/integration/Elsa.Resilience.IntegrationTests/Usings.cs @@ -0,0 +1 @@ +global using Xunit;