From 1e41f6fdf82b502b63c5bc1f43e2a254826a6628 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 13:34:25 +0200 Subject: [PATCH 01/20] Add Elsa runtime readiness health checks --- Directory.Packages.props | 2 + doc/wiki/README.md | 1 + doc/wiki/build-run-operate.md | 2 +- doc/wiki/health-checks.md | 59 +++++++++++++++++++ src/apps/Elsa.Server.Web/Program.cs | 17 +++++- .../Elsa.Workflows.Runtime.csproj | 1 + .../Extensions/HealthCheckExtensions.cs | 35 +++++++++++ .../ElsaDistributedLockHealthCheck.cs | 40 +++++++++++++ .../HealthChecks/ElsaRuntimeHealthCheck.cs | 40 +++++++++++++ .../ElsaWorkflowPersistenceHealthCheck.cs | 44 ++++++++++++++ .../ElsaDistributedLockHealthCheckTests.cs | 53 +++++++++++++++++ .../ElsaRuntimeHealthCheckTests.cs | 57 ++++++++++++++++++ ...ElsaWorkflowPersistenceHealthCheckTests.cs | 46 +++++++++++++++ 13 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 doc/wiki/health-checks.md create mode 100644 src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs create mode 100644 src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs create mode 100644 src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs create mode 100644 src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs create mode 100644 test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs create mode 100644 test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs create mode 100644 test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 91bf85044..cf643d6ea 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -30,6 +30,7 @@ + @@ -71,6 +72,7 @@ + diff --git a/doc/wiki/README.md b/doc/wiki/README.md index e5bb48117..8da5e2b6a 100644 --- a/doc/wiki/README.md +++ b/doc/wiki/README.md @@ -48,6 +48,7 @@ flowchart LR | [Persistence](persistence.md) | In-memory stores, EF Core stores, provider packages, migrations, and multi-provider rules. | | [Diagnostics Structured Logs](diagnostics-structured-logs.md) | `ILogger` capture, live feed, REST/SignalR surface, redaction, and SQLite persistence. | | [Diagnostics Console Logs](diagnostics-console-logs.md) | Raw stdout/stderr capture, live feed, REST/SignalR surface, and redaction. | +| [Health Checks](health-checks.md) | Elsa runtime readiness probes, liveness/readiness mapping, and Kubernetes probe guidance. | | [Identity, Tenancy, And Security](identity-tenancy-security.md) | Users, applications, roles, API keys, tenant resolution, and authorization touch points. | | [Testing Guide](testing-guide.md) | Test project layout, fixture choices, and targeted commands. | | [Extension Guide](extension-guide.md) | How to add features, activities, expression providers, stores, endpoints, and ingress sources. | diff --git a/doc/wiki/build-run-operate.md b/doc/wiki/build-run-operate.md index 079ac17e7..70c13a373 100644 --- a/doc/wiki/build-run-operate.md +++ b/doc/wiki/build-run-operate.md @@ -135,7 +135,7 @@ Console log diagnostics endpoints include (when `Elsa.Diagnostics.ConsoleLogs` i - `POST /elsa/api/diagnostics/console-logs/recent` - `GET /elsa/api/diagnostics/console-logs/sources` -Health checks are mapped to `/` in the reference server. +Health checks are mapped to `/` and `/health/live` for process liveness and `/health/ready` for Elsa runtime readiness in the reference server. See [Health Checks](health-checks.md) for Kubernetes liveness/readiness recommendations and Elsa-specific runtime readiness probes. ## Runtime Knobs diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md new file mode 100644 index 000000000..42898f5e5 --- /dev/null +++ b/doc/wiki/health-checks.md @@ -0,0 +1,59 @@ +# Health Checks + +Elsa hosts can opt in to Elsa-specific readiness checks on top of the normal ASP.NET Core process liveness check. The first slice focuses on runtime dependencies that are available through existing Elsa service contracts and avoids exposing connection strings, credentials, or provider-specific details. + +## Opt In + +Register ASP.NET Core health checks and then add Elsa readiness checks: + +```csharp +services + .AddHealthChecks() + .AddElsaReadinessChecks(includeDistributedLocks: true); +``` + +Use `includeDistributedLocks: true` when the host enables distributed runtime behavior or relies on distributed locks for workflow coordination. Leave it `false` for simple single-node hosts that do not want a lock probe. + +Map separate endpoints for liveness and readiness: + +```csharp +app.MapHealthChecks("/health/live", new() +{ + Predicate = _ => false +}); + +app.MapHealthChecks("/health/ready", new() +{ + Predicate = check => check.Tags.Contains("readiness") +}); +``` + +## Current Elsa Readiness Checks + +- `elsa-runtime`: creates a workflow runtime client and reports `Degraded` when the runtime is paused or draining, because it is process-up but not accepting new work. +- `elsa-workflow-persistence`: performs small read-only probes against workflow definitions, workflow instances, triggers, and the bookmark queue store. +- `elsa-distributed-locks`: optionally verifies that the configured distributed lock provider can acquire and release a probe lock. + +The health check data only includes subsystem categories and operational state such as readiness reason and active execution-cycle count. It does not include connection strings, lock names, credentials, tenant IDs, or workflow payloads. + +## Kubernetes Recommendation + +Use liveness to answer "is the ASP.NET Core process alive?" and readiness to answer "can this Elsa instance accept workflow traffic?" + +```yaml +livenessProbe: + httpGet: + path: /health/live + port: http + periodSeconds: 10 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health/ready + port: http + periodSeconds: 10 + failureThreshold: 3 +``` + +Do not point liveness at Elsa readiness checks. A paused, draining, or temporarily database-degraded runtime should be removed from service by readiness without forcing Kubernetes to restart the process. diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 667108ce4..671f84fe9 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -152,7 +152,9 @@ services.Configure(options => { options.InactivityThreshold = Ti services.Configure(options => options.Ttl = TimeSpan.FromSeconds(3600)); services.Configure(options => options.CacheDuration = TimeSpan.FromDays(1)); services.Configure(options => options.DefaultIncidentStrategy = typeof(ContinueWithIncidentsStrategy)); -services.AddHealthChecks(); +services + .AddHealthChecks() + .AddElsaReadinessChecks(includeDistributedLocks: true); services.AddControllers(); services.AddCors(cors => cors.AddDefaultPolicy(policy => policy.AllowAnyHeader().AllowAnyMethod().AllowAnyOrigin().WithExposedHeaders("*"))); @@ -168,7 +170,18 @@ if (app.Environment.IsDevelopment()) app.UseCors(); // Health checks. -app.MapHealthChecks("/"); +app.MapHealthChecks("/health/live", new() +{ + Predicate = _ => false +}); +app.MapHealthChecks("/health/ready", new() +{ + Predicate = check => check.Tags.Contains("readiness") +}); +app.MapHealthChecks("/", new() +{ + Predicate = _ => false +}); // Routing used for SignalR. app.UseRouting(); diff --git a/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj b/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj index c25e07d24..f089e7d64 100644 --- a/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj +++ b/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj @@ -10,6 +10,7 @@ + diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs new file mode 100644 index 000000000..fb3053791 --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -0,0 +1,35 @@ +using Elsa.Workflows.Runtime.HealthChecks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Elsa.Extensions; + +/// +/// Adds Elsa workflow runtime readiness checks to ASP.NET Core health checks. +/// +public static class HealthCheckExtensions +{ + private static readonly string[] ReadinessTags = ["elsa", "readiness"]; + + /// + /// Adds conservative Elsa-specific readiness probes for the workflow runtime and its core stores. + /// + /// The health checks builder. + /// Whether to probe workflow management/runtime stores. + /// Whether to probe the configured distributed lock provider. + public static IHealthChecksBuilder AddElsaReadinessChecks( + this IHealthChecksBuilder builder, + bool includePersistence = true, + bool includeDistributedLocks = false) + { + builder.AddCheck("elsa-runtime", tags: ReadinessTags); + + if (includePersistence) + builder.AddCheck("elsa-workflow-persistence", tags: ReadinessTags); + + if (includeDistributedLocks) + builder.AddCheck("elsa-distributed-locks", tags: ReadinessTags); + + return builder; + } +} diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs new file mode 100644 index 000000000..5f701e63b --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -0,0 +1,40 @@ +using Elsa.Common; +using Medallion.Threading; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Elsa.Workflows.Runtime.HealthChecks; + +/// +/// Verifies that the configured distributed lock provider can acquire and release a probe lock. +/// +public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributedLockProvider) : IHealthCheck +{ + /// + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + var lockName = $"elsa-health-check-{Guid.NewGuid():N}"; + await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, TimeSpan.Zero, cancellationToken); + if (handle == null) + { + return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary + { + ["category"] = "distributed-locks" + }); + } + + return HealthCheckResult.Healthy("Elsa distributed lock provider is reachable.", new Dictionary + { + ["category"] = "distributed-locks" + }); + } + catch (Exception e) when (!e.IsFatal()) + { + return HealthCheckResult.Unhealthy("Elsa distributed lock provider is not reachable.", e, new Dictionary + { + ["category"] = "distributed-locks" + }); + } + } +} diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs new file mode 100644 index 000000000..093512f09 --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs @@ -0,0 +1,40 @@ +using Elsa.Common; +using Elsa.Workflows.Runtime.Services; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Elsa.Workflows.Runtime.HealthChecks; + +/// +/// Reports whether the workflow runtime can create clients and is currently accepting new work. +/// +public class ElsaRuntimeHealthCheck(IWorkflowRuntime workflowRuntime, IQuiescenceSignal quiescenceSignal) : IHealthCheck +{ + /// + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + await workflowRuntime.CreateClientAsync(cancellationToken); + + var state = quiescenceSignal.CurrentState; + var data = new Dictionary + { + ["category"] = "runtime", + ["acceptingNewWork"] = state.IsAcceptingNewWork, + ["reason"] = state.Reason.ToString(), + ["activeExecutionCycles"] = quiescenceSignal.ActiveExecutionCycleCount + }; + + return state.IsAcceptingNewWork + ? HealthCheckResult.Healthy("Elsa workflow runtime is ready.", data) + : HealthCheckResult.Degraded("Elsa workflow runtime is paused or draining.", data: data); + } + catch (Exception e) when (!e.IsFatal()) + { + return HealthCheckResult.Unhealthy("Elsa workflow runtime could not create a workflow client.", e, new Dictionary + { + ["category"] = "runtime" + }); + } + } +} diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs new file mode 100644 index 000000000..84d799251 --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -0,0 +1,44 @@ +using Elsa.Common; +using Elsa.Workflows.Management; +using Elsa.Workflows.Management.Filters; +using Elsa.Workflows.Runtime.Filters; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Elsa.Workflows.Runtime.HealthChecks; + +/// +/// Performs small read-only probes against the workflow management and runtime stores. +/// +public class ElsaWorkflowPersistenceHealthCheck( + IWorkflowDefinitionStore workflowDefinitionStore, + IWorkflowInstanceStore workflowInstanceStore, + ITriggerStore triggerStore, + IBookmarkQueueStore bookmarkQueueStore) : IHealthCheck +{ + private const string ProbeId = "__elsa_health_check_probe__"; + + /// + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, cancellationToken); + await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, cancellationToken); + await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, cancellationToken); + await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, cancellationToken); + + return HealthCheckResult.Healthy("Elsa workflow stores are reachable.", new Dictionary + { + ["category"] = "persistence", + ["probes"] = "workflow-definitions,workflow-instances,triggers,bookmark-queue" + }); + } + catch (Exception e) when (!e.IsFatal()) + { + return HealthCheckResult.Unhealthy("Elsa workflow stores are not reachable.", e, new Dictionary + { + ["category"] = "persistence" + }); + } + } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs new file mode 100644 index 000000000..42814a861 --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -0,0 +1,53 @@ +using Elsa.Workflows.Runtime.HealthChecks; +using Medallion.Threading; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using NSubstitute; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class ElsaDistributedLockHealthCheckTests +{ + private readonly IDistributedLockProvider _distributedLockProvider = Substitute.For(); + private readonly IDistributedLock _distributedLock = Substitute.For(); + private readonly ElsaDistributedLockHealthCheck _sut; + + public ElsaDistributedLockHealthCheckTests() + { + _distributedLockProvider.CreateLock(Arg.Any()).Returns(_distributedLock); + _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) + .Returns(new ValueTask(Substitute.For())); + _sut = new ElsaDistributedLockHealthCheck(_distributedLockProvider); + } + + [Fact] + public async Task ReturnsHealthyWhenProbeLockCanBeAcquired() + { + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } + + [Fact] + public async Task ReturnsDegradedWhenProbeLockCannotBeAcquired() + { + _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) + .Returns(new ValueTask((IDistributedSynchronizationHandle?)null)); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } + + [Fact] + public async Task ReturnsUnhealthyWhenProviderThrows() + { + _distributedLockProvider.CreateLock(Arg.Any()).Returns(_ => throw new InvalidOperationException("lock backend unavailable")); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs new file mode 100644 index 000000000..55385acac --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs @@ -0,0 +1,57 @@ +using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Services; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using NSubstitute; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class ElsaRuntimeHealthCheckTests +{ + private readonly IWorkflowRuntime _workflowRuntime = Substitute.For(); + private readonly IQuiescenceSignal _quiescenceSignal = Substitute.For(); + private readonly ElsaRuntimeHealthCheck _sut; + + public ElsaRuntimeHealthCheckTests() + { + _workflowRuntime.CreateClientAsync(Arg.Any()).Returns(new ValueTask(Substitute.For())); + _quiescenceSignal.CurrentState.Returns(QuiescenceState.Initial("test")); + _quiescenceSignal.ActiveExecutionCycleCount.Returns(0); + _sut = new ElsaRuntimeHealthCheck(_workflowRuntime, _quiescenceSignal); + } + + [Fact] + public async Task ReturnsHealthyWhenRuntimeAcceptsNewWork() + { + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("runtime", result.Data["category"]); + Assert.True((bool)result.Data["acceptingNewWork"]); + } + + [Fact] + public async Task ReturnsDegradedWhenRuntimeIsPaused() + { + _quiescenceSignal.CurrentState.Returns(QuiescenceState.Initial("test") with + { + Reason = QuiescenceReason.AdministrativePause + }); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("AdministrativePause", result.Data["reason"]); + Assert.False((bool)result.Data["acceptingNewWork"]); + } + + [Fact] + public async Task ReturnsUnhealthyWhenRuntimeClientCannotBeCreated() + { + _workflowRuntime.CreateClientAsync(Arg.Any()).Returns>(_ => throw new InvalidOperationException("boom")); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("runtime", result.Data["category"]); + } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs new file mode 100644 index 000000000..df8f0ca3c --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -0,0 +1,46 @@ +using Elsa.Workflows.Management; +using Elsa.Workflows.Management.Filters; +using Elsa.Workflows.Runtime.Filters; +using Elsa.Workflows.Runtime.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using NSubstitute; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class ElsaWorkflowPersistenceHealthCheckTests +{ + private readonly IWorkflowDefinitionStore _workflowDefinitionStore = Substitute.For(); + private readonly IWorkflowInstanceStore _workflowInstanceStore = Substitute.For(); + private readonly ITriggerStore _triggerStore = Substitute.For(); + private readonly IBookmarkQueueStore _bookmarkQueueStore = Substitute.For(); + private readonly ElsaWorkflowPersistenceHealthCheck _sut; + + public ElsaWorkflowPersistenceHealthCheckTests() + { + _workflowDefinitionStore.FindAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(null)); + _workflowInstanceStore.CountAsync(Arg.Any(), Arg.Any()).Returns(new ValueTask(0)); + _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns(new ValueTask((Elsa.Workflows.Runtime.Entities.StoredTrigger?)null)); + _bookmarkQueueStore.FindAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(null)); + _sut = new ElsaWorkflowPersistenceHealthCheck(_workflowDefinitionStore, _workflowInstanceStore, _triggerStore, _bookmarkQueueStore); + } + + [Fact] + public async Task ReturnsHealthyWhenAllStoresCanBeRead() + { + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("persistence", result.Data["category"]); + } + + [Fact] + public async Task ReturnsUnhealthyWhenAStoreProbeFails() + { + _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("persistence", result.Data["category"]); + } +} From a8390d8d645aba9e5a00005d22b934ca733af151 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 13:47:39 +0200 Subject: [PATCH 02/20] Address health check review feedback --- doc/wiki/health-checks.md | 15 ++++++++++++- src/apps/Elsa.Server.Web/Program.cs | 9 +++++++- .../ElsaDistributedLockHealthCheck.cs | 6 ++++-- .../ElsaWorkflowPersistenceHealthCheck.cs | 21 +++++++++++++------ .../ElsaDistributedLockHealthCheckTests.cs | 3 +++ ...ElsaWorkflowPersistenceHealthCheckTests.cs | 4 +++- 6 files changed, 47 insertions(+), 11 deletions(-) diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index 42898f5e5..587bb44c2 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -14,6 +14,14 @@ services Use `includeDistributedLocks: true` when the host enables distributed runtime behavior or relies on distributed locks for workflow coordination. Leave it `false` for simple single-node hosts that do not want a lock probe. +Persistence probes are enabled by default. Set `includePersistence: false` when a host uses a custom or minimal persistence setup, or when it only needs the `elsa-runtime` readiness probe: + +```csharp +services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false); +``` + Map separate endpoints for liveness and readiness: ```csharp @@ -24,7 +32,12 @@ app.MapHealthChecks("/health/live", new() app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains("readiness") + Predicate = check => check.Tags.Contains("readiness"), + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } }); ``` diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 671f84fe9..98e0844cb 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -26,6 +26,8 @@ using Elsa.Workflows.Runtime.Distributed.Extensions; using Elsa.Workflows.Runtime.Options; using Elsa.Workflows.Runtime.Tasks; using JetBrains.Annotations; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; // ReSharper disable RedundantAssignment @@ -176,7 +178,12 @@ app.MapHealthChecks("/health/live", new() }); app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains("readiness") + Predicate = check => check.Tags.Contains("readiness"), + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } }); app.MapHealthChecks("/", new() { diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index 5f701e63b..ce7abe68c 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -9,13 +9,15 @@ namespace Elsa.Workflows.Runtime.HealthChecks; /// public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributedLockProvider) : IHealthCheck { + private const string LockName = "elsa-health-check"; + private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1); + /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { try { - var lockName = $"elsa-health-check-{Guid.NewGuid():N}"; - await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, TimeSpan.Zero, cancellationToken); + await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, LockAcquisitionTimeout, cancellationToken); if (handle == null) { return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 84d799251..0a735c9b5 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -20,12 +20,14 @@ public class ElsaWorkflowPersistenceHealthCheck( /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { + var failedStore = ""; + try { - await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, cancellationToken); - await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, cancellationToken); - await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, cancellationToken); - await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, cancellationToken); + await ProbeAsync("workflow-definitions", async ct => await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)); + await ProbeAsync("workflow-instances", async ct => await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)); + await ProbeAsync("triggers", async ct => await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, ct)); + await ProbeAsync("bookmark-queue", async ct => await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)); return HealthCheckResult.Healthy("Elsa workflow stores are reachable.", new Dictionary { @@ -35,10 +37,17 @@ public class ElsaWorkflowPersistenceHealthCheck( } catch (Exception e) when (!e.IsFatal()) { - return HealthCheckResult.Unhealthy("Elsa workflow stores are not reachable.", e, new Dictionary + return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", e, new Dictionary { - ["category"] = "persistence" + ["category"] = "persistence", + ["failedStore"] = failedStore }); } + + async Task ProbeAsync(string store, Func probe) + { + failedStore = store; + await probe(cancellationToken); + } } } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index 42814a861..e531a09b5 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -7,6 +7,7 @@ namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; public class ElsaDistributedLockHealthCheckTests { + private static readonly TimeSpan ExpectedLockAcquisitionTimeout = TimeSpan.FromSeconds(1); private readonly IDistributedLockProvider _distributedLockProvider = Substitute.For(); private readonly IDistributedLock _distributedLock = Substitute.For(); private readonly ElsaDistributedLockHealthCheck _sut; @@ -26,6 +27,8 @@ public class ElsaDistributedLockHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("distributed-locks", result.Data["category"]); + _distributedLockProvider.Received(1).CreateLock("elsa-health-check"); + await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any()); } [Fact] diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index df8f0ca3c..893bb3c5a 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -34,13 +34,15 @@ public class ElsaWorkflowPersistenceHealthCheckTests } [Fact] - public async Task ReturnsUnhealthyWhenAStoreProbeFails() + public async Task ReturnsUnhealthyWithFailedStoreWhenAStoreProbeFails() { _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); var result = await _sut.CheckHealthAsync(new HealthCheckContext()); Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("Elsa workflow store 'triggers' is not reachable.", result.Description); Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("triggers", result.Data["failedStore"]); } } From 481c1aaa817f0c462adad617e2ffe718ec1e6907 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 13:51:43 +0200 Subject: [PATCH 03/20] Guard persistence health check store probes --- .../ElsaWorkflowPersistenceHealthCheck.cs | 58 +++++++++++++------ ...ElsaWorkflowPersistenceHealthCheckTests.cs | 22 ++++++- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 0a735c9b5..6d0b1a52e 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -2,6 +2,7 @@ using Elsa.Common; using Elsa.Workflows.Management; using Elsa.Workflows.Management.Filters; using Elsa.Workflows.Runtime.Filters; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; namespace Elsa.Workflows.Runtime.HealthChecks; @@ -9,11 +10,7 @@ namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Performs small read-only probes against the workflow management and runtime stores. /// -public class ElsaWorkflowPersistenceHealthCheck( - IWorkflowDefinitionStore workflowDefinitionStore, - IWorkflowInstanceStore workflowInstanceStore, - ITriggerStore triggerStore, - IBookmarkQueueStore bookmarkQueueStore) : IHealthCheck +public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider) : IHealthCheck { private const string ProbeId = "__elsa_health_check_probe__"; @@ -21,33 +18,58 @@ public class ElsaWorkflowPersistenceHealthCheck( public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { var failedStore = ""; + var probes = new List(); + var skippedProbes = new List(); try { - await ProbeAsync("workflow-definitions", async ct => await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)); - await ProbeAsync("workflow-instances", async ct => await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)); - await ProbeAsync("triggers", async ct => await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, ct)); - await ProbeAsync("bookmark-queue", async ct => await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)); + await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)); + await ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)); + await ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)); + await ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)); - return HealthCheckResult.Healthy("Elsa workflow stores are reachable.", new Dictionary - { - ["category"] = "persistence", - ["probes"] = "workflow-definitions,workflow-instances,triggers,bookmark-queue" - }); + var data = CreateData(); + return probes.Count == 0 + ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: data) + : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", data); } catch (Exception e) when (!e.IsFatal()) { return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", e, new Dictionary { ["category"] = "persistence", - ["failedStore"] = failedStore + ["failedStore"] = failedStore, + ["failedProbe"] = failedStore }); } - async Task ProbeAsync(string store, Func probe) + async Task ProbeAsync(string storeName, TStore? store, Func probe) where TStore : class { - failedStore = store; - await probe(cancellationToken); + if (store == null) + { + skippedProbes.Add(storeName); + return; + } + + failedStore = storeName; + probes.Add(storeName); + await probe(store, cancellationToken); + } + + Dictionary CreateData() + { + var data = new Dictionary + { + ["category"] = "persistence" + }; + + if (probes.Count > 0) + data["probes"] = string.Join(",", probes); + + if (skippedProbes.Count > 0) + data["skippedProbes"] = string.Join(",", skippedProbes); + + return data; } } } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 893bb3c5a..705bb7f5b 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -9,6 +9,7 @@ namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; public class ElsaWorkflowPersistenceHealthCheckTests { + private readonly IServiceProvider _serviceProvider = Substitute.For(); private readonly IWorkflowDefinitionStore _workflowDefinitionStore = Substitute.For(); private readonly IWorkflowInstanceStore _workflowInstanceStore = Substitute.For(); private readonly ITriggerStore _triggerStore = Substitute.For(); @@ -21,7 +22,11 @@ public class ElsaWorkflowPersistenceHealthCheckTests _workflowInstanceStore.CountAsync(Arg.Any(), Arg.Any()).Returns(new ValueTask(0)); _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns(new ValueTask((Elsa.Workflows.Runtime.Entities.StoredTrigger?)null)); _bookmarkQueueStore.FindAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(null)); - _sut = new ElsaWorkflowPersistenceHealthCheck(_workflowDefinitionStore, _workflowInstanceStore, _triggerStore, _bookmarkQueueStore); + _serviceProvider.GetService(typeof(IWorkflowDefinitionStore)).Returns(_workflowDefinitionStore); + _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns(_workflowInstanceStore); + _serviceProvider.GetService(typeof(ITriggerStore)).Returns(_triggerStore); + _serviceProvider.GetService(typeof(IBookmarkQueueStore)).Returns(_bookmarkQueueStore); + _sut = new ElsaWorkflowPersistenceHealthCheck(_serviceProvider); } [Fact] @@ -44,5 +49,20 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("Elsa workflow store 'triggers' is not reachable.", result.Description); Assert.Equal("persistence", result.Data["category"]); Assert.Equal("triggers", result.Data["failedStore"]); + Assert.Equal("triggers", result.Data["failedProbe"]); + } + + [Fact] + public async Task ReturnsHealthyWithSkippedProbesWhenOptionalManagementStoresAreMissing() + { + _serviceProvider.GetService(typeof(IWorkflowDefinitionStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns((object?)null); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("triggers,bookmark-queue", result.Data["probes"]); + Assert.Equal("workflow-definitions,workflow-instances", result.Data["skippedProbes"]); } } From f12204496cc82b271dddf035ac51ac8e03c4b232 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 13:59:56 +0200 Subject: [PATCH 04/20] Avoid shared distributed lock probe names --- .../HealthChecks/ElsaDistributedLockHealthCheck.cs | 2 +- .../HealthChecks/ElsaDistributedLockHealthCheckTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index ce7abe68c..756089f3c 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -9,7 +9,7 @@ namespace Elsa.Workflows.Runtime.HealthChecks; /// public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributedLockProvider) : IHealthCheck { - private const string LockName = "elsa-health-check"; + private static readonly string LockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1); /// diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index e531a09b5..297eaa90a 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -27,7 +27,7 @@ public class ElsaDistributedLockHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("distributed-locks", result.Data["category"]); - _distributedLockProvider.Received(1).CreateLock("elsa-health-check"); + _distributedLockProvider.Received(1).CreateLock(Arg.Is(x => x.StartsWith("elsa-health-check-", StringComparison.Ordinal))); await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any()); } From d6903fdb76cc9035418d7a2c7415ad9615455f56 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 23:33:13 +0200 Subject: [PATCH 05/20] docs: clarify readiness probe timeout --- doc/wiki/health-checks.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index 587bb44c2..e5e1a844e 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -65,8 +65,11 @@ readinessProbe: httpGet: path: /health/ready port: http + timeoutSeconds: 2 periodSeconds: 10 failureThreshold: 3 ``` Do not point liveness at Elsa readiness checks. A paused, draining, or temporarily database-degraded runtime should be removed from service by readiness without forcing Kubernetes to restart the process. + +When `includeDistributedLocks: true` is enabled, set the readiness probe `timeoutSeconds` higher than the distributed-lock acquisition timeout. The built-in distributed-lock readiness check currently waits up to 1 second to acquire its probe lock, so the example uses `timeoutSeconds: 2` to avoid intermittent Kubernetes probe timeouts while the lock provider is still healthy. From 9cdff8c318159624eb3f0a24ad03665170aeaba7 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 23:39:35 +0200 Subject: [PATCH 06/20] Handle missing distributed lock provider in readiness check --- .../ElsaDistributedLockHealthCheck.cs | 12 +++++++++- .../ElsaDistributedLockHealthCheckTests.cs | 24 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index 756089f3c..c585ce685 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -1,5 +1,6 @@ using Elsa.Common; using Medallion.Threading; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; namespace Elsa.Workflows.Runtime.HealthChecks; @@ -7,7 +8,7 @@ namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Verifies that the configured distributed lock provider can acquire and release a probe lock. /// -public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributedLockProvider) : IHealthCheck +public class ElsaDistributedLockHealthCheck(IServiceProvider serviceProvider) : IHealthCheck { private static readonly string LockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1); @@ -17,6 +18,15 @@ public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributed { try { + var distributedLockProvider = serviceProvider.GetService(); + if (distributedLockProvider == null) + { + return HealthCheckResult.Degraded("Elsa distributed lock provider is not registered.", data: new Dictionary + { + ["category"] = "distributed-locks" + }); + } + await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, LockAcquisitionTimeout, cancellationToken); if (handle == null) { diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index 297eaa90a..bfebf682e 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -1,5 +1,6 @@ using Elsa.Workflows.Runtime.HealthChecks; using Medallion.Threading; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using NSubstitute; @@ -17,7 +18,7 @@ public class ElsaDistributedLockHealthCheckTests _distributedLockProvider.CreateLock(Arg.Any()).Returns(_distributedLock); _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) .Returns(new ValueTask(Substitute.For())); - _sut = new ElsaDistributedLockHealthCheck(_distributedLockProvider); + _sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(_distributedLockProvider)); } [Fact] @@ -53,4 +54,25 @@ public class ElsaDistributedLockHealthCheckTests Assert.Equal(HealthStatus.Unhealthy, result.Status); Assert.Equal("distributed-locks", result.Data["category"]); } + + [Fact] + public async Task ReturnsDegradedWhenProviderIsNotRegistered() + { + var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider()); + + var result = await sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } + + private static IServiceProvider CreateServiceProvider(IDistributedLockProvider? distributedLockProvider = null) + { + var services = new ServiceCollection(); + + if (distributedLockProvider != null) + services.AddSingleton(distributedLockProvider); + + return services.BuildServiceProvider(); + } } From 182f524d8332f8d306489fc184c406b519c582ed Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Wed, 20 May 2026 23:43:25 +0200 Subject: [PATCH 07/20] Harden readiness health check reporting --- src/apps/Elsa.Server.Web/Program.cs | 1 - .../HealthChecks/ElsaDistributedLockHealthCheck.cs | 6 ++++-- .../HealthChecks/ElsaRuntimeHealthCheck.cs | 6 ++++-- .../HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs | 9 ++++++--- .../HealthChecks/ElsaDistributedLockHealthCheckTests.cs | 5 +++-- .../HealthChecks/ElsaRuntimeHealthCheckTests.cs | 3 ++- .../ElsaWorkflowPersistenceHealthCheckTests.cs | 3 ++- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 98e0844cb..98483f597 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -26,7 +26,6 @@ using Elsa.Workflows.Runtime.Distributed.Extensions; using Elsa.Workflows.Runtime.Options; using Elsa.Workflows.Runtime.Tasks; using JetBrains.Annotations; -using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index c585ce685..80a940763 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -2,13 +2,14 @@ using Elsa.Common; using Medallion.Threading; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Verifies that the configured distributed lock provider can acquire and release a probe lock. /// -public class ElsaDistributedLockHealthCheck(IServiceProvider serviceProvider) : IHealthCheck +public class ElsaDistributedLockHealthCheck(IServiceProvider serviceProvider, ILogger logger) : IHealthCheck { private static readonly string LockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1); @@ -43,7 +44,8 @@ public class ElsaDistributedLockHealthCheck(IServiceProvider serviceProvider) : } catch (Exception e) when (!e.IsFatal()) { - return HealthCheckResult.Unhealthy("Elsa distributed lock provider is not reachable.", e, new Dictionary + logger.LogWarning(e, "Elsa distributed lock provider is not reachable."); + return HealthCheckResult.Unhealthy("Elsa distributed lock provider is not reachable.", data: new Dictionary { ["category"] = "distributed-locks" }); diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs index 093512f09..0f5d34de0 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs @@ -1,13 +1,14 @@ using Elsa.Common; using Elsa.Workflows.Runtime.Services; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Reports whether the workflow runtime can create clients and is currently accepting new work. /// -public class ElsaRuntimeHealthCheck(IWorkflowRuntime workflowRuntime, IQuiescenceSignal quiescenceSignal) : IHealthCheck +public class ElsaRuntimeHealthCheck(IWorkflowRuntime workflowRuntime, IQuiescenceSignal quiescenceSignal, ILogger logger) : IHealthCheck { /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) @@ -31,7 +32,8 @@ public class ElsaRuntimeHealthCheck(IWorkflowRuntime workflowRuntime, IQuiescenc } catch (Exception e) when (!e.IsFatal()) { - return HealthCheckResult.Unhealthy("Elsa workflow runtime could not create a workflow client.", e, new Dictionary + logger.LogWarning(e, "Elsa workflow runtime could not create a workflow client."); + return HealthCheckResult.Unhealthy("Elsa workflow runtime could not create a workflow client.", data: new Dictionary { ["category"] = "runtime" }); diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 6d0b1a52e..aab33bedd 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -4,13 +4,14 @@ using Elsa.Workflows.Management.Filters; using Elsa.Workflows.Runtime.Filters; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Performs small read-only probes against the workflow management and runtime stores. /// -public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider) : IHealthCheck +public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider, ILogger logger) : IHealthCheck { private const string ProbeId = "__elsa_health_check_probe__"; @@ -35,7 +36,8 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider } catch (Exception e) when (!e.IsFatal()) { - return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", e, new Dictionary + logger.LogWarning(e, "Elsa workflow store {StoreName} is not reachable.", failedStore); + return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", data: new Dictionary { ["category"] = "persistence", ["failedStore"] = failedStore, @@ -45,13 +47,14 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider async Task ProbeAsync(string storeName, TStore? store, Func probe) where TStore : class { + failedStore = storeName; + if (store == null) { skippedProbes.Add(storeName); return; } - failedStore = storeName; probes.Add(storeName); await probe(store, cancellationToken); } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index bfebf682e..8f744fb8e 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -2,6 +2,7 @@ using Elsa.Workflows.Runtime.HealthChecks; using Medallion.Threading; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; @@ -18,7 +19,7 @@ public class ElsaDistributedLockHealthCheckTests _distributedLockProvider.CreateLock(Arg.Any()).Returns(_distributedLock); _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) .Returns(new ValueTask(Substitute.For())); - _sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(_distributedLockProvider)); + _sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(_distributedLockProvider), NullLogger.Instance); } [Fact] @@ -58,7 +59,7 @@ public class ElsaDistributedLockHealthCheckTests [Fact] public async Task ReturnsDegradedWhenProviderIsNotRegistered() { - var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider()); + var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(), NullLogger.Instance); var result = await sut.CheckHealthAsync(new HealthCheckContext()); diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs index 55385acac..445ff4c3d 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs @@ -1,6 +1,7 @@ using Elsa.Workflows.Runtime.HealthChecks; using Elsa.Workflows.Runtime.Services; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; @@ -16,7 +17,7 @@ public class ElsaRuntimeHealthCheckTests _workflowRuntime.CreateClientAsync(Arg.Any()).Returns(new ValueTask(Substitute.For())); _quiescenceSignal.CurrentState.Returns(QuiescenceState.Initial("test")); _quiescenceSignal.ActiveExecutionCycleCount.Returns(0); - _sut = new ElsaRuntimeHealthCheck(_workflowRuntime, _quiescenceSignal); + _sut = new ElsaRuntimeHealthCheck(_workflowRuntime, _quiescenceSignal, NullLogger.Instance); } [Fact] diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 705bb7f5b..3834263dd 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -3,6 +3,7 @@ using Elsa.Workflows.Management.Filters; using Elsa.Workflows.Runtime.Filters; using Elsa.Workflows.Runtime.HealthChecks; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; @@ -26,7 +27,7 @@ public class ElsaWorkflowPersistenceHealthCheckTests _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns(_workflowInstanceStore); _serviceProvider.GetService(typeof(ITriggerStore)).Returns(_triggerStore); _serviceProvider.GetService(typeof(IBookmarkQueueStore)).Returns(_bookmarkQueueStore); - _sut = new ElsaWorkflowPersistenceHealthCheck(_serviceProvider); + _sut = new ElsaWorkflowPersistenceHealthCheck(_serviceProvider, NullLogger.Instance); } [Fact] From f149c27ab03438e5f460304a19103f45467796a2 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 00:09:46 +0200 Subject: [PATCH 08/20] Address health check review feedback --- doc/wiki/health-checks.md | 14 ++++- src/apps/Elsa.Server.Web/Program.cs | 2 +- .../Extensions/HealthCheckExtensions.cs | 15 ++++- .../ElsaDistributedLockHealthCheck.cs | 10 ++- .../ElsaWorkflowPersistenceHealthCheck.cs | 61 ++++++++++--------- .../ElsaReadinessHealthCheckOptions.cs | 12 ++++ .../ElsaDistributedLockHealthCheckTests.cs | 13 +++- ...ElsaWorkflowPersistenceHealthCheckTests.cs | 17 ++++++ 8 files changed, 103 insertions(+), 41 deletions(-) create mode 100644 src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index e5e1a844e..a36d567a4 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -7,9 +7,13 @@ Elsa hosts can opt in to Elsa-specific readiness checks on top of the normal ASP Register ASP.NET Core health checks and then add Elsa readiness checks: ```csharp +using System; + services .AddHealthChecks() - .AddElsaReadinessChecks(includeDistributedLocks: true); + .AddElsaReadinessChecks( + includeDistributedLocks: true, + configureOptions: options => options.DistributedLockAcquisitionTimeout = TimeSpan.FromSeconds(1)); ``` Use `includeDistributedLocks: true` when the host enables distributed runtime behavior or relies on distributed locks for workflow coordination. Leave it `false` for simple single-node hosts that do not want a lock probe. @@ -25,6 +29,10 @@ services Map separate endpoints for liveness and readiness: ```csharp +using Elsa.Extensions; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Diagnostics.HealthChecks; + app.MapHealthChecks("/health/live", new() { Predicate = _ => false @@ -32,7 +40,7 @@ app.MapHealthChecks("/health/live", new() app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains("readiness"), + Predicate = check => check.Tags.Contains(HealthCheckExtensions.ReadinessTag), ResultStatusCodes = { [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, @@ -72,4 +80,4 @@ readinessProbe: Do not point liveness at Elsa readiness checks. A paused, draining, or temporarily database-degraded runtime should be removed from service by readiness without forcing Kubernetes to restart the process. -When `includeDistributedLocks: true` is enabled, set the readiness probe `timeoutSeconds` higher than the distributed-lock acquisition timeout. The built-in distributed-lock readiness check currently waits up to 1 second to acquire its probe lock, so the example uses `timeoutSeconds: 2` to avoid intermittent Kubernetes probe timeouts while the lock provider is still healthy. +When `includeDistributedLocks: true` is enabled, set the readiness probe `timeoutSeconds` higher than the configured distributed-lock acquisition timeout. The built-in distributed-lock readiness check waits up to 1 second by default to acquire its probe lock, so the example uses `timeoutSeconds: 2` to avoid intermittent Kubernetes probe timeouts while the lock provider is still healthy. diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 98483f597..3990bf81d 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -177,7 +177,7 @@ app.MapHealthChecks("/health/live", new() }); app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains("readiness"), + Predicate = check => check.Tags.Contains(HealthCheckExtensions.ReadinessTag), ResultStatusCodes = { [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs index fb3053791..6c9b7195d 100644 --- a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -1,4 +1,5 @@ using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Options; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -9,7 +10,12 @@ namespace Elsa.Extensions; /// public static class HealthCheckExtensions { - private static readonly string[] ReadinessTags = ["elsa", "readiness"]; + /// + /// Tag applied to Elsa readiness checks. + /// + public const string ReadinessTag = "readiness"; + + private static readonly string[] ReadinessTags = ["elsa", ReadinessTag]; /// /// Adds conservative Elsa-specific readiness probes for the workflow runtime and its core stores. @@ -17,11 +23,16 @@ public static class HealthCheckExtensions /// The health checks builder. /// Whether to probe workflow management/runtime stores. /// Whether to probe the configured distributed lock provider. + /// Configures Elsa readiness probe behavior. public static IHealthChecksBuilder AddElsaReadinessChecks( this IHealthChecksBuilder builder, bool includePersistence = true, - bool includeDistributedLocks = false) + bool includeDistributedLocks = false, + Action? configureOptions = null) { + if (configureOptions != null) + builder.Services.Configure(configureOptions); + builder.AddCheck("elsa-runtime", tags: ReadinessTags); if (includePersistence) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index 80a940763..b794d5675 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -1,18 +1,22 @@ using Elsa.Common; +using Elsa.Workflows.Runtime.Options; using Medallion.Threading; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Verifies that the configured distributed lock provider can acquire and release a probe lock. /// -public class ElsaDistributedLockHealthCheck(IServiceProvider serviceProvider, ILogger logger) : IHealthCheck +public class ElsaDistributedLockHealthCheck( + IServiceProvider serviceProvider, + IOptions options, + ILogger logger) : IHealthCheck { private static readonly string LockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; - private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1); /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) @@ -28,7 +32,7 @@ public class ElsaDistributedLockHealthCheck(IServiceProvider serviceProvider, IL }); } - await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, LockAcquisitionTimeout, cancellationToken); + await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, options.Value.DistributedLockAcquisitionTimeout, cancellationToken); if (handle == null) { return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index aab33bedd..9ee15a50c 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -18,45 +18,46 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { - var failedStore = ""; - var probes = new List(); - var skippedProbes = new List(); + var probeResults = await Task.WhenAll( + ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)), + ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)), + ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)), + ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct))); - try + var probes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); + var skippedProbes = probeResults.Where(x => x.Skipped).Select(x => x.StoreName).ToList(); + var failedProbe = probeResults.FirstOrDefault(x => x.Exception != null); + if (failedProbe != null) { - await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)); - await ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)); - await ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)); - await ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)); - var data = CreateData(); - return probes.Count == 0 - ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: data) - : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", data); - } - catch (Exception e) when (!e.IsFatal()) - { - logger.LogWarning(e, "Elsa workflow store {StoreName} is not reachable.", failedStore); - return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", data: new Dictionary - { - ["category"] = "persistence", - ["failedStore"] = failedStore, - ["failedProbe"] = failedStore - }); + data["failedStore"] = failedProbe.StoreName; + data["failedProbe"] = failedProbe.StoreName; + + logger.LogWarning(failedProbe.Exception, "Elsa workflow store {StoreName} is not reachable.", failedProbe.StoreName); + return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedProbe.StoreName}' is not reachable.", data: data); } - async Task ProbeAsync(string storeName, TStore? store, Func probe) where TStore : class - { - failedStore = storeName; + var healthyData = CreateData(); + return probes.Count == 0 + ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: healthyData) + : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", healthyData); + async Task ProbeAsync(string storeName, TStore? store, Func probe) where TStore : class + { if (store == null) { - skippedProbes.Add(storeName); - return; + return new ProbeResult(storeName, true, null); } - probes.Add(storeName); - await probe(store, cancellationToken); + try + { + await probe(store, cancellationToken); + return new ProbeResult(storeName, false, null); + } + catch (Exception e) when (!e.IsFatal()) + { + return new ProbeResult(storeName, false, e); + } } Dictionary CreateData() @@ -75,4 +76,6 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider return data; } } + + private sealed record ProbeResult(string StoreName, bool Skipped, Exception? Exception); } diff --git a/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs b/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs new file mode 100644 index 000000000..351531bbe --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs @@ -0,0 +1,12 @@ +namespace Elsa.Workflows.Runtime.Options; + +/// +/// Configures Elsa workflow runtime readiness health checks. +/// +public class ElsaReadinessHealthCheckOptions +{ + /// + /// The maximum time the distributed-lock readiness probe waits to acquire its probe lock. + /// + public TimeSpan DistributedLockAcquisitionTimeout { get; set; } = TimeSpan.FromSeconds(1); +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index 8f744fb8e..a3d35614c 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -1,15 +1,17 @@ using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Options; using Medallion.Threading; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using NSubstitute; namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; public class ElsaDistributedLockHealthCheckTests { - private static readonly TimeSpan ExpectedLockAcquisitionTimeout = TimeSpan.FromSeconds(1); + private static readonly TimeSpan ExpectedLockAcquisitionTimeout = TimeSpan.FromMilliseconds(250); private readonly IDistributedLockProvider _distributedLockProvider = Substitute.For(); private readonly IDistributedLock _distributedLock = Substitute.For(); private readonly ElsaDistributedLockHealthCheck _sut; @@ -19,7 +21,7 @@ public class ElsaDistributedLockHealthCheckTests _distributedLockProvider.CreateLock(Arg.Any()).Returns(_distributedLock); _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) .Returns(new ValueTask(Substitute.For())); - _sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(_distributedLockProvider), NullLogger.Instance); + _sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(_distributedLockProvider), CreateOptions(), NullLogger.Instance); } [Fact] @@ -59,7 +61,7 @@ public class ElsaDistributedLockHealthCheckTests [Fact] public async Task ReturnsDegradedWhenProviderIsNotRegistered() { - var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(), NullLogger.Instance); + var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(), CreateOptions(), NullLogger.Instance); var result = await sut.CheckHealthAsync(new HealthCheckContext()); @@ -76,4 +78,9 @@ public class ElsaDistributedLockHealthCheckTests return services.BuildServiceProvider(); } + + private static IOptions CreateOptions() => Microsoft.Extensions.Options.Options.Create(new ElsaReadinessHealthCheckOptions + { + DistributedLockAcquisitionTimeout = ExpectedLockAcquisitionTimeout + }); } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 3834263dd..457668082 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -66,4 +66,21 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("triggers,bookmark-queue", result.Data["probes"]); Assert.Equal("workflow-definitions,workflow-instances", result.Data["skippedProbes"]); } + + [Fact] + public async Task ReturnsDegradedWithSkippedProbesWhenNoStoresAreRegistered() + { + _serviceProvider.GetService(typeof(IWorkflowDefinitionStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(ITriggerStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(IBookmarkQueueStore)).Returns((object?)null); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("No Elsa workflow persistence stores are registered.", result.Description); + Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["skippedProbes"]); + Assert.False(result.Data.ContainsKey("probes")); + } } From b5c457c202f3f8c3669063eaea8443c25f7cc164 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 00:24:16 +0200 Subject: [PATCH 09/20] Preserve health check cancellation semantics --- .../HealthChecks/ElsaDistributedLockHealthCheck.cs | 4 ++++ .../HealthChecks/ElsaRuntimeHealthCheck.cs | 4 ++++ .../HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index b794d5675..fe67a5d2f 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -46,6 +46,10 @@ public class ElsaDistributedLockHealthCheck( ["category"] = "distributed-locks" }); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception e) when (!e.IsFatal()) { logger.LogWarning(e, "Elsa distributed lock provider is not reachable."); diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs index 0f5d34de0..cfdfb7396 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs @@ -30,6 +30,10 @@ public class ElsaRuntimeHealthCheck(IWorkflowRuntime workflowRuntime, IQuiescenc ? HealthCheckResult.Healthy("Elsa workflow runtime is ready.", data) : HealthCheckResult.Degraded("Elsa workflow runtime is paused or draining.", data: data); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception e) when (!e.IsFatal()) { logger.LogWarning(e, "Elsa workflow runtime could not create a workflow client."); diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 9ee15a50c..e86346e4c 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -54,6 +54,10 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider await probe(store, cancellationToken); return new ProbeResult(storeName, false, null); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception e) when (!e.IsFatal()) { return new ProbeResult(storeName, false, e); From 45d8a4cfae89c9dcfd722030355dd7cfaf426380 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 00:47:34 +0200 Subject: [PATCH 10/20] Remove unused health check usings --- .../HealthChecks/ElsaRuntimeHealthCheck.cs | 1 - .../HealthChecks/ElsaRuntimeHealthCheckTests.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs index cfdfb7396..63f07a0ea 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs @@ -1,5 +1,4 @@ using Elsa.Common; -using Elsa.Workflows.Runtime.Services; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs index 445ff4c3d..ec00da511 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs @@ -1,5 +1,4 @@ using Elsa.Workflows.Runtime.HealthChecks; -using Elsa.Workflows.Runtime.Services; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; From d2c2e5c4297ab1290e6fc5f016fe45a9e0e597ef Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 00:56:37 +0200 Subject: [PATCH 11/20] Refine readiness health check probe data --- .../Extensions/HealthCheckExtensions.cs | 4 +++- .../ElsaWorkflowPersistenceHealthCheck.cs | 14 +++++++++----- .../ElsaWorkflowPersistenceHealthCheckTests.cs | 3 +++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs index 6c9b7195d..8d636d417 100644 --- a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -30,8 +30,10 @@ public static class HealthCheckExtensions bool includeDistributedLocks = false, Action? configureOptions = null) { + var optionsBuilder = builder.Services.AddOptions(); + if (configureOptions != null) - builder.Services.Configure(configureOptions); + optionsBuilder.Configure(configureOptions); builder.AddCheck("elsa-runtime", tags: ReadinessTags); diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index e86346e4c..0d4679949 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -13,7 +13,7 @@ namespace Elsa.Workflows.Runtime.HealthChecks; /// public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider, ILogger logger) : IHealthCheck { - private const string ProbeId = "__elsa_health_check_probe__"; + private const string ProbeId = "00000000-0000-0000-0000-000000000000"; /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) @@ -24,7 +24,8 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)), ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct))); - var probes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); + var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); + var successfulProbes = probeResults.Where(x => !x.Skipped && x.Exception == null).Select(x => x.StoreName).ToList(); var skippedProbes = probeResults.Where(x => x.Skipped).Select(x => x.StoreName).ToList(); var failedProbe = probeResults.FirstOrDefault(x => x.Exception != null); if (failedProbe != null) @@ -38,7 +39,7 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider } var healthyData = CreateData(); - return probes.Count == 0 + return attemptedProbes.Count == 0 ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: healthyData) : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", healthyData); @@ -71,8 +72,11 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider ["category"] = "persistence" }; - if (probes.Count > 0) - data["probes"] = string.Join(",", probes); + if (successfulProbes.Count > 0) + data["probes"] = string.Join(",", successfulProbes); + + if (attemptedProbes.Count > 0) + data["attemptedProbes"] = string.Join(",", attemptedProbes); if (skippedProbes.Count > 0) data["skippedProbes"] = string.Join(",", skippedProbes); diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 457668082..7d6192334 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -51,6 +51,8 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("persistence", result.Data["category"]); Assert.Equal("triggers", result.Data["failedStore"]); Assert.Equal("triggers", result.Data["failedProbe"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); + Assert.Equal("workflow-definitions,workflow-instances,bookmark-queue", result.Data["probes"]); } [Fact] @@ -64,6 +66,7 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("persistence", result.Data["category"]); Assert.Equal("triggers,bookmark-queue", result.Data["probes"]); + Assert.Equal("triggers,bookmark-queue", result.Data["attemptedProbes"]); Assert.Equal("workflow-definitions,workflow-instances", result.Data["skippedProbes"]); } From 2ade23bc827f1345b32e39f1d8bd822fee4b0732 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 22:56:42 +0000 Subject: [PATCH 12/20] Address health check review feedback Agent-Logs-Url: https://github.com/elsa-workflows/elsa-core/sessions/71c9cf76-ccac-4020-87f4-1e4122d34645 Co-authored-by: sfmskywalker <938393+sfmskywalker@users.noreply.github.com> --- .../ElsaWorkflowPersistenceHealthCheck.cs | 4 ++++ ...ElsaWorkflowPersistenceHealthCheckTests.cs | 8 ++++++- .../HealthCheckExtensionsTests.cs | 23 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 0d4679949..55d5478fb 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -27,6 +27,7 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); var successfulProbes = probeResults.Where(x => !x.Skipped && x.Exception == null).Select(x => x.StoreName).ToList(); var skippedProbes = probeResults.Where(x => x.Skipped).Select(x => x.StoreName).ToList(); + var failedProbes = probeResults.Where(x => x.Exception != null).Select(x => x.StoreName).ToList(); var failedProbe = probeResults.FirstOrDefault(x => x.Exception != null); if (failedProbe != null) { @@ -78,6 +79,9 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider if (attemptedProbes.Count > 0) data["attemptedProbes"] = string.Join(",", attemptedProbes); + if (failedProbes.Count > 0) + data["failedProbes"] = string.Join(",", failedProbes); + if (skippedProbes.Count > 0) data["skippedProbes"] = string.Join(",", skippedProbes); diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 7d6192334..02fcd8f11 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -37,6 +37,11 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["probes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); + await _workflowDefinitionStore.Received(1).FindAsync( + Arg.Is(x => x.Id == "00000000-0000-0000-0000-000000000000"), + Arg.Any()); } [Fact] @@ -51,8 +56,9 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("persistence", result.Data["category"]); Assert.Equal("triggers", result.Data["failedStore"]); Assert.Equal("triggers", result.Data["failedProbe"]); - Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); Assert.Equal("workflow-definitions,workflow-instances,bookmark-queue", result.Data["probes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); + Assert.Equal("triggers", result.Data["failedProbes"]); } [Fact] diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs new file mode 100644 index 000000000..5bb78efc4 --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs @@ -0,0 +1,23 @@ +using Elsa.Extensions; +using Elsa.Workflows.Runtime.Options; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class HealthCheckExtensionsTests +{ + [Fact] + public void AddElsaReadinessChecksRegistersReadinessOptions() + { + var services = new ServiceCollection(); + + services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false, includeDistributedLocks: true); + + using var serviceProvider = services.BuildServiceProvider(); + + Assert.NotNull(serviceProvider.GetRequiredService>()); + } +} From bcccb80ad57ab5e34bf0aaccc892cf5aa64932ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 23:02:49 +0000 Subject: [PATCH 13/20] Use unique distributed lock health probes Agent-Logs-Url: https://github.com/elsa-workflows/elsa-core/sessions/71c9cf76-ccac-4020-87f4-1e4122d34645 Co-authored-by: sfmskywalker <938393+sfmskywalker@users.noreply.github.com> --- .../ElsaDistributedLockHealthCheck.cs | 5 ++--- .../ElsaDistributedLockHealthCheckTests.cs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index fe67a5d2f..3d1a4b1a2 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -16,8 +16,6 @@ public class ElsaDistributedLockHealthCheck( IOptions options, ILogger logger) : IHealthCheck { - private static readonly string LockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; - /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { @@ -32,7 +30,8 @@ public class ElsaDistributedLockHealthCheck( }); } - await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, options.Value.DistributedLockAcquisitionTimeout, cancellationToken); + var lockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; + await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, options.Value.DistributedLockAcquisitionTimeout, cancellationToken); if (handle == null) { return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index a3d35614c..ee7400b6a 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -35,6 +35,24 @@ public class ElsaDistributedLockHealthCheckTests await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any()); } + [Fact] + public async Task UsesUniqueProbeLockNameForEachCheck() + { + var lockNames = new List(); + var distributedLockProvider = Substitute.For(); + var distributedLock = Substitute.For(); + distributedLockProvider.CreateLock(Arg.Do(lockNames.Add)).Returns(distributedLock); + distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) + .Returns(new ValueTask(Substitute.For())); + var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(distributedLockProvider), CreateOptions(), NullLogger.Instance); + + await sut.CheckHealthAsync(new HealthCheckContext()); + await sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(2, lockNames.Count); + Assert.NotEqual(lockNames[0], lockNames[1]); + } + [Fact] public async Task ReturnsDegradedWhenProbeLockCannotBeAcquired() { From a0d6f6b24b55010edf14b2a7903334fc68a33a82 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 01:16:36 +0200 Subject: [PATCH 14/20] Address health check review feedback --- src/apps/Elsa.Server.Web/Program.cs | 6 +- .../ElsaWorkflowPersistenceHealthCheck.cs | 12 ++-- ...ElsaWorkflowPersistenceHealthCheckTests.cs | 59 +++++++++++++++++++ 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 3990bf81d..dddf8ea85 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -186,7 +186,11 @@ app.MapHealthChecks("/health/ready", new() }); app.MapHealthChecks("/", new() { - Predicate = _ => false + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } }); // Routing used for SignalR. diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 55d5478fb..3c6dca7fb 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -18,11 +18,13 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { - var probeResults = await Task.WhenAll( - ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)), - ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)), - ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)), - ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct))); + var probeResults = new List + { + await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)), + await ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)), + await ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)), + await ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)) + }; var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); var successfulProbes = probeResults.Where(x => !x.Skipped && x.Exception == null).Select(x => x.StoreName).ToList(); diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 02fcd8f11..26c37cc95 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -44,6 +44,25 @@ public class ElsaWorkflowPersistenceHealthCheckTests Arg.Any()); } + [Fact] + public async Task ProbesStoresSequentially() + { + var tracker = new ProbeConcurrencyTracker(); + _workflowDefinitionStore.FindAsync(Arg.Any(), Arg.Any()) + .Returns(_ => TrackProbeAsync(tracker, null)); + _workflowInstanceStore.CountAsync(Arg.Any(), Arg.Any()) + .Returns(_ => new ValueTask(TrackProbeAsync(tracker, 0L))); + _triggerStore.FindAsync(Arg.Any(), Arg.Any()) + .Returns(_ => new ValueTask(TrackProbeAsync(tracker, null))); + _bookmarkQueueStore.FindAsync(Arg.Any(), Arg.Any()) + .Returns(_ => TrackProbeAsync(tracker, null)); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(1, tracker.MaxConcurrentProbes); + } + [Fact] public async Task ReturnsUnhealthyWithFailedStoreWhenAStoreProbeFails() { @@ -92,4 +111,44 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["skippedProbes"]); Assert.False(result.Data.ContainsKey("probes")); } + + private static async Task TrackProbeAsync(ProbeConcurrencyTracker tracker, T result) + { + tracker.Enter(); + + try + { + await Task.Delay(10); + return result; + } + finally + { + tracker.Exit(); + } + } + + private sealed class ProbeConcurrencyTracker + { + private int _currentProbes; + private int _maxConcurrentProbes; + + public int MaxConcurrentProbes => Volatile.Read(ref _maxConcurrentProbes); + + public void Enter() + { + var currentProbes = Interlocked.Increment(ref _currentProbes); + + while (true) + { + var maxConcurrentProbes = MaxConcurrentProbes; + if (currentProbes <= maxConcurrentProbes) + return; + + if (Interlocked.CompareExchange(ref _maxConcurrentProbes, currentProbes, maxConcurrentProbes) == maxConcurrentProbes) + return; + } + } + + public void Exit() => Interlocked.Decrement(ref _currentProbes); + } } From 09641c1fabcd8781f59428aacc5dfcb522fda0e3 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 01:32:03 +0200 Subject: [PATCH 15/20] Keep root health check as liveness probe --- doc/wiki/health-checks.md | 2 ++ src/apps/Elsa.Server.Web/Program.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index a36d567a4..e8d9f5fca 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -49,6 +49,8 @@ app.MapHealthChecks("/health/ready", new() }); ``` +The reference server also keeps `/` process-liveness compatible for existing probes by mapping it with `Predicate = _ => false`, the same as `/health/live`. Use `/health/ready` for checks that should return `503` while the runtime is paused, draining, or dependency-degraded. + ## Current Elsa Readiness Checks - `elsa-runtime`: creates a workflow runtime client and reports `Degraded` when the runtime is paused or draining, because it is process-up but not accepting new work. diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index dddf8ea85..0623e9103 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -186,6 +186,7 @@ app.MapHealthChecks("/health/ready", new() }); app.MapHealthChecks("/", new() { + Predicate = _ => false, ResultStatusCodes = { [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, From 8c1618b5d47c2e6d297e3d25db51c5395c41d3ea Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 23:39:21 +0000 Subject: [PATCH 16/20] Address latest health check review feedback Agent-Logs-Url: https://github.com/elsa-workflows/elsa-core/sessions/6a652ca4-f0c8-4284-938d-6de0eb7a2bea Co-authored-by: sfmskywalker <938393+sfmskywalker@users.noreply.github.com> --- doc/wiki/health-checks.md | 2 +- src/apps/Elsa.Server.Web/Program.cs | 9 ++------- .../Extensions/HealthCheckExtensions.cs | 7 ++++++- .../ElsaDistributedLockHealthCheck.cs | 2 +- .../ElsaWorkflowPersistenceHealthCheck.cs | 20 +++++++++++++++---- .../ElsaDistributedLockHealthCheckTests.cs | 1 + 6 files changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index e8d9f5fca..db2a68d88 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -40,7 +40,7 @@ app.MapHealthChecks("/health/live", new() app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains(HealthCheckExtensions.ReadinessTag), + Predicate = check => check.Tags.Contains(HealthCheckExtensions.ElsaTag) && check.Tags.Contains(HealthCheckExtensions.ReadinessTag), ResultStatusCodes = { [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 0623e9103..e61770819 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -177,7 +177,7 @@ app.MapHealthChecks("/health/live", new() }); app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains(HealthCheckExtensions.ReadinessTag), + Predicate = check => check.Tags.Contains(HealthCheckExtensions.ElsaTag) && check.Tags.Contains(HealthCheckExtensions.ReadinessTag), ResultStatusCodes = { [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, @@ -186,12 +186,7 @@ app.MapHealthChecks("/health/ready", new() }); app.MapHealthChecks("/", new() { - Predicate = _ => false, - ResultStatusCodes = - { - [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, - [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable - } + Predicate = _ => false }); // Routing used for SignalR. diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs index 8d636d417..8adcafb8f 100644 --- a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -10,12 +10,17 @@ namespace Elsa.Extensions; /// public static class HealthCheckExtensions { + /// + /// Tag applied to Elsa health checks. + /// + public const string ElsaTag = "elsa"; + /// /// Tag applied to Elsa readiness checks. /// public const string ReadinessTag = "readiness"; - private static readonly string[] ReadinessTags = ["elsa", ReadinessTag]; + private static readonly string[] ReadinessTags = [ElsaTag, ReadinessTag]; /// /// Adds conservative Elsa-specific readiness probes for the workflow runtime and its core stores. diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index 3d1a4b1a2..5d5b32655 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -30,7 +30,7 @@ public class ElsaDistributedLockHealthCheck( }); } - var lockName = $"elsa-health-check-{Environment.MachineName}-{Guid.NewGuid():N}"; + var lockName = $"elsa-health-check-{Guid.NewGuid():N}"; await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, options.Value.DistributedLockAcquisitionTimeout, cancellationToken); if (handle == null) { diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 3c6dca7fb..529f54547 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -20,10 +20,22 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider { var probeResults = new List { - await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)), - await ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)), - await ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct)), - await ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)) + await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => + { + await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct); + }), + await ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => + { + await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct); + }), + await ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => + { + await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct); + }), + await ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => + { + await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct); + }) }; var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index ee7400b6a..d4e8e888a 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -50,6 +50,7 @@ public class ElsaDistributedLockHealthCheckTests await sut.CheckHealthAsync(new HealthCheckContext()); Assert.Equal(2, lockNames.Count); + Assert.All(lockNames, x => Assert.DoesNotContain(Environment.MachineName, x, StringComparison.Ordinal)); Assert.NotEqual(lockNames[0], lockNames[1]); } From 12408cca37e11e74e5674ca2cef5cb712dfb174f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 23:43:25 +0000 Subject: [PATCH 17/20] Address validation feedback Agent-Logs-Url: https://github.com/elsa-workflows/elsa-core/sessions/6a652ca4-f0c8-4284-938d-6de0eb7a2bea Co-authored-by: sfmskywalker <938393+sfmskywalker@users.noreply.github.com> --- .../HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs | 1 + .../HealthChecks/ElsaDistributedLockHealthCheckTests.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 529f54547..41bd1046c 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -18,6 +18,7 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { + // These probes verify store reachability only; returned entities and counts are intentionally ignored. var probeResults = new List { await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index d4e8e888a..115456c40 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -50,7 +50,7 @@ public class ElsaDistributedLockHealthCheckTests await sut.CheckHealthAsync(new HealthCheckContext()); Assert.Equal(2, lockNames.Count); - Assert.All(lockNames, x => Assert.DoesNotContain(Environment.MachineName, x, StringComparison.Ordinal)); + Assert.All(lockNames, x => Assert.DoesNotContain(Environment.MachineName, x, StringComparison.OrdinalIgnoreCase)); Assert.NotEqual(lockNames[0], lockNames[1]); } From b546864b4a1869092460aee464ab962eeb2dc85d Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 01:47:17 +0200 Subject: [PATCH 18/20] Address health check review feedback --- .../Extensions/HealthCheckExtensions.cs | 2 +- .../ElsaDistributedLockHealthCheckTests.cs | 10 +++++++++- .../HealthCheckExtensionsTests.cs | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs index 8adcafb8f..a440a2cda 100644 --- a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -18,7 +18,7 @@ public static class HealthCheckExtensions /// /// Tag applied to Elsa readiness checks. /// - public const string ReadinessTag = "readiness"; + public const string ReadinessTag = "elsa-readiness"; private static readonly string[] ReadinessTags = [ElsaTag, ReadinessTag]; diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index 115456c40..fdbaf77ed 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -31,7 +31,7 @@ public class ElsaDistributedLockHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("distributed-locks", result.Data["category"]); - _distributedLockProvider.Received(1).CreateLock(Arg.Is(x => x.StartsWith("elsa-health-check-", StringComparison.Ordinal))); + _distributedLockProvider.Received(1).CreateLock(Arg.Is(x => IsProbeLockName(x))); await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any()); } @@ -52,6 +52,7 @@ public class ElsaDistributedLockHealthCheckTests Assert.Equal(2, lockNames.Count); Assert.All(lockNames, x => Assert.DoesNotContain(Environment.MachineName, x, StringComparison.OrdinalIgnoreCase)); Assert.NotEqual(lockNames[0], lockNames[1]); + Assert.All(lockNames, x => Assert.True(IsProbeLockName(x))); } [Fact] @@ -102,4 +103,11 @@ public class ElsaDistributedLockHealthCheckTests { DistributedLockAcquisitionTimeout = ExpectedLockAcquisitionTimeout }); + + private static bool IsProbeLockName(string lockName) + { + const string prefix = "elsa-health-check-"; + return lockName.StartsWith(prefix, StringComparison.Ordinal) + && Guid.TryParseExact(lockName[prefix.Length..], "N", out _); + } } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs index 5bb78efc4..9033fb664 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs @@ -1,6 +1,7 @@ using Elsa.Extensions; using Elsa.Workflows.Runtime.Options; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; @@ -20,4 +21,22 @@ public class HealthCheckExtensionsTests Assert.NotNull(serviceProvider.GetRequiredService>()); } + + [Fact] + public void AddElsaReadinessChecksUsesElsaSpecificReadinessTag() + { + var services = new ServiceCollection(); + + services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false); + + using var serviceProvider = services.BuildServiceProvider(); + var registrations = serviceProvider.GetRequiredService>().Value.Registrations; + + var registration = Assert.Single(registrations); + Assert.Contains("elsa", registration.Tags); + Assert.Contains(HealthCheckExtensions.ReadinessTag, registration.Tags); + Assert.DoesNotContain("readiness", registration.Tags); + } } From 33d94ab3845a5f41a15498f59e1dc84c617da7ce Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 01:52:05 +0200 Subject: [PATCH 19/20] Document health check extension namespace --- doc/wiki/health-checks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index db2a68d88..c2582c728 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -7,6 +7,7 @@ Elsa hosts can opt in to Elsa-specific readiness checks on top of the normal ASP Register ASP.NET Core health checks and then add Elsa readiness checks: ```csharp +using Elsa.Extensions; using System; services From b7076fd0cfff053ffa59dc918f58833050403a56 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Thu, 21 May 2026 02:13:19 +0200 Subject: [PATCH 20/20] Address health check review feedback --- doc/wiki/health-checks.md | 2 + .../Extensions/HealthCheckExtensions.cs | 4 + .../ElsaDistributedLockHealthCheck.cs | 25 ++-- .../ElsaWorkflowPersistenceHealthCheck.cs | 123 +++++++++++------- .../ElsaReadinessHealthCheckOptions.cs | 5 + ...ElsaWorkflowPersistenceHealthCheckTests.cs | 39 +++++- .../HealthCheckExtensionsTests.cs | 19 +++ 7 files changed, 145 insertions(+), 72 deletions(-) diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index c2582c728..3fbb0b916 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -27,6 +27,8 @@ services .AddElsaReadinessChecks(includePersistence: false); ``` +Persistence probing stops after the first failed store by default to keep readiness responses bounded during dependency outages. Set `ContinuePersistenceProbesAfterFailure = true` when richer per-store diagnostics are more important than the fastest readiness response. + Map separate endpoints for liveness and readiness: ```csharp diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs index a440a2cda..ea81ad17c 100644 --- a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -40,6 +40,10 @@ public static class HealthCheckExtensions if (configureOptions != null) optionsBuilder.Configure(configureOptions); + optionsBuilder + .Validate(x => x.DistributedLockAcquisitionTimeout > TimeSpan.Zero, "Distributed lock acquisition timeout must be greater than zero.") + .ValidateOnStart(); + builder.AddCheck("elsa-runtime", tags: ReadinessTags); if (includePersistence) diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index 5d5b32655..a532120b3 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -24,26 +24,17 @@ public class ElsaDistributedLockHealthCheck( var distributedLockProvider = serviceProvider.GetService(); if (distributedLockProvider == null) { - return HealthCheckResult.Degraded("Elsa distributed lock provider is not registered.", data: new Dictionary - { - ["category"] = "distributed-locks" - }); + return HealthCheckResult.Degraded("Elsa distributed lock provider is not registered.", data: CreateData()); } var lockName = $"elsa-health-check-{Guid.NewGuid():N}"; await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, options.Value.DistributedLockAcquisitionTimeout, cancellationToken); if (handle == null) { - return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary - { - ["category"] = "distributed-locks" - }); + return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: CreateData()); } - return HealthCheckResult.Healthy("Elsa distributed lock provider is reachable.", new Dictionary - { - ["category"] = "distributed-locks" - }); + return HealthCheckResult.Healthy("Elsa distributed lock provider is reachable.", CreateData()); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -52,10 +43,12 @@ public class ElsaDistributedLockHealthCheck( catch (Exception e) when (!e.IsFatal()) { logger.LogWarning(e, "Elsa distributed lock provider is not reachable."); - return HealthCheckResult.Unhealthy("Elsa distributed lock provider is not reachable.", data: new Dictionary - { - ["category"] = "distributed-locks" - }); + return HealthCheckResult.Unhealthy("Elsa distributed lock provider is not reachable.", data: CreateData()); } + + static Dictionary CreateData() => new() + { + ["category"] = "distributed-locks" + }; } } diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 41bd1046c..461e91e9f 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -2,16 +2,21 @@ using Elsa.Common; using Elsa.Workflows.Management; using Elsa.Workflows.Management.Filters; using Elsa.Workflows.Runtime.Filters; +using Elsa.Workflows.Runtime.Options; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Elsa.Workflows.Runtime.HealthChecks; /// /// Performs small read-only probes against the workflow management and runtime stores. /// -public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider, ILogger logger) : IHealthCheck +public class ElsaWorkflowPersistenceHealthCheck( + IServiceProvider serviceProvider, + IOptions options, + ILogger logger) : IHealthCheck { private const string ProbeId = "00000000-0000-0000-0000-000000000000"; @@ -19,45 +24,85 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { // These probes verify store reachability only; returned entities and counts are intentionally ignored. - var probeResults = new List - { - await ProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => + var probeResults = new List(); + + if (!await AddProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => { await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct); - }), - await ProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => + })) + return CreateResult(); + + if (!await AddProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => { await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct); - }), - await ProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => + })) + return CreateResult(); + + if (!await AddProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => { await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct); - }), - await ProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => + })) + return CreateResult(); + + if (!await AddProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => { await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct); - }) - }; + })) + return CreateResult(); - var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); - var successfulProbes = probeResults.Where(x => !x.Skipped && x.Exception == null).Select(x => x.StoreName).ToList(); - var skippedProbes = probeResults.Where(x => x.Skipped).Select(x => x.StoreName).ToList(); - var failedProbes = probeResults.Where(x => x.Exception != null).Select(x => x.StoreName).ToList(); - var failedProbe = probeResults.FirstOrDefault(x => x.Exception != null); - if (failedProbe != null) + return CreateResult(); + + async Task AddProbeAsync(string storeName, TStore? store, Func probe) where TStore : class { - var data = CreateData(); - data["failedStore"] = failedProbe.StoreName; - data["failedProbe"] = failedProbe.StoreName; - - logger.LogWarning(failedProbe.Exception, "Elsa workflow store {StoreName} is not reachable.", failedProbe.StoreName); - return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedProbe.StoreName}' is not reachable.", data: data); + var result = await ProbeAsync(storeName, store, probe); + probeResults.Add(result); + return result.Exception == null || options.Value.ContinuePersistenceProbesAfterFailure; } - var healthyData = CreateData(); - return attemptedProbes.Count == 0 - ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: healthyData) - : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", healthyData); + HealthCheckResult CreateResult() + { + var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); + var successfulProbes = probeResults.Where(x => !x.Skipped && x.Exception == null).Select(x => x.StoreName).ToList(); + var skippedProbes = probeResults.Where(x => x.Skipped).Select(x => x.StoreName).ToList(); + var failedProbes = probeResults.Where(x => x.Exception != null).Select(x => x.StoreName).ToList(); + var failedProbe = probeResults.FirstOrDefault(x => x.Exception != null); + if (failedProbe != null) + { + var data = CreateData(); + data["failedStore"] = failedProbe.StoreName; + data["failedProbe"] = failedProbe.StoreName; + + logger.LogWarning(failedProbe.Exception, "Elsa workflow store {StoreName} is not reachable.", failedProbe.StoreName); + return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedProbe.StoreName}' is not reachable.", data: data); + } + + var healthyData = CreateData(); + return attemptedProbes.Count == 0 + ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: healthyData) + : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", healthyData); + + Dictionary CreateData() + { + var data = new Dictionary + { + ["category"] = "persistence" + }; + + if (successfulProbes.Count > 0) + data["successfulProbes"] = string.Join(",", successfulProbes); + + if (attemptedProbes.Count > 0) + data["attemptedProbes"] = string.Join(",", attemptedProbes); + + if (failedProbes.Count > 0) + data["failedProbes"] = string.Join(",", failedProbes); + + if (skippedProbes.Count > 0) + data["skippedProbes"] = string.Join(",", skippedProbes); + + return data; + } + } async Task ProbeAsync(string storeName, TStore? store, Func probe) where TStore : class { @@ -80,28 +125,6 @@ public class ElsaWorkflowPersistenceHealthCheck(IServiceProvider serviceProvider return new ProbeResult(storeName, false, e); } } - - Dictionary CreateData() - { - var data = new Dictionary - { - ["category"] = "persistence" - }; - - if (successfulProbes.Count > 0) - data["probes"] = string.Join(",", successfulProbes); - - if (attemptedProbes.Count > 0) - data["attemptedProbes"] = string.Join(",", attemptedProbes); - - if (failedProbes.Count > 0) - data["failedProbes"] = string.Join(",", failedProbes); - - if (skippedProbes.Count > 0) - data["skippedProbes"] = string.Join(",", skippedProbes); - - return data; - } } private sealed record ProbeResult(string StoreName, bool Skipped, Exception? Exception); diff --git a/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs b/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs index 351531bbe..3f082fe8a 100644 --- a/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs +++ b/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs @@ -9,4 +9,9 @@ public class ElsaReadinessHealthCheckOptions /// The maximum time the distributed-lock readiness probe waits to acquire its probe lock. /// public TimeSpan DistributedLockAcquisitionTimeout { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Whether persistence readiness probing should continue after the first failed store probe. + /// + public bool ContinuePersistenceProbesAfterFailure { get; set; } } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index 26c37cc95..a602d9a64 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -2,6 +2,7 @@ using Elsa.Workflows.Management; using Elsa.Workflows.Management.Filters; using Elsa.Workflows.Runtime.Filters; using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Options; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; @@ -27,7 +28,7 @@ public class ElsaWorkflowPersistenceHealthCheckTests _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns(_workflowInstanceStore); _serviceProvider.GetService(typeof(ITriggerStore)).Returns(_triggerStore); _serviceProvider.GetService(typeof(IBookmarkQueueStore)).Returns(_bookmarkQueueStore); - _sut = new ElsaWorkflowPersistenceHealthCheck(_serviceProvider, NullLogger.Instance); + _sut = CreateSut(); } [Fact] @@ -37,7 +38,7 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("persistence", result.Data["category"]); - Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["probes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["successfulProbes"]); Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); await _workflowDefinitionStore.Received(1).FindAsync( Arg.Is(x => x.Id == "00000000-0000-0000-0000-000000000000"), @@ -64,7 +65,7 @@ public class ElsaWorkflowPersistenceHealthCheckTests } [Fact] - public async Task ReturnsUnhealthyWithFailedStoreWhenAStoreProbeFails() + public async Task ReturnsUnhealthyWithFailedStoreAndStopsProbingWhenAStoreProbeFails() { _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); @@ -75,9 +76,25 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("persistence", result.Data["category"]); Assert.Equal("triggers", result.Data["failedStore"]); Assert.Equal("triggers", result.Data["failedProbe"]); - Assert.Equal("workflow-definitions,workflow-instances,bookmark-queue", result.Data["probes"]); + Assert.Equal("workflow-definitions,workflow-instances", result.Data["successfulProbes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers", result.Data["attemptedProbes"]); + Assert.Equal("triggers", result.Data["failedProbes"]); + await _bookmarkQueueStore.DidNotReceive().FindAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ReturnsUnhealthyWithAllProbeDataWhenContinuationIsEnabled() + { + var sut = CreateSut(continueAfterFailure: true); + _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); + + var result = await sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("workflow-definitions,workflow-instances,bookmark-queue", result.Data["successfulProbes"]); Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); Assert.Equal("triggers", result.Data["failedProbes"]); + await _bookmarkQueueStore.Received(1).FindAsync(Arg.Any(), Arg.Any()); } [Fact] @@ -90,7 +107,7 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("persistence", result.Data["category"]); - Assert.Equal("triggers,bookmark-queue", result.Data["probes"]); + Assert.Equal("triggers,bookmark-queue", result.Data["successfulProbes"]); Assert.Equal("triggers,bookmark-queue", result.Data["attemptedProbes"]); Assert.Equal("workflow-definitions,workflow-instances", result.Data["skippedProbes"]); } @@ -109,7 +126,17 @@ public class ElsaWorkflowPersistenceHealthCheckTests Assert.Equal("No Elsa workflow persistence stores are registered.", result.Description); Assert.Equal("persistence", result.Data["category"]); Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["skippedProbes"]); - Assert.False(result.Data.ContainsKey("probes")); + Assert.False(result.Data.ContainsKey("successfulProbes")); + } + + private ElsaWorkflowPersistenceHealthCheck CreateSut(bool continueAfterFailure = false) + { + var options = Microsoft.Extensions.Options.Options.Create(new ElsaReadinessHealthCheckOptions + { + ContinuePersistenceProbesAfterFailure = continueAfterFailure + }); + + return new ElsaWorkflowPersistenceHealthCheck(_serviceProvider, options, NullLogger.Instance); } private static async Task TrackProbeAsync(ProbeConcurrencyTracker tracker, T result) diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs index 9033fb664..c2ba45795 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs @@ -22,6 +22,25 @@ public class HealthCheckExtensionsTests Assert.NotNull(serviceProvider.GetRequiredService>()); } + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void AddElsaReadinessChecksRejectsNonPositiveDistributedLockTimeout(int timeoutMilliseconds) + { + var services = new ServiceCollection(); + + services + .AddHealthChecks() + .AddElsaReadinessChecks( + includePersistence: false, + includeDistributedLocks: true, + configureOptions: options => options.DistributedLockAcquisitionTimeout = TimeSpan.FromMilliseconds(timeoutMilliseconds)); + + using var serviceProvider = services.BuildServiceProvider(); + + Assert.Throws(() => serviceProvider.GetRequiredService>().Value); + } + [Fact] public void AddElsaReadinessChecksUsesElsaSpecificReadinessTag() {