diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md index 42898f5e5..587bb44c2 100644 --- a/doc/wiki/health-checks.md +++ b/doc/wiki/health-checks.md @@ -14,6 +14,14 @@ services Use `includeDistributedLocks: true` when the host enables distributed runtime behavior or relies on distributed locks for workflow coordination. Leave it `false` for simple single-node hosts that do not want a lock probe. +Persistence probes are enabled by default. Set `includePersistence: false` when a host uses a custom or minimal persistence setup, or when it only needs the `elsa-runtime` readiness probe: + +```csharp +services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false); +``` + Map separate endpoints for liveness and readiness: ```csharp @@ -24,7 +32,12 @@ app.MapHealthChecks("/health/live", new() app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains("readiness") + Predicate = check => check.Tags.Contains("readiness"), + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } }); ``` diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 671f84fe9..98e0844cb 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -26,6 +26,8 @@ using Elsa.Workflows.Runtime.Distributed.Extensions; using Elsa.Workflows.Runtime.Options; using Elsa.Workflows.Runtime.Tasks; using JetBrains.Annotations; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; // ReSharper disable RedundantAssignment @@ -176,7 +178,12 @@ app.MapHealthChecks("/health/live", new() }); app.MapHealthChecks("/health/ready", new() { - Predicate = check => check.Tags.Contains("readiness") + Predicate = check => check.Tags.Contains("readiness"), + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } }); app.MapHealthChecks("/", new() { diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs index 5f701e63b..ce7abe68c 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -9,13 +9,15 @@ namespace Elsa.Workflows.Runtime.HealthChecks; /// public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributedLockProvider) : IHealthCheck { + private const string LockName = "elsa-health-check"; + private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1); + /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { try { - var lockName = $"elsa-health-check-{Guid.NewGuid():N}"; - await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, TimeSpan.Zero, cancellationToken); + await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, LockAcquisitionTimeout, cancellationToken); if (handle == null) { return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs index 84d799251..0a735c9b5 100644 --- a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -20,12 +20,14 @@ public class ElsaWorkflowPersistenceHealthCheck( /// public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { + var failedStore = ""; + try { - await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, cancellationToken); - await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, cancellationToken); - await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, cancellationToken); - await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, cancellationToken); + await ProbeAsync("workflow-definitions", async ct => await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct)); + await ProbeAsync("workflow-instances", async ct => await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct)); + await ProbeAsync("triggers", async ct => await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, ct)); + await ProbeAsync("bookmark-queue", async ct => await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct)); return HealthCheckResult.Healthy("Elsa workflow stores are reachable.", new Dictionary { @@ -35,10 +37,17 @@ public class ElsaWorkflowPersistenceHealthCheck( } catch (Exception e) when (!e.IsFatal()) { - return HealthCheckResult.Unhealthy("Elsa workflow stores are not reachable.", e, new Dictionary + return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", e, new Dictionary { - ["category"] = "persistence" + ["category"] = "persistence", + ["failedStore"] = failedStore }); } + + async Task ProbeAsync(string store, Func probe) + { + failedStore = store; + await probe(cancellationToken); + } } } diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs index 42814a861..e531a09b5 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -7,6 +7,7 @@ namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; public class ElsaDistributedLockHealthCheckTests { + private static readonly TimeSpan ExpectedLockAcquisitionTimeout = TimeSpan.FromSeconds(1); private readonly IDistributedLockProvider _distributedLockProvider = Substitute.For(); private readonly IDistributedLock _distributedLock = Substitute.For(); private readonly ElsaDistributedLockHealthCheck _sut; @@ -26,6 +27,8 @@ public class ElsaDistributedLockHealthCheckTests Assert.Equal(HealthStatus.Healthy, result.Status); Assert.Equal("distributed-locks", result.Data["category"]); + _distributedLockProvider.Received(1).CreateLock("elsa-health-check"); + await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any()); } [Fact] diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs index df8f0ca3c..893bb3c5a 100644 --- a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -34,13 +34,15 @@ public class ElsaWorkflowPersistenceHealthCheckTests } [Fact] - public async Task ReturnsUnhealthyWhenAStoreProbeFails() + public async Task ReturnsUnhealthyWithFailedStoreWhenAStoreProbeFails() { _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); var result = await _sut.CheckHealthAsync(new HealthCheckContext()); Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("Elsa workflow store 'triggers' is not reachable.", result.Description); Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("triggers", result.Data["failedStore"]); } }