Address health check review feedback

doc/wiki/health-checks.md

@@ -14,6 +14,14 @@ services
 
 Use `includeDistributedLocks: true` when the host enables distributed runtime behavior or relies on distributed locks for workflow coordination. Leave it `false` for simple single-node hosts that do not want a lock probe.
 
+Persistence probes are enabled by default. Set `includePersistence: false` when a host uses a custom or minimal persistence setup, or when it only needs the `elsa-runtime` readiness probe:
+
+```csharp
+services
+    .AddHealthChecks()
+    .AddElsaReadinessChecks(includePersistence: false);
+```
+
 Map separate endpoints for liveness and readiness:
 
 ```csharp
@@ -24,7 +32,12 @@ app.MapHealthChecks("/health/live", new()
 
 app.MapHealthChecks("/health/ready", new()
 {
-    Predicate = check => check.Tags.Contains("readiness")
+    Predicate = check => check.Tags.Contains("readiness"),
+    ResultStatusCodes =
+    {
+        [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable,
+        [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable
+    }
 });
 ```
 

src/apps/Elsa.Server.Web/Program.cs

@@ -26,6 +26,8 @@ using Elsa.Workflows.Runtime.Distributed.Extensions;
 using Elsa.Workflows.Runtime.Options;
 using Elsa.Workflows.Runtime.Tasks;
 using JetBrains.Annotations;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Options;
 
 // ReSharper disable RedundantAssignment
@@ -176,7 +178,12 @@ app.MapHealthChecks("/health/live", new()
 });
 app.MapHealthChecks("/health/ready", new()
 {
-    Predicate = check => check.Tags.Contains("readiness")
+    Predicate = check => check.Tags.Contains("readiness"),
+    ResultStatusCodes =
+    {
+        [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable,
+        [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable
+    }
 });
 app.MapHealthChecks("/", new()
 {

src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs

@@ -9,13 +9,15 @@ namespace Elsa.Workflows.Runtime.HealthChecks;
 /// </summary>
 public class ElsaDistributedLockHealthCheck(IDistributedLockProvider distributedLockProvider) : IHealthCheck
 {
+    private const string LockName = "elsa-health-check";
+    private static readonly TimeSpan LockAcquisitionTimeout = TimeSpan.FromSeconds(1);
+
     /// <inheritdoc />
     public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
     {
         try
         {
-            var lockName = $"elsa-health-check-{Guid.NewGuid():N}";
-            await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, TimeSpan.Zero, cancellationToken);
+            await using var handle = await distributedLockProvider.TryAcquireLockAsync(LockName, LockAcquisitionTimeout, cancellationToken);
             if (handle == null)
             {
                 return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: new Dictionary<string, object>

src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs

@@ -20,12 +20,14 @@ public class ElsaWorkflowPersistenceHealthCheck(
     /// <inheritdoc />
     public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
     {
+        var failedStore = "";
+
         try
         {
-            await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, cancellationToken);
-            await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, cancellationToken);
-            await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, cancellationToken);
-            await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, cancellationToken);
+            await ProbeAsync("workflow-definitions", async ct => await workflowDefinitionStore.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct));
+            await ProbeAsync("workflow-instances", async ct => await workflowInstanceStore.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct));
+            await ProbeAsync("triggers", async ct => await triggerStore.FindAsync(new TriggerFilter { Id = ProbeId }, ct));
+            await ProbeAsync("bookmark-queue", async ct => await bookmarkQueueStore.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct));
 
             return HealthCheckResult.Healthy("Elsa workflow stores are reachable.", new Dictionary<string, object>
             {
@@ -35,10 +37,17 @@ public class ElsaWorkflowPersistenceHealthCheck(
         }
         catch (Exception e) when (!e.IsFatal())
         {
-            return HealthCheckResult.Unhealthy("Elsa workflow stores are not reachable.", e, new Dictionary<string, object>
+            return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedStore}' is not reachable.", e, new Dictionary<string, object>
             {
-                ["category"] = "persistence"
+                ["category"] = "persistence",
+                ["failedStore"] = failedStore
             });
         }
+
+        async Task ProbeAsync(string store, Func<CancellationToken, Task> probe)
+        {
+            failedStore = store;
+            await probe(cancellationToken);
+        }
     }
 }

test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs

@@ -7,6 +7,7 @@ namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks;
 
 public class ElsaDistributedLockHealthCheckTests
 {
+    private static readonly TimeSpan ExpectedLockAcquisitionTimeout = TimeSpan.FromSeconds(1);
     private readonly IDistributedLockProvider _distributedLockProvider = Substitute.For<IDistributedLockProvider>();
     private readonly IDistributedLock _distributedLock = Substitute.For<IDistributedLock>();
     private readonly ElsaDistributedLockHealthCheck _sut;
@@ -26,6 +27,8 @@ public class ElsaDistributedLockHealthCheckTests
 
         Assert.Equal(HealthStatus.Healthy, result.Status);
         Assert.Equal("distributed-locks", result.Data["category"]);
+        _distributedLockProvider.Received(1).CreateLock("elsa-health-check");
+        await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any<CancellationToken>());
     }
 
     [Fact]

test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs

@@ -34,13 +34,15 @@ public class ElsaWorkflowPersistenceHealthCheckTests
     }
 
     [Fact]
-    public async Task ReturnsUnhealthyWhenAStoreProbeFails()
+    public async Task ReturnsUnhealthyWithFailedStoreWhenAStoreProbeFails()
     {
         _triggerStore.FindAsync(Arg.Any<TriggerFilter>(), Arg.Any<CancellationToken>()).Returns<ValueTask<Elsa.Workflows.Runtime.Entities.StoredTrigger?>>(_ => throw new InvalidOperationException("store unavailable"));
 
         var result = await _sut.CheckHealthAsync(new HealthCheckContext());
 
         Assert.Equal(HealthStatus.Unhealthy, result.Status);
+        Assert.Equal("Elsa workflow store 'triggers' is not reachable.", result.Description);
         Assert.Equal("persistence", result.Data["category"]);
+        Assert.Equal("triggers", result.Data["failedStore"]);
     }
 }