diff --git a/Directory.Packages.props b/Directory.Packages.props index 91bf85044..cf643d6ea 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -30,6 +30,7 @@ + @@ -71,6 +72,7 @@ + diff --git a/doc/wiki/README.md b/doc/wiki/README.md index e5bb48117..8da5e2b6a 100644 --- a/doc/wiki/README.md +++ b/doc/wiki/README.md @@ -48,6 +48,7 @@ flowchart LR | [Persistence](persistence.md) | In-memory stores, EF Core stores, provider packages, migrations, and multi-provider rules. | | [Diagnostics Structured Logs](diagnostics-structured-logs.md) | `ILogger` capture, live feed, REST/SignalR surface, redaction, and SQLite persistence. | | [Diagnostics Console Logs](diagnostics-console-logs.md) | Raw stdout/stderr capture, live feed, REST/SignalR surface, and redaction. | +| [Health Checks](health-checks.md) | Elsa runtime readiness probes, liveness/readiness mapping, and Kubernetes probe guidance. | | [Identity, Tenancy, And Security](identity-tenancy-security.md) | Users, applications, roles, API keys, tenant resolution, and authorization touch points. | | [Testing Guide](testing-guide.md) | Test project layout, fixture choices, and targeted commands. | | [Extension Guide](extension-guide.md) | How to add features, activities, expression providers, stores, endpoints, and ingress sources. | diff --git a/doc/wiki/build-run-operate.md b/doc/wiki/build-run-operate.md index f3446bcaf..99aabe337 100644 --- a/doc/wiki/build-run-operate.md +++ b/doc/wiki/build-run-operate.md @@ -139,7 +139,7 @@ Console log diagnostics endpoints include (when `Elsa.Diagnostics.ConsoleLogs` i - `POST /elsa/api/diagnostics/console-logs/recent` - `GET /elsa/api/diagnostics/console-logs/sources` -Health checks are mapped to `/` in the reference server. +Health checks are mapped to `/` and `/health/live` for process liveness and `/health/ready` for Elsa runtime readiness in the reference server. See [Health Checks](health-checks.md) for Kubernetes liveness/readiness recommendations and Elsa-specific runtime readiness probes. ## Runtime Knobs diff --git a/doc/wiki/health-checks.md b/doc/wiki/health-checks.md new file mode 100644 index 000000000..3fbb0b916 --- /dev/null +++ b/doc/wiki/health-checks.md @@ -0,0 +1,88 @@ +# Health Checks + +Elsa hosts can opt in to Elsa-specific readiness checks on top of the normal ASP.NET Core process liveness check. The first slice focuses on runtime dependencies that are available through existing Elsa service contracts and avoids exposing connection strings, credentials, or provider-specific details. + +## Opt In + +Register ASP.NET Core health checks and then add Elsa readiness checks: + +```csharp +using Elsa.Extensions; +using System; + +services + .AddHealthChecks() + .AddElsaReadinessChecks( + includeDistributedLocks: true, + configureOptions: options => options.DistributedLockAcquisitionTimeout = TimeSpan.FromSeconds(1)); +``` + +Use `includeDistributedLocks: true` when the host enables distributed runtime behavior or relies on distributed locks for workflow coordination. Leave it `false` for simple single-node hosts that do not want a lock probe. + +Persistence probes are enabled by default. Set `includePersistence: false` when a host uses a custom or minimal persistence setup, or when it only needs the `elsa-runtime` readiness probe: + +```csharp +services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false); +``` + +Persistence probing stops after the first failed store by default to keep readiness responses bounded during dependency outages. Set `ContinuePersistenceProbesAfterFailure = true` when richer per-store diagnostics are more important than the fastest readiness response. + +Map separate endpoints for liveness and readiness: + +```csharp +using Elsa.Extensions; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +app.MapHealthChecks("/health/live", new() +{ + Predicate = _ => false +}); + +app.MapHealthChecks("/health/ready", new() +{ + Predicate = check => check.Tags.Contains(HealthCheckExtensions.ElsaTag) && check.Tags.Contains(HealthCheckExtensions.ReadinessTag), + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } +}); +``` + +The reference server also keeps `/` process-liveness compatible for existing probes by mapping it with `Predicate = _ => false`, the same as `/health/live`. Use `/health/ready` for checks that should return `503` while the runtime is paused, draining, or dependency-degraded. + +## Current Elsa Readiness Checks + +- `elsa-runtime`: creates a workflow runtime client and reports `Degraded` when the runtime is paused or draining, because it is process-up but not accepting new work. +- `elsa-workflow-persistence`: performs small read-only probes against workflow definitions, workflow instances, triggers, and the bookmark queue store. +- `elsa-distributed-locks`: optionally verifies that the configured distributed lock provider can acquire and release a probe lock. + +The health check data only includes subsystem categories and operational state such as readiness reason and active execution-cycle count. It does not include connection strings, lock names, credentials, tenant IDs, or workflow payloads. + +## Kubernetes Recommendation + +Use liveness to answer "is the ASP.NET Core process alive?" and readiness to answer "can this Elsa instance accept workflow traffic?" + +```yaml +livenessProbe: + httpGet: + path: /health/live + port: http + periodSeconds: 10 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /health/ready + port: http + timeoutSeconds: 2 + periodSeconds: 10 + failureThreshold: 3 +``` + +Do not point liveness at Elsa readiness checks. A paused, draining, or temporarily database-degraded runtime should be removed from service by readiness without forcing Kubernetes to restart the process. + +When `includeDistributedLocks: true` is enabled, set the readiness probe `timeoutSeconds` higher than the configured distributed-lock acquisition timeout. The built-in distributed-lock readiness check waits up to 1 second by default to acquire its probe lock, so the example uses `timeoutSeconds: 2` to avoid intermittent Kubernetes probe timeouts while the lock provider is still healthy. diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index ac0403b53..4590049b6 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -26,6 +26,7 @@ using Elsa.Workflows.Runtime.Distributed.Extensions; using Elsa.Workflows.Runtime.Options; using Elsa.Workflows.Runtime.Tasks; using JetBrains.Annotations; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; // ReSharper disable RedundantAssignment @@ -153,7 +154,9 @@ services.Configure(options => { options.InactivityThreshold = Ti services.Configure(options => options.Ttl = TimeSpan.FromSeconds(3600)); services.Configure(options => options.CacheDuration = TimeSpan.FromDays(1)); services.Configure(options => options.DefaultIncidentStrategy = typeof(ContinueWithIncidentsStrategy)); -services.AddHealthChecks(); +services + .AddHealthChecks() + .AddElsaReadinessChecks(includeDistributedLocks: true); services.AddControllers(); services.AddCors(cors => cors.AddDefaultPolicy(policy => policy.AllowAnyHeader().AllowAnyMethod().AllowAnyOrigin().WithExposedHeaders("*"))); @@ -169,7 +172,23 @@ if (app.Environment.IsDevelopment()) app.UseCors(); // Health checks. -app.MapHealthChecks("/"); +app.MapHealthChecks("/health/live", new() +{ + Predicate = _ => false +}); +app.MapHealthChecks("/health/ready", new() +{ + Predicate = check => check.Tags.Contains(HealthCheckExtensions.ElsaTag) && check.Tags.Contains(HealthCheckExtensions.ReadinessTag), + ResultStatusCodes = + { + [HealthStatus.Degraded] = StatusCodes.Status503ServiceUnavailable, + [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable + } +}); +app.MapHealthChecks("/", new() +{ + Predicate = _ => false +}); // Routing used for SignalR. app.UseRouting(); diff --git a/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj b/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj index c25e07d24..f089e7d64 100644 --- a/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj +++ b/src/modules/Elsa.Workflows.Runtime/Elsa.Workflows.Runtime.csproj @@ -10,6 +10,7 @@ + diff --git a/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs new file mode 100644 index 000000000..ea81ad17c --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/Extensions/HealthCheckExtensions.cs @@ -0,0 +1,57 @@ +using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Options; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Elsa.Extensions; + +/// +/// Adds Elsa workflow runtime readiness checks to ASP.NET Core health checks. +/// +public static class HealthCheckExtensions +{ + /// + /// Tag applied to Elsa health checks. + /// + public const string ElsaTag = "elsa"; + + /// + /// Tag applied to Elsa readiness checks. + /// + public const string ReadinessTag = "elsa-readiness"; + + private static readonly string[] ReadinessTags = [ElsaTag, ReadinessTag]; + + /// + /// Adds conservative Elsa-specific readiness probes for the workflow runtime and its core stores. + /// + /// The health checks builder. + /// Whether to probe workflow management/runtime stores. + /// Whether to probe the configured distributed lock provider. + /// Configures Elsa readiness probe behavior. + public static IHealthChecksBuilder AddElsaReadinessChecks( + this IHealthChecksBuilder builder, + bool includePersistence = true, + bool includeDistributedLocks = false, + Action? configureOptions = null) + { + var optionsBuilder = builder.Services.AddOptions(); + + if (configureOptions != null) + optionsBuilder.Configure(configureOptions); + + optionsBuilder + .Validate(x => x.DistributedLockAcquisitionTimeout > TimeSpan.Zero, "Distributed lock acquisition timeout must be greater than zero.") + .ValidateOnStart(); + + builder.AddCheck("elsa-runtime", tags: ReadinessTags); + + if (includePersistence) + builder.AddCheck("elsa-workflow-persistence", tags: ReadinessTags); + + if (includeDistributedLocks) + builder.AddCheck("elsa-distributed-locks", tags: ReadinessTags); + + return builder; + } +} diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs new file mode 100644 index 000000000..a532120b3 --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaDistributedLockHealthCheck.cs @@ -0,0 +1,54 @@ +using Elsa.Common; +using Elsa.Workflows.Runtime.Options; +using Medallion.Threading; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Elsa.Workflows.Runtime.HealthChecks; + +/// +/// Verifies that the configured distributed lock provider can acquire and release a probe lock. +/// +public class ElsaDistributedLockHealthCheck( + IServiceProvider serviceProvider, + IOptions options, + ILogger logger) : IHealthCheck +{ + /// + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + var distributedLockProvider = serviceProvider.GetService(); + if (distributedLockProvider == null) + { + return HealthCheckResult.Degraded("Elsa distributed lock provider is not registered.", data: CreateData()); + } + + var lockName = $"elsa-health-check-{Guid.NewGuid():N}"; + await using var handle = await distributedLockProvider.TryAcquireLockAsync(lockName, options.Value.DistributedLockAcquisitionTimeout, cancellationToken); + if (handle == null) + { + return HealthCheckResult.Degraded("Elsa distributed lock provider was reachable, but the probe lock was not acquired.", data: CreateData()); + } + + return HealthCheckResult.Healthy("Elsa distributed lock provider is reachable.", CreateData()); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) when (!e.IsFatal()) + { + logger.LogWarning(e, "Elsa distributed lock provider is not reachable."); + return HealthCheckResult.Unhealthy("Elsa distributed lock provider is not reachable.", data: CreateData()); + } + + static Dictionary CreateData() => new() + { + ["category"] = "distributed-locks" + }; + } +} diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs new file mode 100644 index 000000000..63f07a0ea --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaRuntimeHealthCheck.cs @@ -0,0 +1,45 @@ +using Elsa.Common; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace Elsa.Workflows.Runtime.HealthChecks; + +/// +/// Reports whether the workflow runtime can create clients and is currently accepting new work. +/// +public class ElsaRuntimeHealthCheck(IWorkflowRuntime workflowRuntime, IQuiescenceSignal quiescenceSignal, ILogger logger) : IHealthCheck +{ + /// + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + await workflowRuntime.CreateClientAsync(cancellationToken); + + var state = quiescenceSignal.CurrentState; + var data = new Dictionary + { + ["category"] = "runtime", + ["acceptingNewWork"] = state.IsAcceptingNewWork, + ["reason"] = state.Reason.ToString(), + ["activeExecutionCycles"] = quiescenceSignal.ActiveExecutionCycleCount + }; + + return state.IsAcceptingNewWork + ? HealthCheckResult.Healthy("Elsa workflow runtime is ready.", data) + : HealthCheckResult.Degraded("Elsa workflow runtime is paused or draining.", data: data); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) when (!e.IsFatal()) + { + logger.LogWarning(e, "Elsa workflow runtime could not create a workflow client."); + return HealthCheckResult.Unhealthy("Elsa workflow runtime could not create a workflow client.", data: new Dictionary + { + ["category"] = "runtime" + }); + } + } +} diff --git a/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs new file mode 100644 index 000000000..461e91e9f --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/HealthChecks/ElsaWorkflowPersistenceHealthCheck.cs @@ -0,0 +1,131 @@ +using Elsa.Common; +using Elsa.Workflows.Management; +using Elsa.Workflows.Management.Filters; +using Elsa.Workflows.Runtime.Filters; +using Elsa.Workflows.Runtime.Options; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Elsa.Workflows.Runtime.HealthChecks; + +/// +/// Performs small read-only probes against the workflow management and runtime stores. +/// +public class ElsaWorkflowPersistenceHealthCheck( + IServiceProvider serviceProvider, + IOptions options, + ILogger logger) : IHealthCheck +{ + private const string ProbeId = "00000000-0000-0000-0000-000000000000"; + + /// + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + // These probes verify store reachability only; returned entities and counts are intentionally ignored. + var probeResults = new List(); + + if (!await AddProbeAsync("workflow-definitions", serviceProvider.GetService(), async (store, ct) => + { + await store.FindAsync(new WorkflowDefinitionFilter { Id = ProbeId }, ct); + })) + return CreateResult(); + + if (!await AddProbeAsync("workflow-instances", serviceProvider.GetService(), async (store, ct) => + { + await store.CountAsync(new WorkflowInstanceFilter { Id = ProbeId }, ct); + })) + return CreateResult(); + + if (!await AddProbeAsync("triggers", serviceProvider.GetService(), async (store, ct) => + { + await store.FindAsync(new TriggerFilter { Id = ProbeId }, ct); + })) + return CreateResult(); + + if (!await AddProbeAsync("bookmark-queue", serviceProvider.GetService(), async (store, ct) => + { + await store.FindAsync(new BookmarkQueueFilter { Id = ProbeId }, ct); + })) + return CreateResult(); + + return CreateResult(); + + async Task AddProbeAsync(string storeName, TStore? store, Func probe) where TStore : class + { + var result = await ProbeAsync(storeName, store, probe); + probeResults.Add(result); + return result.Exception == null || options.Value.ContinuePersistenceProbesAfterFailure; + } + + HealthCheckResult CreateResult() + { + var attemptedProbes = probeResults.Where(x => !x.Skipped).Select(x => x.StoreName).ToList(); + var successfulProbes = probeResults.Where(x => !x.Skipped && x.Exception == null).Select(x => x.StoreName).ToList(); + var skippedProbes = probeResults.Where(x => x.Skipped).Select(x => x.StoreName).ToList(); + var failedProbes = probeResults.Where(x => x.Exception != null).Select(x => x.StoreName).ToList(); + var failedProbe = probeResults.FirstOrDefault(x => x.Exception != null); + if (failedProbe != null) + { + var data = CreateData(); + data["failedStore"] = failedProbe.StoreName; + data["failedProbe"] = failedProbe.StoreName; + + logger.LogWarning(failedProbe.Exception, "Elsa workflow store {StoreName} is not reachable.", failedProbe.StoreName); + return HealthCheckResult.Unhealthy($"Elsa workflow store '{failedProbe.StoreName}' is not reachable.", data: data); + } + + var healthyData = CreateData(); + return attemptedProbes.Count == 0 + ? HealthCheckResult.Degraded("No Elsa workflow persistence stores are registered.", data: healthyData) + : HealthCheckResult.Healthy("Elsa workflow stores are reachable.", healthyData); + + Dictionary CreateData() + { + var data = new Dictionary + { + ["category"] = "persistence" + }; + + if (successfulProbes.Count > 0) + data["successfulProbes"] = string.Join(",", successfulProbes); + + if (attemptedProbes.Count > 0) + data["attemptedProbes"] = string.Join(",", attemptedProbes); + + if (failedProbes.Count > 0) + data["failedProbes"] = string.Join(",", failedProbes); + + if (skippedProbes.Count > 0) + data["skippedProbes"] = string.Join(",", skippedProbes); + + return data; + } + } + + async Task ProbeAsync(string storeName, TStore? store, Func probe) where TStore : class + { + if (store == null) + { + return new ProbeResult(storeName, true, null); + } + + try + { + await probe(store, cancellationToken); + return new ProbeResult(storeName, false, null); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) when (!e.IsFatal()) + { + return new ProbeResult(storeName, false, e); + } + } + } + + private sealed record ProbeResult(string StoreName, bool Skipped, Exception? Exception); +} diff --git a/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs b/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs new file mode 100644 index 000000000..3f082fe8a --- /dev/null +++ b/src/modules/Elsa.Workflows.Runtime/Options/ElsaReadinessHealthCheckOptions.cs @@ -0,0 +1,17 @@ +namespace Elsa.Workflows.Runtime.Options; + +/// +/// Configures Elsa workflow runtime readiness health checks. +/// +public class ElsaReadinessHealthCheckOptions +{ + /// + /// The maximum time the distributed-lock readiness probe waits to acquire its probe lock. + /// + public TimeSpan DistributedLockAcquisitionTimeout { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Whether persistence readiness probing should continue after the first failed store probe. + /// + public bool ContinuePersistenceProbesAfterFailure { get; set; } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs new file mode 100644 index 000000000..fdbaf77ed --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaDistributedLockHealthCheckTests.cs @@ -0,0 +1,113 @@ +using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Options; +using Medallion.Threading; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class ElsaDistributedLockHealthCheckTests +{ + private static readonly TimeSpan ExpectedLockAcquisitionTimeout = TimeSpan.FromMilliseconds(250); + private readonly IDistributedLockProvider _distributedLockProvider = Substitute.For(); + private readonly IDistributedLock _distributedLock = Substitute.For(); + private readonly ElsaDistributedLockHealthCheck _sut; + + public ElsaDistributedLockHealthCheckTests() + { + _distributedLockProvider.CreateLock(Arg.Any()).Returns(_distributedLock); + _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) + .Returns(new ValueTask(Substitute.For())); + _sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(_distributedLockProvider), CreateOptions(), NullLogger.Instance); + } + + [Fact] + public async Task ReturnsHealthyWhenProbeLockCanBeAcquired() + { + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + _distributedLockProvider.Received(1).CreateLock(Arg.Is(x => IsProbeLockName(x))); + await _distributedLock.Received(1).TryAcquireAsync(ExpectedLockAcquisitionTimeout, Arg.Any()); + } + + [Fact] + public async Task UsesUniqueProbeLockNameForEachCheck() + { + var lockNames = new List(); + var distributedLockProvider = Substitute.For(); + var distributedLock = Substitute.For(); + distributedLockProvider.CreateLock(Arg.Do(lockNames.Add)).Returns(distributedLock); + distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) + .Returns(new ValueTask(Substitute.For())); + var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(distributedLockProvider), CreateOptions(), NullLogger.Instance); + + await sut.CheckHealthAsync(new HealthCheckContext()); + await sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(2, lockNames.Count); + Assert.All(lockNames, x => Assert.DoesNotContain(Environment.MachineName, x, StringComparison.OrdinalIgnoreCase)); + Assert.NotEqual(lockNames[0], lockNames[1]); + Assert.All(lockNames, x => Assert.True(IsProbeLockName(x))); + } + + [Fact] + public async Task ReturnsDegradedWhenProbeLockCannotBeAcquired() + { + _distributedLock.TryAcquireAsync(Arg.Any(), Arg.Any()) + .Returns(new ValueTask((IDistributedSynchronizationHandle?)null)); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } + + [Fact] + public async Task ReturnsUnhealthyWhenProviderThrows() + { + _distributedLockProvider.CreateLock(Arg.Any()).Returns(_ => throw new InvalidOperationException("lock backend unavailable")); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } + + [Fact] + public async Task ReturnsDegradedWhenProviderIsNotRegistered() + { + var sut = new ElsaDistributedLockHealthCheck(CreateServiceProvider(), CreateOptions(), NullLogger.Instance); + + var result = await sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("distributed-locks", result.Data["category"]); + } + + private static IServiceProvider CreateServiceProvider(IDistributedLockProvider? distributedLockProvider = null) + { + var services = new ServiceCollection(); + + if (distributedLockProvider != null) + services.AddSingleton(distributedLockProvider); + + return services.BuildServiceProvider(); + } + + private static IOptions CreateOptions() => Microsoft.Extensions.Options.Options.Create(new ElsaReadinessHealthCheckOptions + { + DistributedLockAcquisitionTimeout = ExpectedLockAcquisitionTimeout + }); + + private static bool IsProbeLockName(string lockName) + { + const string prefix = "elsa-health-check-"; + return lockName.StartsWith(prefix, StringComparison.Ordinal) + && Guid.TryParseExact(lockName[prefix.Length..], "N", out _); + } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs new file mode 100644 index 000000000..ec00da511 --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaRuntimeHealthCheckTests.cs @@ -0,0 +1,57 @@ +using Elsa.Workflows.Runtime.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class ElsaRuntimeHealthCheckTests +{ + private readonly IWorkflowRuntime _workflowRuntime = Substitute.For(); + private readonly IQuiescenceSignal _quiescenceSignal = Substitute.For(); + private readonly ElsaRuntimeHealthCheck _sut; + + public ElsaRuntimeHealthCheckTests() + { + _workflowRuntime.CreateClientAsync(Arg.Any()).Returns(new ValueTask(Substitute.For())); + _quiescenceSignal.CurrentState.Returns(QuiescenceState.Initial("test")); + _quiescenceSignal.ActiveExecutionCycleCount.Returns(0); + _sut = new ElsaRuntimeHealthCheck(_workflowRuntime, _quiescenceSignal, NullLogger.Instance); + } + + [Fact] + public async Task ReturnsHealthyWhenRuntimeAcceptsNewWork() + { + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("runtime", result.Data["category"]); + Assert.True((bool)result.Data["acceptingNewWork"]); + } + + [Fact] + public async Task ReturnsDegradedWhenRuntimeIsPaused() + { + _quiescenceSignal.CurrentState.Returns(QuiescenceState.Initial("test") with + { + Reason = QuiescenceReason.AdministrativePause + }); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("AdministrativePause", result.Data["reason"]); + Assert.False((bool)result.Data["acceptingNewWork"]); + } + + [Fact] + public async Task ReturnsUnhealthyWhenRuntimeClientCannotBeCreated() + { + _workflowRuntime.CreateClientAsync(Arg.Any()).Returns>(_ => throw new InvalidOperationException("boom")); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("runtime", result.Data["category"]); + } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs new file mode 100644 index 000000000..a602d9a64 --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/ElsaWorkflowPersistenceHealthCheckTests.cs @@ -0,0 +1,181 @@ +using Elsa.Workflows.Management; +using Elsa.Workflows.Management.Filters; +using Elsa.Workflows.Runtime.Filters; +using Elsa.Workflows.Runtime.HealthChecks; +using Elsa.Workflows.Runtime.Options; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class ElsaWorkflowPersistenceHealthCheckTests +{ + private readonly IServiceProvider _serviceProvider = Substitute.For(); + private readonly IWorkflowDefinitionStore _workflowDefinitionStore = Substitute.For(); + private readonly IWorkflowInstanceStore _workflowInstanceStore = Substitute.For(); + private readonly ITriggerStore _triggerStore = Substitute.For(); + private readonly IBookmarkQueueStore _bookmarkQueueStore = Substitute.For(); + private readonly ElsaWorkflowPersistenceHealthCheck _sut; + + public ElsaWorkflowPersistenceHealthCheckTests() + { + _workflowDefinitionStore.FindAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(null)); + _workflowInstanceStore.CountAsync(Arg.Any(), Arg.Any()).Returns(new ValueTask(0)); + _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns(new ValueTask((Elsa.Workflows.Runtime.Entities.StoredTrigger?)null)); + _bookmarkQueueStore.FindAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(null)); + _serviceProvider.GetService(typeof(IWorkflowDefinitionStore)).Returns(_workflowDefinitionStore); + _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns(_workflowInstanceStore); + _serviceProvider.GetService(typeof(ITriggerStore)).Returns(_triggerStore); + _serviceProvider.GetService(typeof(IBookmarkQueueStore)).Returns(_bookmarkQueueStore); + _sut = CreateSut(); + } + + [Fact] + public async Task ReturnsHealthyWhenAllStoresCanBeRead() + { + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["successfulProbes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); + await _workflowDefinitionStore.Received(1).FindAsync( + Arg.Is(x => x.Id == "00000000-0000-0000-0000-000000000000"), + Arg.Any()); + } + + [Fact] + public async Task ProbesStoresSequentially() + { + var tracker = new ProbeConcurrencyTracker(); + _workflowDefinitionStore.FindAsync(Arg.Any(), Arg.Any()) + .Returns(_ => TrackProbeAsync(tracker, null)); + _workflowInstanceStore.CountAsync(Arg.Any(), Arg.Any()) + .Returns(_ => new ValueTask(TrackProbeAsync(tracker, 0L))); + _triggerStore.FindAsync(Arg.Any(), Arg.Any()) + .Returns(_ => new ValueTask(TrackProbeAsync(tracker, null))); + _bookmarkQueueStore.FindAsync(Arg.Any(), Arg.Any()) + .Returns(_ => TrackProbeAsync(tracker, null)); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal(1, tracker.MaxConcurrentProbes); + } + + [Fact] + public async Task ReturnsUnhealthyWithFailedStoreAndStopsProbingWhenAStoreProbeFails() + { + _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("Elsa workflow store 'triggers' is not reachable.", result.Description); + Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("triggers", result.Data["failedStore"]); + Assert.Equal("triggers", result.Data["failedProbe"]); + Assert.Equal("workflow-definitions,workflow-instances", result.Data["successfulProbes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers", result.Data["attemptedProbes"]); + Assert.Equal("triggers", result.Data["failedProbes"]); + await _bookmarkQueueStore.DidNotReceive().FindAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ReturnsUnhealthyWithAllProbeDataWhenContinuationIsEnabled() + { + var sut = CreateSut(continueAfterFailure: true); + _triggerStore.FindAsync(Arg.Any(), Arg.Any()).Returns>(_ => throw new InvalidOperationException("store unavailable")); + + var result = await sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Equal("workflow-definitions,workflow-instances,bookmark-queue", result.Data["successfulProbes"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["attemptedProbes"]); + Assert.Equal("triggers", result.Data["failedProbes"]); + await _bookmarkQueueStore.Received(1).FindAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ReturnsHealthyWithSkippedProbesWhenOptionalManagementStoresAreMissing() + { + _serviceProvider.GetService(typeof(IWorkflowDefinitionStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns((object?)null); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("triggers,bookmark-queue", result.Data["successfulProbes"]); + Assert.Equal("triggers,bookmark-queue", result.Data["attemptedProbes"]); + Assert.Equal("workflow-definitions,workflow-instances", result.Data["skippedProbes"]); + } + + [Fact] + public async Task ReturnsDegradedWithSkippedProbesWhenNoStoresAreRegistered() + { + _serviceProvider.GetService(typeof(IWorkflowDefinitionStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(IWorkflowInstanceStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(ITriggerStore)).Returns((object?)null); + _serviceProvider.GetService(typeof(IBookmarkQueueStore)).Returns((object?)null); + + var result = await _sut.CheckHealthAsync(new HealthCheckContext()); + + Assert.Equal(HealthStatus.Degraded, result.Status); + Assert.Equal("No Elsa workflow persistence stores are registered.", result.Description); + Assert.Equal("persistence", result.Data["category"]); + Assert.Equal("workflow-definitions,workflow-instances,triggers,bookmark-queue", result.Data["skippedProbes"]); + Assert.False(result.Data.ContainsKey("successfulProbes")); + } + + private ElsaWorkflowPersistenceHealthCheck CreateSut(bool continueAfterFailure = false) + { + var options = Microsoft.Extensions.Options.Options.Create(new ElsaReadinessHealthCheckOptions + { + ContinuePersistenceProbesAfterFailure = continueAfterFailure + }); + + return new ElsaWorkflowPersistenceHealthCheck(_serviceProvider, options, NullLogger.Instance); + } + + private static async Task TrackProbeAsync(ProbeConcurrencyTracker tracker, T result) + { + tracker.Enter(); + + try + { + await Task.Delay(10); + return result; + } + finally + { + tracker.Exit(); + } + } + + private sealed class ProbeConcurrencyTracker + { + private int _currentProbes; + private int _maxConcurrentProbes; + + public int MaxConcurrentProbes => Volatile.Read(ref _maxConcurrentProbes); + + public void Enter() + { + var currentProbes = Interlocked.Increment(ref _currentProbes); + + while (true) + { + var maxConcurrentProbes = MaxConcurrentProbes; + if (currentProbes <= maxConcurrentProbes) + return; + + if (Interlocked.CompareExchange(ref _maxConcurrentProbes, currentProbes, maxConcurrentProbes) == maxConcurrentProbes) + return; + } + } + + public void Exit() => Interlocked.Decrement(ref _currentProbes); + } +} diff --git a/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs new file mode 100644 index 000000000..c2ba45795 --- /dev/null +++ b/test/unit/Elsa.Workflows.Runtime.UnitTests/HealthChecks/HealthCheckExtensionsTests.cs @@ -0,0 +1,61 @@ +using Elsa.Extensions; +using Elsa.Workflows.Runtime.Options; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; + +namespace Elsa.Workflows.Runtime.UnitTests.HealthChecks; + +public class HealthCheckExtensionsTests +{ + [Fact] + public void AddElsaReadinessChecksRegistersReadinessOptions() + { + var services = new ServiceCollection(); + + services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false, includeDistributedLocks: true); + + using var serviceProvider = services.BuildServiceProvider(); + + Assert.NotNull(serviceProvider.GetRequiredService>()); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void AddElsaReadinessChecksRejectsNonPositiveDistributedLockTimeout(int timeoutMilliseconds) + { + var services = new ServiceCollection(); + + services + .AddHealthChecks() + .AddElsaReadinessChecks( + includePersistence: false, + includeDistributedLocks: true, + configureOptions: options => options.DistributedLockAcquisitionTimeout = TimeSpan.FromMilliseconds(timeoutMilliseconds)); + + using var serviceProvider = services.BuildServiceProvider(); + + Assert.Throws(() => serviceProvider.GetRequiredService>().Value); + } + + [Fact] + public void AddElsaReadinessChecksUsesElsaSpecificReadinessTag() + { + var services = new ServiceCollection(); + + services + .AddHealthChecks() + .AddElsaReadinessChecks(includePersistence: false); + + using var serviceProvider = services.BuildServiceProvider(); + var registrations = serviceProvider.GetRequiredService>().Value.Registrations; + + var registration = Assert.Single(registrations); + Assert.Contains("elsa", registration.Tags); + Assert.Contains(HealthCheckExtensions.ReadinessTag, registration.Tags); + Assert.DoesNotContain("readiness", registration.Tags); + } +}