From fd31b5b60500dc433776d7086184e52fec4744d2 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Fri, 14 Mar 2025 23:17:54 +0100 Subject: [PATCH 01/11] Update OTEL fields and tags Added and updated multiple package references, including OpenTelemetry, Datadog.Trace.Bundle, and various Microsoft.Extensions libraries. This ensures compatibility with the latest dependencies and introduces enhanced features for resilience and tracing. --- Directory.Packages.props | 456 ++++++++++-------- Elsa.sln | 1 + docker/ElsaServer-Datadog.Dockerfile | 2 +- docker/docker-compose-datadog.yml | 134 ++--- .../Elsa.Server.Web/Elsa.Server.Web.csproj | 7 + src/apps/Elsa.Server.Web/Program.cs | 81 +++- src/apps/Elsa.Server.Web/appsettings.json | 1 + .../Handlers/DefaultErrorSpanHandler.cs | 17 +- .../FaultExceptionErrorSpanHandler.cs | 14 +- ...metryTracingActivityExecutionMiddleware.cs | 55 ++- ...metryTracingWorkflowExecutionMiddleware.cs | 104 ++-- 11 files changed, 480 insertions(+), 392 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 626f995bf..8ef10f029 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,206 +1,254 @@ - - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Elsa.sln b/Elsa.sln index f78fa138b..f5901cf8a 100644 --- a/Elsa.sln +++ b/Elsa.sln @@ -87,6 +87,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{986E54 docker\ElsaStudio.Dockerfile = docker\ElsaStudio.Dockerfile docker\otel-collector-config.yaml = docker\otel-collector-config.yaml docker\init-db-postgres.sh = docker\init-db-postgres.sh + docker\docker-compose-datadog+otel-collector.yml = docker\docker-compose-datadog+otel-collector.yml EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Elasticsearch", "src\modules\Elsa.Elasticsearch\Elsa.Elasticsearch.csproj", "{3246883E-2FA7-4B4A-BDC5-99039A2869BC}" diff --git a/docker/ElsaServer-Datadog.Dockerfile b/docker/ElsaServer-Datadog.Dockerfile index a74069cf2..7d5022c63 100644 --- a/docker/ElsaServer-Datadog.Dockerfile +++ b/docker/ElsaServer-Datadog.Dockerfile @@ -10,7 +10,7 @@ COPY ./NuGet.Config ./ COPY *.props ./ # Restore packages. -RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj" +RUN dotnet restore "./src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj" # Build and publish (UseAppHost=false creates platform independent binaries). WORKDIR /source/src/bundles/Elsa.Server.Web diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml index e90991c45..54b2098e5 100644 --- a/docker/docker-compose-datadog.yml +++ b/docker/docker-compose-datadog.yml @@ -1,115 +1,77 @@ services: - postgres: - image: postgres:latest - command: -c 'max_connections=2000' - environment: - POSTGRES_USER: elsa - POSTGRES_PASSWORD: elsa - POSTGRES_DB: elsa - volumes: - - postgres-data:/var/lib/postgresql/data - ports: - - "5432:5432" - - rabbitmq: - image: "rabbitmq:3-management" - ports: - - "15672:15672" - - "5672:5672" - - redis: - image: redis:latest - ports: - - "127.0.0.1:6379:6379" - - elsa-server: - pull_policy: always - build: - context: ../. - dockerfile: ./docker/ElsaServer-Datadog.Dockerfile - depends_on: - - postgres - - rabbitmq - - redis - - otel-collector - environment: - DD_AGENT_HOST: datadog-agent - DD_ENV: development - DD_TRACE_DEBUG: true - DD_TRACE_OTEL_ENABLED: true - DD_SERVICE: "Elsa Server" - DD_VERSION: "3.3.0" - # OpenTelemetry environment variables - OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" # Point to OpenTelemetry Collector - OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" # Use gRPC for OTLP - OTEL_TRACES_EXPORTER: "otlp" - OTEL_METRICS_EXPORTER: "otlp" - OTEL_LOGS_EXPORTER: "otpl" - OTEL_RESOURCE_ATTRIBUTES: "service.name=elsa-server-local,service.version=3.2.1-blueberry,deployment.environment=development" - OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES: "Elsa.Workflows" - OTEL_DOTNET_AUTO_INSTRUMENTATION_ENABLED: "true" - OTEL_LOG_LEVEL: "debug" - OTEL_DOTNET_AUTO_RESOURCE_DETECTOR_ENABLED: "true" - OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED: "true" - OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED: "true" - OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED: "true" - - ASPNETCORE_ENVIRONMENT: Development - PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 - PYTHONNET_RUNTIME: coreclr - CONNECTIONSTRINGS__POSTGRESQL: "Server=postgres;Username=elsa;Database=elsa;Port=5432;Password=elsa;SSLMode=Prefer" - CONNECTIONSTRINGS__RABBITMQ: "amqp://guest:guest@rabbitmq:5672/" - CONNECTIONSTRINGS__REDIS: "redis:6379" - DISTRIBUTEDLOCKPROVIDER: "Postgres" - ports: - - "13000:8080" - elsa-studio: pull_policy: always build: context: ../. dockerfile: ./docker/ElsaStudio.Dockerfile + depends_on: + - elsa-server environment: ASPNETCORE_ENVIRONMENT: Development ELSASERVER__URL: "http://localhost:13000/elsa/api" ports: - "14000:8080" - - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - command: [ "--config", "/etc/otel-collector-config.yaml", "--feature-gates", "-component.UseLocalHostAsDefaultHost" ] + + elsa-server: + pull_policy: always + build: + context: ../. + dockerfile: ./docker/ElsaServer.Dockerfile + depends_on: + - datadog-agent environment: - DD_API_KEY: "secret api key" - DD_SITE: "datadoghq.eu" + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: false + OTEL_RESOURCE_ATTRIBUTES: service.name=aspnet-app + OTEL_EXPORTER_OTLP_ENDPOINT: http://datadog-agent:4317 + OTEL_EXPORTER_OTLP_HEADERS: "DD_API_KEY=4cdbf04b80065d715d9d12ee1fd84e22" + DD_AGENT_HOST: datadog-agent + DD_ENV: development + DD_TRACE_DEBUG: true + DD_TRACE_ENABLED: true + DD_TRACE_OTEL_ENABLED: true + DD_TRACE_LOGGING_ENABLED: true + DD_SERVICE: "Elsa Server" + DD_VERSION: "3.4.0" + ASPNETCORE_ENVIRONMENT: Development + PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 + PYTHONNET_RUNTIME: coreclr ports: - - "13133:13133" - - "4317:4317" - - "4318:4318" + - "13000:8080" datadog-agent: - image: datadog/agent:latest + image: gcr.io/datadoghq/agent:7 environment: - DD_API_KEY: "" + DD_API_KEY: "b430503d709680c951cfe3979ae35068" DD_SITE: "datadoghq.eu" - DD_HOSTNAME: "otel-collector" + DD_HOSTNAME: "datadog-agent" DD_LOGS_ENABLED: "true" DD_OTLP_CONFIG_LOGS_ENABLED: "true" + DD_OTLP_CONFIG_TRACES_ENABLED: "true" + DD_OTLP_CONFIG_METRICS_ENABLED: "true" DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true" DD_APM_ENABLED: "true" DD_APM_NON_LOCAL_TRAFFIC: "true" - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 # The Datadog Agent expects traces from OpenTelemetry Collector + DD_DOGSTATSD_NON_LOCAL_TRAFFIC: "true" + DD_APM_RECEIVER_SOCKET: /var/run/datadog/apm.socket + DD_DOGSTATSD_SOCKET: /var/run/datadog/dsd.socket + DD_APM_INSTRUMENTATION_ENABLED: true + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: 0.0.0.0:4318 - + # Service autodiscovery - DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server" + DD_AC_INCLUDE: "name:elsa-workflows" DD_AC_EXCLUDE: "name:datadog-agent" - + ports: - - "8126:8126" - - "14317:4317" - - "14318:4318" + - "8126:8126" # APM + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + + volumes: + - /var/run/datadog:/var/run/datadog + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc/:/host/proc/:ro + - /sys/fs/cgroup:/host/sys/fs/cgroup:ro volumes: postgres-data: \ No newline at end of file diff --git a/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj b/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj index e48fa21f3..4e4903a35 100644 --- a/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj +++ b/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj @@ -59,6 +59,7 @@ + @@ -68,7 +69,13 @@ + + + + + + diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 4035178eb..29eb7d12a 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -68,6 +68,10 @@ using Medallion.Threading.Redis; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Options; using OpenTelemetry; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Proto.Cluster.Kubernetes; using Proto.Persistence.Sqlite; @@ -96,7 +100,7 @@ const bool useTenantsFromConfiguration = true; const bool useSecrets = false; const bool disableVariableWrappers = false; const bool disableVariableCopying = false; -const bool useOtel = false; +const bool useManualOtelInstrumentation = true; var builder = WebApplication.CreateBuilder(args); var services = builder.Services; @@ -123,14 +127,46 @@ var sqlDatabaseProvider = Enum.Parse(configuration["Databas TypeAliasRegistry.RegisterAlias("OrderReceivedProducerFactory", typeof(GenericProducerFactory)); TypeAliasRegistry.RegisterAlias("OrderReceivedConsumerFactory", typeof(GenericConsumerFactory)); -if (useOtel) +if (useManualOtelInstrumentation) { -// Configure OpenTelemetry Tracing - using var tracerProvider = Sdk.CreateTracerProviderBuilder() - .AddSource("Elsa.Workflows") // Match your ActivitySource name here - .SetSampler(new AlwaysOnSampler()) // Always record traces for testing - .AddConsoleExporter() // Export spans to the console (optional) - .Build(); + // Configure OpenTelemetry Tracing + // using var tracerProvider = Sdk.CreateTracerProviderBuilder() + // .AddSource("Elsa.Workflows") // Match your ActivitySource name here + // .SetSampler(new AlwaysOnSampler()) // Always record traces for testing + // .AddConsoleExporter() // Export spans to the console (optional) + // .Build(); + + services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService("elsa-workflows", serviceVersion: "3.4.0").AddTelemetrySdk()) + .WithTracing(tracing => + { + tracing + .AddSource("*") + .SetSampler(new AlwaysOnSampler()) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSqlClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + ; + }) + .WithMetrics(metrics => + { + metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + ; + }); + + // Enable OpenTelemetry Logging (optional) + builder.Logging.AddOpenTelemetry(options => + { + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + options.ParseStateValues = true; + }); } // Add Elsa services. @@ -221,7 +257,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -269,7 +307,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -322,7 +362,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -474,7 +516,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -577,7 +621,7 @@ services } }); } - + if (useKafka) { elsa.UseKafka(kafka => @@ -585,7 +629,7 @@ services kafka.ConfigureOptions(options => configuration.GetSection("Kafka").Bind(options)); }); } - + if (useSecrets) { elsa @@ -661,8 +705,11 @@ services if (sqlDatabaseProvider == SqlDatabaseProvider.PostgreSql) ef.UsePostgreSql(postgresConnectionString); if (sqlDatabaseProvider == SqlDatabaseProvider.Citus) ef.UsePostgreSql(citusConnectionString); if (sqlDatabaseProvider == SqlDatabaseProvider.YugabyteDb) ef.UsePostgreSql(yugabyteDbConnectionString); - if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) + ef.UseOracle(oracleConnectionString, new() + { + SchemaName = "ELSA" + }); #if !NET9_0 if (sqlDatabaseProvider == SqlDatabaseProvider.MySql) ef.UseMySql(mySqlConnectionString); @@ -682,7 +729,7 @@ services tenantHttpRouting.WithTenantHeader("X-Tenant-ID"); }); } - + elsa.UseWebhooks(webhooks => webhooks.ConfigureSinks += options => builder.Configuration.GetSection("Webhooks").Bind(options)); elsa.InstallDropIns(options => options.DropInRootDirectory = Path.Combine(Directory.GetCurrentDirectory(), "App_Data", "DropIns")); elsa.AddSwagger(); diff --git a/src/apps/Elsa.Server.Web/appsettings.json b/src/apps/Elsa.Server.Web/appsettings.json index 3fc86acdb..91735a81b 100644 --- a/src/apps/Elsa.Server.Web/appsettings.json +++ b/src/apps/Elsa.Server.Web/appsettings.json @@ -2,6 +2,7 @@ "Logging": { "LogLevel": { "Default": "Warning", + "OpenTelemetry": "Debug", "Microsoft.Hosting.Lifetime": "Information", "Elsa": "Information", "Elsa.Workflows.Runtime.Middleware.Workflows.WorkflowHeartbeatMiddleware": "Debug" diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs index 04bebef62..a3003ba35 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs @@ -7,18 +7,9 @@ public class DefaultErrorSpanHandler : ErrorSpanHandlerBase { public override void Handle(ErrorSpanContext context) { - var span = context.Span; - var exception = context.Exception; - var errorMessage = string.IsNullOrWhiteSpace(exception?.Message) ? "Unknown error" : exception.Message; - span.SetTag("error", true); - span.SetTag("error.message", errorMessage); - - if (exception != null) - { - span.SetTag("error.exceptionType", exception.GetType().FullName); - - if (!string.IsNullOrEmpty(exception.StackTrace)) - span.SetTag("error.stackTrace", exception.StackTrace); - } + if (context.Exception is null) + return; + + context.Span.AddException(context.Exception); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs index 14aa2860a..301b2b78d 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs @@ -8,12 +8,16 @@ public class FaultExceptionErrorSpanHandler : ErrorSpanHandlerBase { public override void Handle(ErrorSpanContext context) { - if(context.Exception is not FaultException faultException) + if (context.Exception is not FaultException faultException) return; - + var span = context.Span; - span.SetTag("error.code", faultException.Code); - span.SetTag("error.category", faultException.Category); - span.SetTag("error.faultType", faultException.Type); + var tags = new Dictionary + { + ["exception.code"] = faultException.Code, + ["exception.category"] = faultException.Category, + ["exception.type"] = faultException.Type + }; + span.AddException(context.Exception, new(tags.ToArray())); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 5410d70bf..60a2acd46 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -19,7 +19,7 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD public async ValueTask InvokeAsync(ActivityExecutionContext context) { var activity = context.Activity; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity("ActivityExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute activity {activity.Type}", ActivityKind.Internal, Activity.Current?.Context ?? default); if (span == null) { @@ -27,46 +27,51 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD return; } - span.SetTag("activity.nodeId", activity.NodeId); + span.SetTag("operation.name", "elsa.activity.execution"); + span.SetTag("activity.id", activity.NodeId); + span.SetTag("activity.node.id", activity.NodeId); span.SetTag("activity.type", activity.Type); span.SetTag("activity.name", activity.Name); - span.SetTag("activityInstance.id", context.Id); - span.SetTag("activityExecution.startTimeUtc", span.StartTimeUtc); - span.SetTag("tenantId", context.WorkflowExecutionContext.Workflow.Identity.TenantId); - - span.AddEvent(new("Executing", tags: CreateStatusTags(context))); - + span.SetTag("activity.version", activity.Version); + span.SetTag("activity.instance.id", context.Id); + span.SetTag("activity.tenant.id", context.WorkflowExecutionContext.Workflow.Identity.TenantId); + + span.AddEvent(new("executing")); + await next(context); if (context.Status == ActivityStatus.Faulted) { - span.AddEvent(new("Faulted", tags: CreateStatusTags(context))); + span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("activityInstance.hasIncidents", true); + span.SetTag("activity.incidents", true); var errorSpanHandlers = context.GetServices(); var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception); - - foreach (var handler in errorSpanHandlers) + + foreach (var handler in errorSpanHandlers) handler.Handle(errorSpanHandlerContext); } - else + else if (context.Status == ActivityStatus.Canceled) { - span.AddEvent(new("Executed", tags: CreateStatusTags(context))); + span.AddEvent(new("canceled")); span.SetStatus(ActivityStatusCode.Ok); } - - var now = systemClock.UtcNow; - span.SetTag("activityExecution.endTimeUtc", now); - span.SetTag("activityExecution.durationMs", (now - span.StartTimeUtc).TotalMilliseconds); - } - - private ActivityTagsCollection CreateStatusTags(ActivityExecutionContext context) - { - return new(new Dictionary + else if (context.Status == ActivityStatus.Running) { - ["activityInstance.status"] = context.Status.ToString() - }); + span.AddEvent(new("running")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.Status == ActivityStatus.Completed) + { + span.AddEvent(new("completed")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.Status == ActivityStatus.Pending) + { + span.AddEvent(new("pending")); + span.SetStatus(ActivityStatusCode.Ok); + } } } diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index c0e88ef64..7c61eac04 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -1,10 +1,13 @@ using System.Diagnostics; +using System.Runtime.InteropServices.Marshalling; using System.Text.Json; using Elsa.Common; using Elsa.Expressions.Services; using Elsa.Extensions; using Elsa.OpenTelemetry.Helpers; using Elsa.Workflows; +using Elsa.Workflows.Activities; +using Elsa.Workflows.Models; using Elsa.Workflows.Pipelines.WorkflowExecution; using Elsa.Workflows.Serialization.Converters; using JetBrains.Annotations; @@ -26,7 +29,8 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD { var workflowInstanceId = context.Id; var workflow = context.Workflow; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity("WorkflowExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute workflow {workflow.WorkflowMetadata.Name}", ActivityKind.Server, Activity.Current?.Context ?? default); + var now = systemClock.UtcNow; if (span == null) // No listener is registered. { @@ -34,61 +38,79 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD return; } - span.SetTag("workflowInstance.id", workflowInstanceId); - span.SetTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId); - span.SetTag("workflowDefinition.version", workflow.Identity.Version); - span.SetTag("workflowDefinition.name", workflow.WorkflowMetadata.Name); - span.SetTag("workflowExecution.startTimeUtc", span.StartTimeUtc); - span.SetTag("tenantId", workflow.Identity.TenantId); - - if(context.TriggerActivityId != null) + span.SetTag("operation.name", "elsa.workflow.execution"); + span.SetTag("workflow.definition.id", workflow.Identity.DefinitionId); + span.SetTag("workflow.definition.version", workflow.Identity.Version); + span.SetTag("workflow.definition.name", workflow.WorkflowMetadata.Name); + span.SetTag("workflow.definition.tenant.id", workflow.Identity.TenantId); + span.SetTag("workflow.instance.id", workflowInstanceId); + + if (context.TriggerActivityId != null) { - var activity = context.FindActivityById(context.TriggerActivityId) ?? throw new Exception($"Trigger activity with ID {context.TriggerActivityId} not found. This should not happen."); - span.SetTag("workflowExecution.trigger.activityId", activity.Id); - span.SetTag("workflowExecution.trigger.activityName", activity.Name); - span.SetTag("workflowExecution.trigger.activityType", activity.Type); + var activity = context.FindActivityById(context.TriggerActivityId) ?? throw new($"Trigger activity with ID {context.TriggerActivityId} not found. This should not happen."); + span.SetTag("workflow.trigger.activity.id", activity.Id); + span.SetTag("workflow.trigger.activity.name", activity.Name); + span.SetTag("workflow.trigger.activity.type", activity.Type); + span.SetTag("workflow.trigger.activity.version", activity.Version); } - - span.AddEvent(new ActivityEvent("Executing", tags: CreateStatusTags(context))); + + span.AddEvent(new("executing")); await Next(context); if (context.SubStatus == WorkflowSubStatus.Faulted) { - span.AddEvent(new ActivityEvent("Faulted", tags: CreateStatusTags(context))); + span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("error", true); } - else + else if (context.SubStatus == WorkflowSubStatus.Finished) { - span.AddEvent(new ActivityEvent("Executed", tags: CreateStatusTags(context))); + span.AddEvent(new("finished")); span.SetStatus(ActivityStatusCode.Ok); } - - if(context.Incidents.Any()) + else if (context.SubStatus == WorkflowSubStatus.Cancelled) { - span.SetStatus(ActivityStatusCode.Error); - span.SetTag("workflowInstance.hasIncidents", true); - span.SetTag("error", true); - - if (context.Incidents.Count > 0) - span.SetTag("error.message", JsonSerializer.Serialize(context.Incidents, _incidentSerializerOptions)); + span.AddEvent(new("canceled")); + span.SetStatus(ActivityStatusCode.Ok); } - - if (!string.IsNullOrWhiteSpace(context.CorrelationId)) - span.SetTag("workflowInstance.correlationId", context.CorrelationId); - - var now = systemClock.UtcNow; - span.SetTag("workflowExecution.endTimeUtc", now); - span.SetTag("workflowExecution.durationMs", (now - span.StartTimeUtc).TotalMilliseconds); - } - - private ActivityTagsCollection CreateStatusTags(WorkflowExecutionContext context) - { - return new ActivityTagsCollection(new Dictionary + else if (context.SubStatus == WorkflowSubStatus.Suspended) { - ["workflowInstance.status"] = context.Status.ToString(), - ["workflowInstance.subStatus"] = context.SubStatus.ToString() + span.AddEvent(new("suspended")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.SubStatus == WorkflowSubStatus.Pending) + { + span.AddEvent(new("pending")); + span.SetStatus(ActivityStatusCode.Ok); + } + + if (context.Incidents.Any()) + { + span.SetTag("workflow.incidents", true); + span.SetTag("workflow.incidents.count", context.Incidents.Count); + + foreach (var incident in context.Incidents) + span.AddEvent(new("incident", incident.Timestamp, CreateIncidentTags(incident))); + } + + if (!string.IsNullOrWhiteSpace(context.CorrelationId)) + span.SetTag("workflow.correlation_id", context.CorrelationId); + } + + private ActivityTagsCollection CreateIncidentTags(ActivityIncident incident) + { + var tags = new ActivityTagsCollection(new Dictionary + { + ["incident.message"] = incident.Message, }); + + if (incident.Exception != null) + { + tags["incident.exception.message"] = incident.Exception.Message; + tags["incident.exception.stackTrace"] = incident.Exception.StackTrace; + tags["incident.exception.type"] = incident.Exception.GetType().FullName; + } + + return tags; } } From e393cca031aaad012aa4764ae58f4a049e549d21 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Fri, 14 Mar 2025 23:27:16 +0100 Subject: [PATCH 02/11] Update error status and exception type handling Enhanced error status to include detailed message when the workflow is faulted, providing better context in logs. Fixed the retrieval of exception type by replacing `GetType()` with `.Type` for improved accuracy in incident tagging. --- .../OpenTelemetryTracingWorkflowExecutionMiddleware.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index 7c61eac04..5ec20b78b 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -60,7 +60,7 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD if (context.SubStatus == WorkflowSubStatus.Faulted) { span.AddEvent(new("faulted")); - span.SetStatus(ActivityStatusCode.Error); + span.SetStatus(ActivityStatusCode.Error, "The workflow entered the Faulted state. See incidents for details."); } else if (context.SubStatus == WorkflowSubStatus.Finished) { @@ -107,7 +107,7 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD { tags["incident.exception.message"] = incident.Exception.Message; tags["incident.exception.stackTrace"] = incident.Exception.StackTrace; - tags["incident.exception.type"] = incident.Exception.GetType().FullName; + tags["incident.exception.type"] = incident.Exception.Type.FullName; } return tags; From b624b2b4afd8b3930507b849411fc4b2c740d3a0 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Fri, 14 Mar 2025 23:37:46 +0100 Subject: [PATCH 03/11] Refactor error span handling with prioritization and predicates Introduce `Order` property and `CanHandle` method in `IErrorSpanHandler` to enable prioritized and conditional processing of error spans. Update handler implementations to utilize these properties and streamline logic. This enhances flexibility and improves the extensibility of error span handling. --- .../Abstractions/ErrorSpanHandlerBase.cs | 2 ++ .../Elsa.OpenTelemetry/Contracts/IErrorSpanHandler.cs | 3 ++- .../Handlers/DefaultErrorSpanHandler.cs | 9 +++++---- .../Handlers/FaultExceptionErrorSpanHandler.cs | 6 +++--- .../OpenTelemetryTracingActivityExecutionMiddleware.cs | 6 ++++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/modules/Elsa.OpenTelemetry/Abstractions/ErrorSpanHandlerBase.cs b/src/modules/Elsa.OpenTelemetry/Abstractions/ErrorSpanHandlerBase.cs index 420936119..06e77bd4f 100644 --- a/src/modules/Elsa.OpenTelemetry/Abstractions/ErrorSpanHandlerBase.cs +++ b/src/modules/Elsa.OpenTelemetry/Abstractions/ErrorSpanHandlerBase.cs @@ -5,5 +5,7 @@ namespace Elsa.OpenTelemetry.Abstractions; public abstract class ErrorSpanHandlerBase : IErrorSpanHandler { + public virtual float Order => 0; + public abstract bool CanHandle(ErrorSpanContext context); public abstract void Handle(ErrorSpanContext context); } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Contracts/IErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Contracts/IErrorSpanHandler.cs index d15c3dee4..70b16f645 100644 --- a/src/modules/Elsa.OpenTelemetry/Contracts/IErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Contracts/IErrorSpanHandler.cs @@ -1,10 +1,11 @@ -using Elsa.OpenTelemetry.Abstractions; using Elsa.OpenTelemetry.Models; namespace Elsa.OpenTelemetry.Contracts; public interface IErrorSpanHandler { + float Order { get; } + bool CanHandle(ErrorSpanContext context); void Handle(ErrorSpanContext context); } diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs index a3003ba35..aac9fbae9 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs @@ -5,11 +5,12 @@ namespace Elsa.OpenTelemetry.Handlers; public class DefaultErrorSpanHandler : ErrorSpanHandlerBase { + public override float Order => 100000; + + public override bool CanHandle(ErrorSpanContext context) => context.Exception != null; + public override void Handle(ErrorSpanContext context) { - if (context.Exception is null) - return; - - context.Span.AddException(context.Exception); + context.Span.AddException(context.Exception!); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs index 301b2b78d..9403e20e9 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs @@ -6,11 +6,11 @@ namespace Elsa.OpenTelemetry.Handlers; public class FaultExceptionErrorSpanHandler : ErrorSpanHandlerBase { + public override bool CanHandle(ErrorSpanContext context) => context.Exception is FaultException; + public override void Handle(ErrorSpanContext context) { - if (context.Exception is not FaultException faultException) - return; - + var faultException = (FaultException)context.Exception!; var span = context.Span; var tags = new Dictionary { diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 60a2acd46..e11106912 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -46,9 +46,11 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD span.SetStatus(ActivityStatusCode.Error); span.SetTag("activity.incidents", true); - var errorSpanHandlers = context.GetServices(); var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception); - + var errorSpanHandlers = context.GetServices() + .OrderBy(x => x.Order) + .Where(x => x.CanHandle(errorSpanHandlerContext)); + foreach (var handler in errorSpanHandlers) handler.Handle(errorSpanHandlerContext); } From 39932a6f0914fe36271affb43657764d68836c5c Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Fri, 14 Mar 2025 23:55:02 +0100 Subject: [PATCH 04/11] Refactor OpenTelemetry tracing and improve span tagging. Introduced new span tags for activities and workflows to enhance trace analysis. Optimized error handling in activity tracing by selecting the first applicable handler. Removed unused manual tracing configuration and commented out console exporters in server setup. --- src/apps/Elsa.Server.Web/Program.cs | 11 ++--------- ...penTelemetryTracingActivityExecutionMiddleware.cs | 12 ++++++++---- ...penTelemetryTracingWorkflowExecutionMiddleware.cs | 1 + 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 29eb7d12a..192fecea0 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -129,13 +129,6 @@ TypeAliasRegistry.RegisterAlias("OrderReceivedConsumerFactory", typeof(GenericCo if (useManualOtelInstrumentation) { - // Configure OpenTelemetry Tracing - // using var tracerProvider = Sdk.CreateTracerProviderBuilder() - // .AddSource("Elsa.Workflows") // Match your ActivitySource name here - // .SetSampler(new AlwaysOnSampler()) // Always record traces for testing - // .AddConsoleExporter() // Export spans to the console (optional) - // .Build(); - services.AddOpenTelemetry() .ConfigureResource(resource => resource.AddService("elsa-workflows", serviceVersion: "3.4.0").AddTelemetrySdk()) .WithTracing(tracing => @@ -146,7 +139,7 @@ if (useManualOtelInstrumentation) .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() .AddSqlClientInstrumentation() - .AddConsoleExporter() + //.AddConsoleExporter() .AddOtlpExporter() ; }) @@ -155,7 +148,7 @@ if (useManualOtelInstrumentation) metrics .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() - .AddConsoleExporter() + //.AddConsoleExporter() .AddOtlpExporter() ; }); diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index e11106912..18b9ac477 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Elsa.Common; +using Elsa.Extensions; using Elsa.OpenTelemetry.Contracts; using Elsa.OpenTelemetry.Helpers; using Elsa.OpenTelemetry.Models; @@ -35,6 +36,10 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD span.SetTag("activity.version", activity.Version); span.SetTag("activity.instance.id", context.Id); span.SetTag("activity.tenant.id", context.WorkflowExecutionContext.Workflow.Identity.TenantId); + + var activityKind = context.ActivityDescriptor.Kind; + if (activityKind == Elsa.Workflows.ActivityKind.Job || (activityKind == Workflows.ActivityKind.Task && activity.GetRunAsynchronously())) + span.SetTag("span.type", "job"); span.AddEvent(new("executing")); @@ -47,12 +52,11 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD span.SetTag("activity.incidents", true); var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception); - var errorSpanHandlers = context.GetServices() + var errorSpanHandler = context.GetServices() .OrderBy(x => x.Order) - .Where(x => x.CanHandle(errorSpanHandlerContext)); + .FirstOrDefault(x => x.CanHandle(errorSpanHandlerContext)); - foreach (var handler in errorSpanHandlers) - handler.Handle(errorSpanHandlerContext); + errorSpanHandler?.Handle(errorSpanHandlerContext); } else if (context.Status == ActivityStatus.Canceled) { diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index 5ec20b78b..ed30d65e1 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -39,6 +39,7 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD } span.SetTag("operation.name", "elsa.workflow.execution"); + span.SetTag("span.type", "workflow"); span.SetTag("workflow.definition.id", workflow.Identity.DefinitionId); span.SetTag("workflow.definition.version", workflow.Identity.Version); span.SetTag("workflow.definition.name", workflow.WorkflowMetadata.Name); From 5f74a83a7784750a716d8eff2ba4fa2edb45bc23 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Sat, 15 Mar 2025 20:07:15 +0100 Subject: [PATCH 05/11] Remove redundant status handling in tracing middleware This commit eliminates unused status checks ("running" and "pending") from OpenTelemetry tracing middlewares. The changes simplify the codebase and improve maintainability by removing unnecessary logic. --- .../OpenTelemetryTracingActivityExecutionMiddleware.cs | 5 ----- .../OpenTelemetryTracingWorkflowExecutionMiddleware.cs | 7 ------- 2 files changed, 12 deletions(-) diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 18b9ac477..5cdd23085 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -63,11 +63,6 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD span.AddEvent(new("canceled")); span.SetStatus(ActivityStatusCode.Ok); } - else if (context.Status == ActivityStatus.Running) - { - span.AddEvent(new("running")); - span.SetStatus(ActivityStatusCode.Ok); - } else if (context.Status == ActivityStatus.Completed) { span.AddEvent(new("completed")); diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index ed30d65e1..cb209a0c0 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -22,8 +22,6 @@ namespace Elsa.OpenTelemetry.Middleware; [UsedImplicitly] public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareDelegate next, ISystemClock systemClock) : WorkflowExecutionMiddleware(next) { - private readonly JsonSerializerOptions? _incidentSerializerOptions = new JsonSerializerOptions().WithConverters(new TypeJsonConverter(WellKnownTypeRegistry.CreateDefault())); - /// public override async ValueTask InvokeAsync(WorkflowExecutionContext context) { @@ -78,11 +76,6 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD span.AddEvent(new("suspended")); span.SetStatus(ActivityStatusCode.Ok); } - else if (context.SubStatus == WorkflowSubStatus.Pending) - { - span.AddEvent(new("pending")); - span.SetStatus(ActivityStatusCode.Ok); - } if (context.Incidents.Any()) { From 83253b2faea8f79207ef30e500f02fbab9c95323 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Sat, 15 Mar 2025 21:14:10 +0100 Subject: [PATCH 06/11] Add support for starting new trace contexts in workflows Introduced `StartNewTrace` input to workflows and updated OpenTelemetry middleware to support initiating new trace contexts. This helps improve trace isolation and linking when desired, enhancing observability during workflow execution. --- ...metryTracingWorkflowExecutionMiddleware.cs | 19 +++++++++++-- .../Activities/BulkDispatchWorkflows.cs | 15 ++++++++--- .../Activities/DispatchWorkflow.cs | 27 ++++++++++++------- .../Services/BackgroundWorkflowDispatcher.cs | 8 +++--- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index cb209a0c0..a379dc384 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -27,8 +27,17 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD { var workflowInstanceId = context.Id; var workflow = context.Workflow; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute workflow {workflow.WorkflowMetadata.Name}", ActivityKind.Server, Activity.Current?.Context ?? default); - var now = systemClock.UtcNow; + var startNewTrace = context.Properties.TryGetValue("StartNewTrace", out var startNewTraceValue) && (bool)startNewTraceValue; + var parentTraceContext = startNewTrace ? default : Activity.Current?.Context ?? default; + var linkedTraceContext = startNewTrace ? Activity.Current : null; + + if(startNewTrace) + { + Activity.Current?.Stop(); + Activity.Current = null; + } + + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute workflow {workflow.WorkflowMetadata.Name}", ActivityKind.Server, parentTraceContext); if (span == null) // No listener is registered. { @@ -36,6 +45,12 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD return; } + if(startNewTrace) + { + if (linkedTraceContext != null) + span.AddLink(new(linkedTraceContext.Context)); + } + span.SetTag("operation.name", "elsa.workflow.execution"); span.SetTag("span.type", "workflow"); span.SetTag("workflow.definition.id", workflow.Identity.DefinitionId); diff --git a/src/modules/Elsa.Workflows.Runtime/Activities/BulkDispatchWorkflows.cs b/src/modules/Elsa.Workflows.Runtime/Activities/BulkDispatchWorkflows.cs index 8074be822..2207e05bd 100644 --- a/src/modules/Elsa.Workflows.Runtime/Activities/BulkDispatchWorkflows.cs +++ b/src/modules/Elsa.Workflows.Runtime/Activities/BulkDispatchWorkflows.cs @@ -78,6 +78,12 @@ public class BulkDispatchWorkflows : Activity Description = "Wait for the dispatched workflows to complete before completing this activity.", DefaultValue = true)] public Input WaitForCompletion { get; set; } = new(true); + + /// + /// Indicates whether a new trace context should be started for the workflow execution. + /// + [Input(Description = "Start a new trace context when using Open Telemetry.", Category = "Open Telemetry")] + public Input StartNewTrace { get; set; } /// /// The channel to dispatch the workflow to. @@ -104,12 +110,13 @@ public class BulkDispatchWorkflows : Activity protected override async ValueTask ExecuteAsync(ActivityExecutionContext context) { var waitForCompletion = WaitForCompletion.GetOrDefault(context); + var startNewTrace = StartNewTrace.GetOrDefault(context); var items = await context.GetItemSource(Items).ToListAsync(context.CancellationToken); var count = items.Count; // Dispatch the child workflows. foreach (var item in items) - await DispatchChildWorkflowAsync(context, item, waitForCompletion); + await DispatchChildWorkflowAsync(context, item, waitForCompletion, startNewTrace); // Store the number of dispatched instances for tracking. context.SetProperty(DispatchedInstancesCountKey, count); @@ -139,7 +146,7 @@ public class BulkDispatchWorkflows : Activity } } - private async ValueTask DispatchChildWorkflowAsync(ActivityExecutionContext context, object item, bool waitForCompletion) + private async ValueTask DispatchChildWorkflowAsync(ActivityExecutionContext context, object item, bool waitForCompletion, bool startNewTrace) { var workflowDefinitionId = WorkflowDefinitionId.Get(context); var workflowDefinitionService = context.GetRequiredService(); @@ -157,8 +164,8 @@ public class BulkDispatchWorkflows : Activity ["ParentInstanceId"] = parentInstanceId }; - if (waitForCompletion) - properties["WaitForCompletion"] = true; + if (waitForCompletion) properties["WaitForCompletion"] = true; + if (startNewTrace) properties["StartNewTrace"] = true; var itemDictionary = new Dictionary { diff --git a/src/modules/Elsa.Workflows.Runtime/Activities/DispatchWorkflow.cs b/src/modules/Elsa.Workflows.Runtime/Activities/DispatchWorkflow.cs index 566c79937..9619e1d73 100644 --- a/src/modules/Elsa.Workflows.Runtime/Activities/DispatchWorkflow.cs +++ b/src/modules/Elsa.Workflows.Runtime/Activities/DispatchWorkflow.cs @@ -20,7 +20,7 @@ namespace Elsa.Workflows.Runtime.Activities; public class DispatchWorkflow : Activity { /// - public DispatchWorkflow([CallerFilePath] string? source = default, [CallerLineNumber] int? line = default) : base(source, line) + public DispatchWorkflow([CallerFilePath] string? source = null, [CallerLineNumber] int? line = null) : base(source, line) { } @@ -32,7 +32,7 @@ public class DispatchWorkflow : Activity Description = "The definition ID of the workflow to dispatch.", UIHint = InputUIHints.WorkflowDefinitionPicker )] - public Input WorkflowDefinitionId { get; set; } = default!; + public Input WorkflowDefinitionId { get; set; } = null!; /// /// The correlation ID to associate the workflow with. @@ -41,19 +41,25 @@ public class DispatchWorkflow : Activity DisplayName = "Correlation ID", Description = "The correlation ID to associate the workflow with." )] - public Input CorrelationId { get; set; } = default!; + public Input CorrelationId { get; set; } = null!; /// /// The input to send to the workflow. /// [Input(Description = "The input to send to the workflow.")] - public Input?> Input { get; set; } = default!; + public Input?> Input { get; set; } = null!; /// /// True to wait for the child workflow to complete before completing this activity, false to "fire and forget". /// [Input(Description = "Wait for the child workflow to complete before completing this activity.")] - public Input WaitForCompletion { get; set; } = default!; + public Input WaitForCompletion { get; set; } = null!; + + /// + /// Indicates whether a new trace context should be started for the workflow execution. + /// + [Input(Description = "Start a new trace context when using Open Telemetry.", Category = "Open Telemetry")] + public Input StartNewTrace { get; set; } /// /// The channel to dispatch the workflow to. @@ -64,7 +70,7 @@ public class DispatchWorkflow : Activity UIHint = InputUIHints.DropDown, UIHandler = typeof(DispatcherChannelOptionsProvider) )] - public Input ChannelName { get; set; } = default!; + public Input ChannelName { get; set; } = null!; /// protected override async ValueTask ExecuteAsync(ActivityExecutionContext context) @@ -103,16 +109,17 @@ public class DispatchWorkflow : Activity var input = Input.GetOrDefault(context) ?? new Dictionary(); var channelName = ChannelName.GetOrDefault(context); + var startNewTrace = StartNewTrace.GetOrDefault(context); var parentInstanceId = context.WorkflowExecutionContext.Id; var properties = new Dictionary { - ["ParentInstanceId"] = parentInstanceId + ["ParentInstanceId"] = parentInstanceId, }; // If we need to wait for the child workflow to complete, set the property. This will be used by the ResumeDispatchWorkflowActivity handler. - if (waitForCompletion) - properties["WaitForCompletion"] = true; - + if (waitForCompletion) properties["WaitForCompletion"] = true; + if (startNewTrace) properties["StartNewTrace"] = true; + input["ParentInstanceId"] = parentInstanceId; var correlationId = CorrelationId.GetOrDefault(context); diff --git a/src/modules/Elsa.Workflows.Runtime/Services/BackgroundWorkflowDispatcher.cs b/src/modules/Elsa.Workflows.Runtime/Services/BackgroundWorkflowDispatcher.cs index e9cbefc06..09e889628 100644 --- a/src/modules/Elsa.Workflows.Runtime/Services/BackgroundWorkflowDispatcher.cs +++ b/src/modules/Elsa.Workflows.Runtime/Services/BackgroundWorkflowDispatcher.cs @@ -22,7 +22,7 @@ public class BackgroundWorkflowDispatcher : IWorkflowDispatcher } /// - public async Task DispatchAsync(DispatchWorkflowDefinitionRequest request, DispatchWorkflowOptions? options = default, CancellationToken cancellationToken = default) + public async Task DispatchAsync(DispatchWorkflowDefinitionRequest request, DispatchWorkflowOptions? options = null, CancellationToken cancellationToken = default) { var command = new DispatchWorkflowDefinitionCommand(request.DefinitionVersionId) { @@ -38,7 +38,7 @@ public class BackgroundWorkflowDispatcher : IWorkflowDispatcher } /// - public async Task DispatchAsync(DispatchWorkflowInstanceRequest request, DispatchWorkflowOptions? options = default, CancellationToken cancellationToken = default) + public async Task DispatchAsync(DispatchWorkflowInstanceRequest request, DispatchWorkflowOptions? options = null, CancellationToken cancellationToken = default) { var command = new DispatchWorkflowInstanceCommand(request.InstanceId){ BookmarkId = request.BookmarkId, @@ -52,7 +52,7 @@ public class BackgroundWorkflowDispatcher : IWorkflowDispatcher } /// - public async Task DispatchAsync(DispatchTriggerWorkflowsRequest request, DispatchWorkflowOptions? options = default, CancellationToken cancellationToken = default) + public async Task DispatchAsync(DispatchTriggerWorkflowsRequest request, DispatchWorkflowOptions? options = null, CancellationToken cancellationToken = default) { var command = new DispatchTriggerWorkflowsCommand(request.ActivityTypeName, request.BookmarkPayload) { @@ -67,7 +67,7 @@ public class BackgroundWorkflowDispatcher : IWorkflowDispatcher } /// - public async Task DispatchAsync(DispatchResumeWorkflowsRequest request, DispatchWorkflowOptions? options = default, CancellationToken cancellationToken = default) + public async Task DispatchAsync(DispatchResumeWorkflowsRequest request, DispatchWorkflowOptions? options = null, CancellationToken cancellationToken = default) { var command = new DispatchResumeWorkflowsCommand(request.ActivityTypeName, request.BookmarkPayload) { From 32da99e3ec7b2fe346ad095898ed0ff529947ab7 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Sun, 16 Mar 2025 19:49:35 +0100 Subject: [PATCH 07/11] Add docker-compose file for Datadog and OpenTelemetry setup Introduce a new docker-compose configuration to integrate Datadog and OpenTelemetry with services like PostgreSQL, RabbitMQ, Redis, and Elsa. Updated the Datadog API key in an existing compose file for security purposes. This setup enables detailed monitoring, tracing, and metrics collection for development environments. --- .../docker-compose-datadog+otel-collector.yml | 115 ++++++++++++++++++ docker/docker-compose-datadog.yml | 2 +- 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 docker/docker-compose-datadog+otel-collector.yml diff --git a/docker/docker-compose-datadog+otel-collector.yml b/docker/docker-compose-datadog+otel-collector.yml new file mode 100644 index 000000000..20a056580 --- /dev/null +++ b/docker/docker-compose-datadog+otel-collector.yml @@ -0,0 +1,115 @@ +services: + postgres: + image: postgres:latest + command: -c 'max_connections=2000' + environment: + POSTGRES_USER: elsa + POSTGRES_PASSWORD: elsa + POSTGRES_DB: elsa + volumes: + - postgres-data:/var/lib/postgresql/data + ports: + - "5432:5432" + + rabbitmq: + image: "rabbitmq:3-management" + ports: + - "15672:15672" + - "5672:5672" + + redis: + image: redis:latest + ports: + - "127.0.0.1:6379:6379" + + elsa-server: + pull_policy: always + build: + context: ../. + dockerfile: ./docker/ElsaServer-Datadog.Dockerfile + depends_on: + - postgres + - rabbitmq + - redis + - otel-collector + environment: + DD_AGENT_HOST: datadog-agent + DD_ENV: development + DD_TRACE_DEBUG: true + DD_TRACE_OTEL_ENABLED: true + DD_SERVICE: "Elsa Server" + DD_VERSION: "3.3.0" + # OpenTelemetry environment variables + OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" # Point to OpenTelemetry Collector + OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" # Use gRPC for OTLP + OTEL_TRACES_EXPORTER: "otlp" + OTEL_METRICS_EXPORTER: "otlp" + OTEL_LOGS_EXPORTER: "otpl" + OTEL_RESOURCE_ATTRIBUTES: "service.name=elsa-server-local,service.version=3.2.1-blueberry,deployment.environment=development" + OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES: "Elsa.Workflows" + OTEL_DOTNET_AUTO_INSTRUMENTATION_ENABLED: "true" + OTEL_LOG_LEVEL: "debug" + OTEL_DOTNET_AUTO_RESOURCE_DETECTOR_ENABLED: "true" + OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED: "true" + OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED: "true" + OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED: "true" + + ASPNETCORE_ENVIRONMENT: Development + PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 + PYTHONNET_RUNTIME: coreclr + CONNECTIONSTRINGS__POSTGRESQL: "Server=postgres;Username=elsa;Database=elsa;Port=5432;Password=elsa;SSLMode=Prefer" + CONNECTIONSTRINGS__RABBITMQ: "amqp://guest:guest@rabbitmq:5672/" + CONNECTIONSTRINGS__REDIS: "redis:6379" + DISTRIBUTEDLOCKPROVIDER: "Postgres" + ports: + - "13000:8080" + + elsa-studio: + pull_policy: always + build: + context: ../. + dockerfile: ./docker/ElsaStudio.Dockerfile + environment: + ASPNETCORE_ENVIRONMENT: Development + ELSASERVER__URL: "http://localhost:13000/elsa/api" + ports: + - "14000:8080" + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + command: [ "--config", "/etc/otel-collector-config.yaml", "--feature-gates", "-component.UseLocalHostAsDefaultHost" ] + environment: + DD_API_KEY: "secret api key" + DD_SITE: "datadoghq.eu" + ports: + - "13133:13133" + - "4317:4317" + - "4318:4318" + + datadog-agent: + image: datadog/agent:latest + environment: + DD_API_KEY: "" + DD_SITE: "datadoghq.eu" + DD_HOSTNAME: "otel-collector" + DD_LOGS_ENABLED: "true" + DD_OTLP_CONFIG_LOGS_ENABLED: "true" + DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true" + DD_APM_ENABLED: "true" + DD_APM_NON_LOCAL_TRAFFIC: "true" + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 # The Datadog Agent expects traces from OpenTelemetry Collector + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: 0.0.0.0:4318 + + # Service autodiscovery + DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server" + DD_AC_EXCLUDE: "name:datadog-agent" + + ports: + - "8126:8126" + - "14317:4317" + - "14318:4318" + +volumes: + postgres-data: \ No newline at end of file diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml index 54b2098e5..23c42357d 100644 --- a/docker/docker-compose-datadog.yml +++ b/docker/docker-compose-datadog.yml @@ -41,7 +41,7 @@ services: datadog-agent: image: gcr.io/datadoghq/agent:7 environment: - DD_API_KEY: "b430503d709680c951cfe3979ae35068" + DD_API_KEY: "" DD_SITE: "datadoghq.eu" DD_HOSTNAME: "datadog-agent" DD_LOGS_ENABLED: "true" From dc7b95835db7566523c493ee55ddd78846fa2ba0 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Sun, 16 Mar 2025 22:35:34 +0100 Subject: [PATCH 08/11] Handle null values explicitly for varbinary columns Added explicit handling for null values in varbinary columns by applying a `CAST(NULL AS varbinary(max))` operation. This ensures proper value insertion and avoids potential type errors. Cleaned up unnecessary comments and minor whitespace inconsistencies for improved readability. --- .../Extensions/BulkUpsertExtensions.cs | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/modules/Elsa.EntityFrameworkCore.Common/Extensions/BulkUpsertExtensions.cs b/src/modules/Elsa.EntityFrameworkCore.Common/Extensions/BulkUpsertExtensions.cs index 810b43ac9..b82b3fe6a 100644 --- a/src/modules/Elsa.EntityFrameworkCore.Common/Extensions/BulkUpsertExtensions.cs +++ b/src/modules/Elsa.EntityFrameworkCore.Common/Extensions/BulkUpsertExtensions.cs @@ -89,10 +89,7 @@ public static class BulkUpsertExtensions var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; var tableName = $"[{entityType.GetSchema()}].[{entityType.GetTableName()}]"; var storeObject = StoreObjectIdentifier.Table(entityType.GetTableName()!, entityType.GetSchema()); - - // Include shadow properties var props = entityType.GetProperties().ToList(); - var keyProp = entityType.FindProperty(keySelector.GetMemberAccess().Name)!; var keyColumnName = $"[{keyProp.GetColumnName(storeObject)}]"; var columnNames = props @@ -119,13 +116,18 @@ public static class BulkUpsertExtensions object? value = property.IsShadowProperty() ? dbContext.Entry(entity).Property(property.Name).CurrentValue : property.PropertyInfo?.GetValue(entity); - - var converter = property.GetTypeMapping().Converter; - if (converter != null) - value = converter.ConvertToProvider(value); - values.Add(paramName); - parameters.Add(value); + var converter = property.GetTypeMapping().Converter; + if (converter != null) + value = converter.ConvertToProvider(value)!; + + // Explicitly cast null values for varbinary columns + if (property.GetColumnType().StartsWith("varbinary", StringComparison.OrdinalIgnoreCase) && value is null) + values.Add("CAST(NULL AS varbinary(max))"); // Explicitly cast null + else + values.Add(paramName); + + parameters.Add(value!); } var line = $"({string.Join(", ", values)}){(i < entities.Count - 1 ? "," : string.Empty)}"; @@ -179,9 +181,9 @@ public static class BulkUpsertExtensions object? value = property.IsShadowProperty() ? dbContext.Entry(entity).Property(property.Name).CurrentValue : property.PropertyInfo?.GetValue(entity); - + var converter = property.GetTypeMapping().Converter; - if (converter != null) + if (converter != null) value = converter.ConvertToProvider(value); placeholders.Add(paramName); @@ -241,9 +243,9 @@ public static class BulkUpsertExtensions object? value = property.IsShadowProperty() ? dbContext.Entry(entity).Property(property.Name).CurrentValue : property.PropertyInfo?.GetValue(entity); - + var converter = property.GetTypeMapping().Converter; - if (converter != null) + if (converter != null) value = converter.ConvertToProvider(value); placeholders.Add(paramName); @@ -303,9 +305,9 @@ public static class BulkUpsertExtensions object? value = property.IsShadowProperty() ? dbContext.Entry(entity).Property(property.Name).CurrentValue : property.PropertyInfo?.GetValue(entity); - + var converter = property.GetTypeMapping().Converter; - if (converter != null) + if (converter != null) value = converter.ConvertToProvider(value); placeholders.Add(paramName); @@ -369,9 +371,9 @@ public static class BulkUpsertExtensions object? value = property.IsShadowProperty() ? dbContext.Entry(entity).Property(property.Name).CurrentValue : property.PropertyInfo?.GetValue(entity); - + var converter = property.GetTypeMapping().Converter; - if (converter != null) + if (converter != null) value = converter.ConvertToProvider(value); parameters.Add(value); From af63b1b59af4698475b138c520507b2568389390 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Mon, 17 Mar 2025 13:42:57 +0100 Subject: [PATCH 09/11] Ignore docker compose for local testing --- .gitignore | 2 + docker/docker-compose-datadog.yml | 77 ------------------------------- 2 files changed, 2 insertions(+), 77 deletions(-) delete mode 100644 docker/docker-compose-datadog.yml diff --git a/.gitignore b/.gitignore index d63bbd6bb..8a28551f4 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,5 @@ unlist.sh /artifacts /docker/data/ + +docker/docker-compose-datadog.yml diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml deleted file mode 100644 index 23c42357d..000000000 --- a/docker/docker-compose-datadog.yml +++ /dev/null @@ -1,77 +0,0 @@ -services: - elsa-studio: - pull_policy: always - build: - context: ../. - dockerfile: ./docker/ElsaStudio.Dockerfile - depends_on: - - elsa-server - environment: - ASPNETCORE_ENVIRONMENT: Development - ELSASERVER__URL: "http://localhost:13000/elsa/api" - ports: - - "14000:8080" - - elsa-server: - pull_policy: always - build: - context: ../. - dockerfile: ./docker/ElsaServer.Dockerfile - depends_on: - - datadog-agent - environment: - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: false - OTEL_RESOURCE_ATTRIBUTES: service.name=aspnet-app - OTEL_EXPORTER_OTLP_ENDPOINT: http://datadog-agent:4317 - OTEL_EXPORTER_OTLP_HEADERS: "DD_API_KEY=4cdbf04b80065d715d9d12ee1fd84e22" - DD_AGENT_HOST: datadog-agent - DD_ENV: development - DD_TRACE_DEBUG: true - DD_TRACE_ENABLED: true - DD_TRACE_OTEL_ENABLED: true - DD_TRACE_LOGGING_ENABLED: true - DD_SERVICE: "Elsa Server" - DD_VERSION: "3.4.0" - ASPNETCORE_ENVIRONMENT: Development - PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 - PYTHONNET_RUNTIME: coreclr - ports: - - "13000:8080" - - datadog-agent: - image: gcr.io/datadoghq/agent:7 - environment: - DD_API_KEY: "" - DD_SITE: "datadoghq.eu" - DD_HOSTNAME: "datadog-agent" - DD_LOGS_ENABLED: "true" - DD_OTLP_CONFIG_LOGS_ENABLED: "true" - DD_OTLP_CONFIG_TRACES_ENABLED: "true" - DD_OTLP_CONFIG_METRICS_ENABLED: "true" - DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true" - DD_APM_ENABLED: "true" - DD_APM_NON_LOCAL_TRAFFIC: "true" - DD_DOGSTATSD_NON_LOCAL_TRAFFIC: "true" - DD_APM_RECEIVER_SOCKET: /var/run/datadog/apm.socket - DD_DOGSTATSD_SOCKET: /var/run/datadog/dsd.socket - DD_APM_INSTRUMENTATION_ENABLED: true - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: 0.0.0.0:4318 - - # Service autodiscovery - DD_AC_INCLUDE: "name:elsa-workflows" - DD_AC_EXCLUDE: "name:datadog-agent" - - ports: - - "8126:8126" # APM - - "4317:4317" # OTLP gRPC - - "4318:4318" # OTLP HTTP - - volumes: - - /var/run/datadog:/var/run/datadog - - /var/run/docker.sock:/var/run/docker.sock:ro - - /proc/:/host/proc/:ro - - /sys/fs/cgroup:/host/sys/fs/cgroup:ro - -volumes: - postgres-data: \ No newline at end of file From 2d485d253d3cdb98c178c30feab15070b3814c87 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Mon, 17 Mar 2025 15:01:37 +0100 Subject: [PATCH 10/11] Remove redundant status setting in telemetry middleware. The `SetStatus(ActivityStatusCode.Ok)` calls were unnecessary as the default behavior already sets the appropriate status. This cleanup simplifies the code and avoids redundant operations, improving maintainability. --- .../OpenTelemetryTracingActivityExecutionMiddleware.cs | 3 --- .../OpenTelemetryTracingWorkflowExecutionMiddleware.cs | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 5cdd23085..8a7c6243b 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -61,17 +61,14 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD else if (context.Status == ActivityStatus.Canceled) { span.AddEvent(new("canceled")); - span.SetStatus(ActivityStatusCode.Ok); } else if (context.Status == ActivityStatus.Completed) { span.AddEvent(new("completed")); - span.SetStatus(ActivityStatusCode.Ok); } else if (context.Status == ActivityStatus.Pending) { span.AddEvent(new("pending")); - span.SetStatus(ActivityStatusCode.Ok); } } } diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index a379dc384..30babe692 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -79,17 +79,14 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD else if (context.SubStatus == WorkflowSubStatus.Finished) { span.AddEvent(new("finished")); - span.SetStatus(ActivityStatusCode.Ok); } else if (context.SubStatus == WorkflowSubStatus.Cancelled) { span.AddEvent(new("canceled")); - span.SetStatus(ActivityStatusCode.Ok); } else if (context.SubStatus == WorkflowSubStatus.Suspended) { span.AddEvent(new("suspended")); - span.SetStatus(ActivityStatusCode.Ok); } if (context.Incidents.Any()) From 9b0aa9d9220ac35840a772f212aa2f654d3ca516 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Mon, 17 Mar 2025 15:05:24 +0100 Subject: [PATCH 11/11] Remove redundant span tags for activity and workflow incidents The tags "activity.incidents" and "workflow.incidents" were removed as they are unnecessary and provide no additional value. This change simplifies the middleware implementation while retaining essential telemetry data. --- .../OpenTelemetryTracingActivityExecutionMiddleware.cs | 1 - .../OpenTelemetryTracingWorkflowExecutionMiddleware.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 8a7c6243b..6115f7e2d 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -49,7 +49,6 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD { span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("activity.incidents", true); var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception); var errorSpanHandler = context.GetServices() diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index 30babe692..837226655 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -91,7 +91,6 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD if (context.Incidents.Any()) { - span.SetTag("workflow.incidents", true); span.SetTag("workflow.incidents.count", context.Incidents.Count); foreach (var incident in context.Incidents)