From fd31b5b60500dc433776d7086184e52fec4744d2 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Fri, 14 Mar 2025 23:17:54 +0100 Subject: [PATCH] Update OTEL fields and tags Added and updated multiple package references, including OpenTelemetry, Datadog.Trace.Bundle, and various Microsoft.Extensions libraries. This ensures compatibility with the latest dependencies and introduces enhanced features for resilience and tracing. --- Directory.Packages.props | 456 ++++++++++-------- Elsa.sln | 1 + docker/ElsaServer-Datadog.Dockerfile | 2 +- docker/docker-compose-datadog.yml | 134 ++--- .../Elsa.Server.Web/Elsa.Server.Web.csproj | 7 + src/apps/Elsa.Server.Web/Program.cs | 81 +++- src/apps/Elsa.Server.Web/appsettings.json | 1 + .../Handlers/DefaultErrorSpanHandler.cs | 17 +- .../FaultExceptionErrorSpanHandler.cs | 14 +- ...metryTracingActivityExecutionMiddleware.cs | 55 ++- ...metryTracingWorkflowExecutionMiddleware.cs | 104 ++-- 11 files changed, 480 insertions(+), 392 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 626f995bf..8ef10f029 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,206 +1,254 @@ - - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Elsa.sln b/Elsa.sln index f78fa138b..f5901cf8a 100644 --- a/Elsa.sln +++ b/Elsa.sln @@ -87,6 +87,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{986E54 docker\ElsaStudio.Dockerfile = docker\ElsaStudio.Dockerfile docker\otel-collector-config.yaml = docker\otel-collector-config.yaml docker\init-db-postgres.sh = docker\init-db-postgres.sh + docker\docker-compose-datadog+otel-collector.yml = docker\docker-compose-datadog+otel-collector.yml EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Elasticsearch", "src\modules\Elsa.Elasticsearch\Elsa.Elasticsearch.csproj", "{3246883E-2FA7-4B4A-BDC5-99039A2869BC}" diff --git a/docker/ElsaServer-Datadog.Dockerfile b/docker/ElsaServer-Datadog.Dockerfile index a74069cf2..7d5022c63 100644 --- a/docker/ElsaServer-Datadog.Dockerfile +++ b/docker/ElsaServer-Datadog.Dockerfile @@ -10,7 +10,7 @@ COPY ./NuGet.Config ./ COPY *.props ./ # Restore packages. -RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj" +RUN dotnet restore "./src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj" # Build and publish (UseAppHost=false creates platform independent binaries). WORKDIR /source/src/bundles/Elsa.Server.Web diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml index e90991c45..54b2098e5 100644 --- a/docker/docker-compose-datadog.yml +++ b/docker/docker-compose-datadog.yml @@ -1,115 +1,77 @@ services: - postgres: - image: postgres:latest - command: -c 'max_connections=2000' - environment: - POSTGRES_USER: elsa - POSTGRES_PASSWORD: elsa - POSTGRES_DB: elsa - volumes: - - postgres-data:/var/lib/postgresql/data - ports: - - "5432:5432" - - rabbitmq: - image: "rabbitmq:3-management" - ports: - - "15672:15672" - - "5672:5672" - - redis: - image: redis:latest - ports: - - "127.0.0.1:6379:6379" - - elsa-server: - pull_policy: always - build: - context: ../. - dockerfile: ./docker/ElsaServer-Datadog.Dockerfile - depends_on: - - postgres - - rabbitmq - - redis - - otel-collector - environment: - DD_AGENT_HOST: datadog-agent - DD_ENV: development - DD_TRACE_DEBUG: true - DD_TRACE_OTEL_ENABLED: true - DD_SERVICE: "Elsa Server" - DD_VERSION: "3.3.0" - # OpenTelemetry environment variables - OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" # Point to OpenTelemetry Collector - OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" # Use gRPC for OTLP - OTEL_TRACES_EXPORTER: "otlp" - OTEL_METRICS_EXPORTER: "otlp" - OTEL_LOGS_EXPORTER: "otpl" - OTEL_RESOURCE_ATTRIBUTES: "service.name=elsa-server-local,service.version=3.2.1-blueberry,deployment.environment=development" - OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES: "Elsa.Workflows" - OTEL_DOTNET_AUTO_INSTRUMENTATION_ENABLED: "true" - OTEL_LOG_LEVEL: "debug" - OTEL_DOTNET_AUTO_RESOURCE_DETECTOR_ENABLED: "true" - OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED: "true" - OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED: "true" - OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED: "true" - - ASPNETCORE_ENVIRONMENT: Development - PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 - PYTHONNET_RUNTIME: coreclr - CONNECTIONSTRINGS__POSTGRESQL: "Server=postgres;Username=elsa;Database=elsa;Port=5432;Password=elsa;SSLMode=Prefer" - CONNECTIONSTRINGS__RABBITMQ: "amqp://guest:guest@rabbitmq:5672/" - CONNECTIONSTRINGS__REDIS: "redis:6379" - DISTRIBUTEDLOCKPROVIDER: "Postgres" - ports: - - "13000:8080" - elsa-studio: pull_policy: always build: context: ../. dockerfile: ./docker/ElsaStudio.Dockerfile + depends_on: + - elsa-server environment: ASPNETCORE_ENVIRONMENT: Development ELSASERVER__URL: "http://localhost:13000/elsa/api" ports: - "14000:8080" - - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - command: [ "--config", "/etc/otel-collector-config.yaml", "--feature-gates", "-component.UseLocalHostAsDefaultHost" ] + + elsa-server: + pull_policy: always + build: + context: ../. + dockerfile: ./docker/ElsaServer.Dockerfile + depends_on: + - datadog-agent environment: - DD_API_KEY: "secret api key" - DD_SITE: "datadoghq.eu" + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: false + OTEL_RESOURCE_ATTRIBUTES: service.name=aspnet-app + OTEL_EXPORTER_OTLP_ENDPOINT: http://datadog-agent:4317 + OTEL_EXPORTER_OTLP_HEADERS: "DD_API_KEY=4cdbf04b80065d715d9d12ee1fd84e22" + DD_AGENT_HOST: datadog-agent + DD_ENV: development + DD_TRACE_DEBUG: true + DD_TRACE_ENABLED: true + DD_TRACE_OTEL_ENABLED: true + DD_TRACE_LOGGING_ENABLED: true + DD_SERVICE: "Elsa Server" + DD_VERSION: "3.4.0" + ASPNETCORE_ENVIRONMENT: Development + PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 + PYTHONNET_RUNTIME: coreclr ports: - - "13133:13133" - - "4317:4317" - - "4318:4318" + - "13000:8080" datadog-agent: - image: datadog/agent:latest + image: gcr.io/datadoghq/agent:7 environment: - DD_API_KEY: "" + DD_API_KEY: "b430503d709680c951cfe3979ae35068" DD_SITE: "datadoghq.eu" - DD_HOSTNAME: "otel-collector" + DD_HOSTNAME: "datadog-agent" DD_LOGS_ENABLED: "true" DD_OTLP_CONFIG_LOGS_ENABLED: "true" + DD_OTLP_CONFIG_TRACES_ENABLED: "true" + DD_OTLP_CONFIG_METRICS_ENABLED: "true" DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true" DD_APM_ENABLED: "true" DD_APM_NON_LOCAL_TRAFFIC: "true" - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 # The Datadog Agent expects traces from OpenTelemetry Collector + DD_DOGSTATSD_NON_LOCAL_TRAFFIC: "true" + DD_APM_RECEIVER_SOCKET: /var/run/datadog/apm.socket + DD_DOGSTATSD_SOCKET: /var/run/datadog/dsd.socket + DD_APM_INSTRUMENTATION_ENABLED: true + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: 0.0.0.0:4318 - + # Service autodiscovery - DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server" + DD_AC_INCLUDE: "name:elsa-workflows" DD_AC_EXCLUDE: "name:datadog-agent" - + ports: - - "8126:8126" - - "14317:4317" - - "14318:4318" + - "8126:8126" # APM + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + + volumes: + - /var/run/datadog:/var/run/datadog + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc/:/host/proc/:ro + - /sys/fs/cgroup:/host/sys/fs/cgroup:ro volumes: postgres-data: \ No newline at end of file diff --git a/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj b/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj index e48fa21f3..4e4903a35 100644 --- a/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj +++ b/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj @@ -59,6 +59,7 @@ + @@ -68,7 +69,13 @@ + + + + + + diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 4035178eb..29eb7d12a 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -68,6 +68,10 @@ using Medallion.Threading.Redis; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Options; using OpenTelemetry; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Proto.Cluster.Kubernetes; using Proto.Persistence.Sqlite; @@ -96,7 +100,7 @@ const bool useTenantsFromConfiguration = true; const bool useSecrets = false; const bool disableVariableWrappers = false; const bool disableVariableCopying = false; -const bool useOtel = false; +const bool useManualOtelInstrumentation = true; var builder = WebApplication.CreateBuilder(args); var services = builder.Services; @@ -123,14 +127,46 @@ var sqlDatabaseProvider = Enum.Parse(configuration["Databas TypeAliasRegistry.RegisterAlias("OrderReceivedProducerFactory", typeof(GenericProducerFactory)); TypeAliasRegistry.RegisterAlias("OrderReceivedConsumerFactory", typeof(GenericConsumerFactory)); -if (useOtel) +if (useManualOtelInstrumentation) { -// Configure OpenTelemetry Tracing - using var tracerProvider = Sdk.CreateTracerProviderBuilder() - .AddSource("Elsa.Workflows") // Match your ActivitySource name here - .SetSampler(new AlwaysOnSampler()) // Always record traces for testing - .AddConsoleExporter() // Export spans to the console (optional) - .Build(); + // Configure OpenTelemetry Tracing + // using var tracerProvider = Sdk.CreateTracerProviderBuilder() + // .AddSource("Elsa.Workflows") // Match your ActivitySource name here + // .SetSampler(new AlwaysOnSampler()) // Always record traces for testing + // .AddConsoleExporter() // Export spans to the console (optional) + // .Build(); + + services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService("elsa-workflows", serviceVersion: "3.4.0").AddTelemetrySdk()) + .WithTracing(tracing => + { + tracing + .AddSource("*") + .SetSampler(new AlwaysOnSampler()) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSqlClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + ; + }) + .WithMetrics(metrics => + { + metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + ; + }); + + // Enable OpenTelemetry Logging (optional) + builder.Logging.AddOpenTelemetry(options => + { + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + options.ParseStateValues = true; + }); } // Add Elsa services. @@ -221,7 +257,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -269,7 +307,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -322,7 +362,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -474,7 +516,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -577,7 +621,7 @@ services } }); } - + if (useKafka) { elsa.UseKafka(kafka => @@ -585,7 +629,7 @@ services kafka.ConfigureOptions(options => configuration.GetSection("Kafka").Bind(options)); }); } - + if (useSecrets) { elsa @@ -661,8 +705,11 @@ services if (sqlDatabaseProvider == SqlDatabaseProvider.PostgreSql) ef.UsePostgreSql(postgresConnectionString); if (sqlDatabaseProvider == SqlDatabaseProvider.Citus) ef.UsePostgreSql(citusConnectionString); if (sqlDatabaseProvider == SqlDatabaseProvider.YugabyteDb) ef.UsePostgreSql(yugabyteDbConnectionString); - if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) + ef.UseOracle(oracleConnectionString, new() + { + SchemaName = "ELSA" + }); #if !NET9_0 if (sqlDatabaseProvider == SqlDatabaseProvider.MySql) ef.UseMySql(mySqlConnectionString); @@ -682,7 +729,7 @@ services tenantHttpRouting.WithTenantHeader("X-Tenant-ID"); }); } - + elsa.UseWebhooks(webhooks => webhooks.ConfigureSinks += options => builder.Configuration.GetSection("Webhooks").Bind(options)); elsa.InstallDropIns(options => options.DropInRootDirectory = Path.Combine(Directory.GetCurrentDirectory(), "App_Data", "DropIns")); elsa.AddSwagger(); diff --git a/src/apps/Elsa.Server.Web/appsettings.json b/src/apps/Elsa.Server.Web/appsettings.json index 3fc86acdb..91735a81b 100644 --- a/src/apps/Elsa.Server.Web/appsettings.json +++ b/src/apps/Elsa.Server.Web/appsettings.json @@ -2,6 +2,7 @@ "Logging": { "LogLevel": { "Default": "Warning", + "OpenTelemetry": "Debug", "Microsoft.Hosting.Lifetime": "Information", "Elsa": "Information", "Elsa.Workflows.Runtime.Middleware.Workflows.WorkflowHeartbeatMiddleware": "Debug" diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs index 04bebef62..a3003ba35 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs @@ -7,18 +7,9 @@ public class DefaultErrorSpanHandler : ErrorSpanHandlerBase { public override void Handle(ErrorSpanContext context) { - var span = context.Span; - var exception = context.Exception; - var errorMessage = string.IsNullOrWhiteSpace(exception?.Message) ? "Unknown error" : exception.Message; - span.SetTag("error", true); - span.SetTag("error.message", errorMessage); - - if (exception != null) - { - span.SetTag("error.exceptionType", exception.GetType().FullName); - - if (!string.IsNullOrEmpty(exception.StackTrace)) - span.SetTag("error.stackTrace", exception.StackTrace); - } + if (context.Exception is null) + return; + + context.Span.AddException(context.Exception); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs index 14aa2860a..301b2b78d 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs @@ -8,12 +8,16 @@ public class FaultExceptionErrorSpanHandler : ErrorSpanHandlerBase { public override void Handle(ErrorSpanContext context) { - if(context.Exception is not FaultException faultException) + if (context.Exception is not FaultException faultException) return; - + var span = context.Span; - span.SetTag("error.code", faultException.Code); - span.SetTag("error.category", faultException.Category); - span.SetTag("error.faultType", faultException.Type); + var tags = new Dictionary + { + ["exception.code"] = faultException.Code, + ["exception.category"] = faultException.Category, + ["exception.type"] = faultException.Type + }; + span.AddException(context.Exception, new(tags.ToArray())); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 5410d70bf..60a2acd46 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -19,7 +19,7 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD public async ValueTask InvokeAsync(ActivityExecutionContext context) { var activity = context.Activity; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity("ActivityExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute activity {activity.Type}", ActivityKind.Internal, Activity.Current?.Context ?? default); if (span == null) { @@ -27,46 +27,51 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD return; } - span.SetTag("activity.nodeId", activity.NodeId); + span.SetTag("operation.name", "elsa.activity.execution"); + span.SetTag("activity.id", activity.NodeId); + span.SetTag("activity.node.id", activity.NodeId); span.SetTag("activity.type", activity.Type); span.SetTag("activity.name", activity.Name); - span.SetTag("activityInstance.id", context.Id); - span.SetTag("activityExecution.startTimeUtc", span.StartTimeUtc); - span.SetTag("tenantId", context.WorkflowExecutionContext.Workflow.Identity.TenantId); - - span.AddEvent(new("Executing", tags: CreateStatusTags(context))); - + span.SetTag("activity.version", activity.Version); + span.SetTag("activity.instance.id", context.Id); + span.SetTag("activity.tenant.id", context.WorkflowExecutionContext.Workflow.Identity.TenantId); + + span.AddEvent(new("executing")); + await next(context); if (context.Status == ActivityStatus.Faulted) { - span.AddEvent(new("Faulted", tags: CreateStatusTags(context))); + span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("activityInstance.hasIncidents", true); + span.SetTag("activity.incidents", true); var errorSpanHandlers = context.GetServices(); var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception); - - foreach (var handler in errorSpanHandlers) + + foreach (var handler in errorSpanHandlers) handler.Handle(errorSpanHandlerContext); } - else + else if (context.Status == ActivityStatus.Canceled) { - span.AddEvent(new("Executed", tags: CreateStatusTags(context))); + span.AddEvent(new("canceled")); span.SetStatus(ActivityStatusCode.Ok); } - - var now = systemClock.UtcNow; - span.SetTag("activityExecution.endTimeUtc", now); - span.SetTag("activityExecution.durationMs", (now - span.StartTimeUtc).TotalMilliseconds); - } - - private ActivityTagsCollection CreateStatusTags(ActivityExecutionContext context) - { - return new(new Dictionary + else if (context.Status == ActivityStatus.Running) { - ["activityInstance.status"] = context.Status.ToString() - }); + span.AddEvent(new("running")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.Status == ActivityStatus.Completed) + { + span.AddEvent(new("completed")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.Status == ActivityStatus.Pending) + { + span.AddEvent(new("pending")); + span.SetStatus(ActivityStatusCode.Ok); + } } } diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index c0e88ef64..7c61eac04 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -1,10 +1,13 @@ using System.Diagnostics; +using System.Runtime.InteropServices.Marshalling; using System.Text.Json; using Elsa.Common; using Elsa.Expressions.Services; using Elsa.Extensions; using Elsa.OpenTelemetry.Helpers; using Elsa.Workflows; +using Elsa.Workflows.Activities; +using Elsa.Workflows.Models; using Elsa.Workflows.Pipelines.WorkflowExecution; using Elsa.Workflows.Serialization.Converters; using JetBrains.Annotations; @@ -26,7 +29,8 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD { var workflowInstanceId = context.Id; var workflow = context.Workflow; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity("WorkflowExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute workflow {workflow.WorkflowMetadata.Name}", ActivityKind.Server, Activity.Current?.Context ?? default); + var now = systemClock.UtcNow; if (span == null) // No listener is registered. { @@ -34,61 +38,79 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD return; } - span.SetTag("workflowInstance.id", workflowInstanceId); - span.SetTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId); - span.SetTag("workflowDefinition.version", workflow.Identity.Version); - span.SetTag("workflowDefinition.name", workflow.WorkflowMetadata.Name); - span.SetTag("workflowExecution.startTimeUtc", span.StartTimeUtc); - span.SetTag("tenantId", workflow.Identity.TenantId); - - if(context.TriggerActivityId != null) + span.SetTag("operation.name", "elsa.workflow.execution"); + span.SetTag("workflow.definition.id", workflow.Identity.DefinitionId); + span.SetTag("workflow.definition.version", workflow.Identity.Version); + span.SetTag("workflow.definition.name", workflow.WorkflowMetadata.Name); + span.SetTag("workflow.definition.tenant.id", workflow.Identity.TenantId); + span.SetTag("workflow.instance.id", workflowInstanceId); + + if (context.TriggerActivityId != null) { - var activity = context.FindActivityById(context.TriggerActivityId) ?? throw new Exception($"Trigger activity with ID {context.TriggerActivityId} not found. This should not happen."); - span.SetTag("workflowExecution.trigger.activityId", activity.Id); - span.SetTag("workflowExecution.trigger.activityName", activity.Name); - span.SetTag("workflowExecution.trigger.activityType", activity.Type); + var activity = context.FindActivityById(context.TriggerActivityId) ?? throw new($"Trigger activity with ID {context.TriggerActivityId} not found. This should not happen."); + span.SetTag("workflow.trigger.activity.id", activity.Id); + span.SetTag("workflow.trigger.activity.name", activity.Name); + span.SetTag("workflow.trigger.activity.type", activity.Type); + span.SetTag("workflow.trigger.activity.version", activity.Version); } - - span.AddEvent(new ActivityEvent("Executing", tags: CreateStatusTags(context))); + + span.AddEvent(new("executing")); await Next(context); if (context.SubStatus == WorkflowSubStatus.Faulted) { - span.AddEvent(new ActivityEvent("Faulted", tags: CreateStatusTags(context))); + span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("error", true); } - else + else if (context.SubStatus == WorkflowSubStatus.Finished) { - span.AddEvent(new ActivityEvent("Executed", tags: CreateStatusTags(context))); + span.AddEvent(new("finished")); span.SetStatus(ActivityStatusCode.Ok); } - - if(context.Incidents.Any()) + else if (context.SubStatus == WorkflowSubStatus.Cancelled) { - span.SetStatus(ActivityStatusCode.Error); - span.SetTag("workflowInstance.hasIncidents", true); - span.SetTag("error", true); - - if (context.Incidents.Count > 0) - span.SetTag("error.message", JsonSerializer.Serialize(context.Incidents, _incidentSerializerOptions)); + span.AddEvent(new("canceled")); + span.SetStatus(ActivityStatusCode.Ok); } - - if (!string.IsNullOrWhiteSpace(context.CorrelationId)) - span.SetTag("workflowInstance.correlationId", context.CorrelationId); - - var now = systemClock.UtcNow; - span.SetTag("workflowExecution.endTimeUtc", now); - span.SetTag("workflowExecution.durationMs", (now - span.StartTimeUtc).TotalMilliseconds); - } - - private ActivityTagsCollection CreateStatusTags(WorkflowExecutionContext context) - { - return new ActivityTagsCollection(new Dictionary + else if (context.SubStatus == WorkflowSubStatus.Suspended) { - ["workflowInstance.status"] = context.Status.ToString(), - ["workflowInstance.subStatus"] = context.SubStatus.ToString() + span.AddEvent(new("suspended")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.SubStatus == WorkflowSubStatus.Pending) + { + span.AddEvent(new("pending")); + span.SetStatus(ActivityStatusCode.Ok); + } + + if (context.Incidents.Any()) + { + span.SetTag("workflow.incidents", true); + span.SetTag("workflow.incidents.count", context.Incidents.Count); + + foreach (var incident in context.Incidents) + span.AddEvent(new("incident", incident.Timestamp, CreateIncidentTags(incident))); + } + + if (!string.IsNullOrWhiteSpace(context.CorrelationId)) + span.SetTag("workflow.correlation_id", context.CorrelationId); + } + + private ActivityTagsCollection CreateIncidentTags(ActivityIncident incident) + { + var tags = new ActivityTagsCollection(new Dictionary + { + ["incident.message"] = incident.Message, }); + + if (incident.Exception != null) + { + tags["incident.exception.message"] = incident.Exception.Message; + tags["incident.exception.stackTrace"] = incident.Exception.StackTrace; + tags["incident.exception.type"] = incident.Exception.GetType().FullName; + } + + return tags; } }