diff --git a/Directory.Packages.props b/Directory.Packages.props index 626f995bf..8ef10f029 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,206 +1,254 @@ - - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Elsa.sln b/Elsa.sln index f78fa138b..f5901cf8a 100644 --- a/Elsa.sln +++ b/Elsa.sln @@ -87,6 +87,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{986E54 docker\ElsaStudio.Dockerfile = docker\ElsaStudio.Dockerfile docker\otel-collector-config.yaml = docker\otel-collector-config.yaml docker\init-db-postgres.sh = docker\init-db-postgres.sh + docker\docker-compose-datadog+otel-collector.yml = docker\docker-compose-datadog+otel-collector.yml EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Elasticsearch", "src\modules\Elsa.Elasticsearch\Elsa.Elasticsearch.csproj", "{3246883E-2FA7-4B4A-BDC5-99039A2869BC}" diff --git a/docker/ElsaServer-Datadog.Dockerfile b/docker/ElsaServer-Datadog.Dockerfile index a74069cf2..7d5022c63 100644 --- a/docker/ElsaServer-Datadog.Dockerfile +++ b/docker/ElsaServer-Datadog.Dockerfile @@ -10,7 +10,7 @@ COPY ./NuGet.Config ./ COPY *.props ./ # Restore packages. -RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj" +RUN dotnet restore "./src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj" # Build and publish (UseAppHost=false creates platform independent binaries). WORKDIR /source/src/bundles/Elsa.Server.Web diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml index e90991c45..54b2098e5 100644 --- a/docker/docker-compose-datadog.yml +++ b/docker/docker-compose-datadog.yml @@ -1,115 +1,77 @@ services: - postgres: - image: postgres:latest - command: -c 'max_connections=2000' - environment: - POSTGRES_USER: elsa - POSTGRES_PASSWORD: elsa - POSTGRES_DB: elsa - volumes: - - postgres-data:/var/lib/postgresql/data - ports: - - "5432:5432" - - rabbitmq: - image: "rabbitmq:3-management" - ports: - - "15672:15672" - - "5672:5672" - - redis: - image: redis:latest - ports: - - "127.0.0.1:6379:6379" - - elsa-server: - pull_policy: always - build: - context: ../. - dockerfile: ./docker/ElsaServer-Datadog.Dockerfile - depends_on: - - postgres - - rabbitmq - - redis - - otel-collector - environment: - DD_AGENT_HOST: datadog-agent - DD_ENV: development - DD_TRACE_DEBUG: true - DD_TRACE_OTEL_ENABLED: true - DD_SERVICE: "Elsa Server" - DD_VERSION: "3.3.0" - # OpenTelemetry environment variables - OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" # Point to OpenTelemetry Collector - OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" # Use gRPC for OTLP - OTEL_TRACES_EXPORTER: "otlp" - OTEL_METRICS_EXPORTER: "otlp" - OTEL_LOGS_EXPORTER: "otpl" - OTEL_RESOURCE_ATTRIBUTES: "service.name=elsa-server-local,service.version=3.2.1-blueberry,deployment.environment=development" - OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES: "Elsa.Workflows" - OTEL_DOTNET_AUTO_INSTRUMENTATION_ENABLED: "true" - OTEL_LOG_LEVEL: "debug" - OTEL_DOTNET_AUTO_RESOURCE_DETECTOR_ENABLED: "true" - OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED: "true" - OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED: "true" - OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED: "true" - - ASPNETCORE_ENVIRONMENT: Development - PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 - PYTHONNET_RUNTIME: coreclr - CONNECTIONSTRINGS__POSTGRESQL: "Server=postgres;Username=elsa;Database=elsa;Port=5432;Password=elsa;SSLMode=Prefer" - CONNECTIONSTRINGS__RABBITMQ: "amqp://guest:guest@rabbitmq:5672/" - CONNECTIONSTRINGS__REDIS: "redis:6379" - DISTRIBUTEDLOCKPROVIDER: "Postgres" - ports: - - "13000:8080" - elsa-studio: pull_policy: always build: context: ../. dockerfile: ./docker/ElsaStudio.Dockerfile + depends_on: + - elsa-server environment: ASPNETCORE_ENVIRONMENT: Development ELSASERVER__URL: "http://localhost:13000/elsa/api" ports: - "14000:8080" - - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - command: [ "--config", "/etc/otel-collector-config.yaml", "--feature-gates", "-component.UseLocalHostAsDefaultHost" ] + + elsa-server: + pull_policy: always + build: + context: ../. + dockerfile: ./docker/ElsaServer.Dockerfile + depends_on: + - datadog-agent environment: - DD_API_KEY: "secret api key" - DD_SITE: "datadoghq.eu" + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: false + OTEL_RESOURCE_ATTRIBUTES: service.name=aspnet-app + OTEL_EXPORTER_OTLP_ENDPOINT: http://datadog-agent:4317 + OTEL_EXPORTER_OTLP_HEADERS: "DD_API_KEY=4cdbf04b80065d715d9d12ee1fd84e22" + DD_AGENT_HOST: datadog-agent + DD_ENV: development + DD_TRACE_DEBUG: true + DD_TRACE_ENABLED: true + DD_TRACE_OTEL_ENABLED: true + DD_TRACE_LOGGING_ENABLED: true + DD_SERVICE: "Elsa Server" + DD_VERSION: "3.4.0" + ASPNETCORE_ENVIRONMENT: Development + PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 + PYTHONNET_RUNTIME: coreclr ports: - - "13133:13133" - - "4317:4317" - - "4318:4318" + - "13000:8080" datadog-agent: - image: datadog/agent:latest + image: gcr.io/datadoghq/agent:7 environment: - DD_API_KEY: "" + DD_API_KEY: "b430503d709680c951cfe3979ae35068" DD_SITE: "datadoghq.eu" - DD_HOSTNAME: "otel-collector" + DD_HOSTNAME: "datadog-agent" DD_LOGS_ENABLED: "true" DD_OTLP_CONFIG_LOGS_ENABLED: "true" + DD_OTLP_CONFIG_TRACES_ENABLED: "true" + DD_OTLP_CONFIG_METRICS_ENABLED: "true" DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true" DD_APM_ENABLED: "true" DD_APM_NON_LOCAL_TRAFFIC: "true" - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 # The Datadog Agent expects traces from OpenTelemetry Collector + DD_DOGSTATSD_NON_LOCAL_TRAFFIC: "true" + DD_APM_RECEIVER_SOCKET: /var/run/datadog/apm.socket + DD_DOGSTATSD_SOCKET: /var/run/datadog/dsd.socket + DD_APM_INSTRUMENTATION_ENABLED: true + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: 0.0.0.0:4317 DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: 0.0.0.0:4318 - + # Service autodiscovery - DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server" + DD_AC_INCLUDE: "name:elsa-workflows" DD_AC_EXCLUDE: "name:datadog-agent" - + ports: - - "8126:8126" - - "14317:4317" - - "14318:4318" + - "8126:8126" # APM + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + + volumes: + - /var/run/datadog:/var/run/datadog + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc/:/host/proc/:ro + - /sys/fs/cgroup:/host/sys/fs/cgroup:ro volumes: postgres-data: \ No newline at end of file diff --git a/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj b/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj index e48fa21f3..4e4903a35 100644 --- a/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj +++ b/src/apps/Elsa.Server.Web/Elsa.Server.Web.csproj @@ -59,6 +59,7 @@ + @@ -68,7 +69,13 @@ + + + + + + diff --git a/src/apps/Elsa.Server.Web/Program.cs b/src/apps/Elsa.Server.Web/Program.cs index 4035178eb..29eb7d12a 100644 --- a/src/apps/Elsa.Server.Web/Program.cs +++ b/src/apps/Elsa.Server.Web/Program.cs @@ -68,6 +68,10 @@ using Medallion.Threading.Redis; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Options; using OpenTelemetry; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Proto.Cluster.Kubernetes; using Proto.Persistence.Sqlite; @@ -96,7 +100,7 @@ const bool useTenantsFromConfiguration = true; const bool useSecrets = false; const bool disableVariableWrappers = false; const bool disableVariableCopying = false; -const bool useOtel = false; +const bool useManualOtelInstrumentation = true; var builder = WebApplication.CreateBuilder(args); var services = builder.Services; @@ -123,14 +127,46 @@ var sqlDatabaseProvider = Enum.Parse(configuration["Databas TypeAliasRegistry.RegisterAlias("OrderReceivedProducerFactory", typeof(GenericProducerFactory)); TypeAliasRegistry.RegisterAlias("OrderReceivedConsumerFactory", typeof(GenericConsumerFactory)); -if (useOtel) +if (useManualOtelInstrumentation) { -// Configure OpenTelemetry Tracing - using var tracerProvider = Sdk.CreateTracerProviderBuilder() - .AddSource("Elsa.Workflows") // Match your ActivitySource name here - .SetSampler(new AlwaysOnSampler()) // Always record traces for testing - .AddConsoleExporter() // Export spans to the console (optional) - .Build(); + // Configure OpenTelemetry Tracing + // using var tracerProvider = Sdk.CreateTracerProviderBuilder() + // .AddSource("Elsa.Workflows") // Match your ActivitySource name here + // .SetSampler(new AlwaysOnSampler()) // Always record traces for testing + // .AddConsoleExporter() // Export spans to the console (optional) + // .Build(); + + services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService("elsa-workflows", serviceVersion: "3.4.0").AddTelemetrySdk()) + .WithTracing(tracing => + { + tracing + .AddSource("*") + .SetSampler(new AlwaysOnSampler()) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSqlClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + ; + }) + .WithMetrics(metrics => + { + metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + ; + }); + + // Enable OpenTelemetry Logging (optional) + builder.Logging.AddOpenTelemetry(options => + { + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + options.ParseStateValues = true; + }); } // Add Elsa services. @@ -221,7 +257,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -269,7 +307,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -322,7 +362,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -474,7 +516,9 @@ services ef.UsePostgreSql(cockroachDbConnectionString); else if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + { + SchemaName = "ELSA" + }); else ef.UseSqlite(sp => sp.GetSqliteConnectionString()); @@ -577,7 +621,7 @@ services } }); } - + if (useKafka) { elsa.UseKafka(kafka => @@ -585,7 +629,7 @@ services kafka.ConfigureOptions(options => configuration.GetSection("Kafka").Bind(options)); }); } - + if (useSecrets) { elsa @@ -661,8 +705,11 @@ services if (sqlDatabaseProvider == SqlDatabaseProvider.PostgreSql) ef.UsePostgreSql(postgresConnectionString); if (sqlDatabaseProvider == SqlDatabaseProvider.Citus) ef.UsePostgreSql(citusConnectionString); if (sqlDatabaseProvider == SqlDatabaseProvider.YugabyteDb) ef.UsePostgreSql(yugabyteDbConnectionString); - if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) ef.UseOracle(oracleConnectionString, new() - { SchemaName = "ELSA"}); + if (sqlDatabaseProvider == SqlDatabaseProvider.Oracle) + ef.UseOracle(oracleConnectionString, new() + { + SchemaName = "ELSA" + }); #if !NET9_0 if (sqlDatabaseProvider == SqlDatabaseProvider.MySql) ef.UseMySql(mySqlConnectionString); @@ -682,7 +729,7 @@ services tenantHttpRouting.WithTenantHeader("X-Tenant-ID"); }); } - + elsa.UseWebhooks(webhooks => webhooks.ConfigureSinks += options => builder.Configuration.GetSection("Webhooks").Bind(options)); elsa.InstallDropIns(options => options.DropInRootDirectory = Path.Combine(Directory.GetCurrentDirectory(), "App_Data", "DropIns")); elsa.AddSwagger(); diff --git a/src/apps/Elsa.Server.Web/appsettings.json b/src/apps/Elsa.Server.Web/appsettings.json index 3fc86acdb..91735a81b 100644 --- a/src/apps/Elsa.Server.Web/appsettings.json +++ b/src/apps/Elsa.Server.Web/appsettings.json @@ -2,6 +2,7 @@ "Logging": { "LogLevel": { "Default": "Warning", + "OpenTelemetry": "Debug", "Microsoft.Hosting.Lifetime": "Information", "Elsa": "Information", "Elsa.Workflows.Runtime.Middleware.Workflows.WorkflowHeartbeatMiddleware": "Debug" diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs index 04bebef62..a3003ba35 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/DefaultErrorSpanHandler.cs @@ -7,18 +7,9 @@ public class DefaultErrorSpanHandler : ErrorSpanHandlerBase { public override void Handle(ErrorSpanContext context) { - var span = context.Span; - var exception = context.Exception; - var errorMessage = string.IsNullOrWhiteSpace(exception?.Message) ? "Unknown error" : exception.Message; - span.SetTag("error", true); - span.SetTag("error.message", errorMessage); - - if (exception != null) - { - span.SetTag("error.exceptionType", exception.GetType().FullName); - - if (!string.IsNullOrEmpty(exception.StackTrace)) - span.SetTag("error.stackTrace", exception.StackTrace); - } + if (context.Exception is null) + return; + + context.Span.AddException(context.Exception); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs index 14aa2860a..301b2b78d 100644 --- a/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs +++ b/src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs @@ -8,12 +8,16 @@ public class FaultExceptionErrorSpanHandler : ErrorSpanHandlerBase { public override void Handle(ErrorSpanContext context) { - if(context.Exception is not FaultException faultException) + if (context.Exception is not FaultException faultException) return; - + var span = context.Span; - span.SetTag("error.code", faultException.Code); - span.SetTag("error.category", faultException.Category); - span.SetTag("error.faultType", faultException.Type); + var tags = new Dictionary + { + ["exception.code"] = faultException.Code, + ["exception.category"] = faultException.Category, + ["exception.type"] = faultException.Type + }; + span.AddException(context.Exception, new(tags.ToArray())); } } \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index 5410d70bf..60a2acd46 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -19,7 +19,7 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD public async ValueTask InvokeAsync(ActivityExecutionContext context) { var activity = context.Activity; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity("ActivityExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute activity {activity.Type}", ActivityKind.Internal, Activity.Current?.Context ?? default); if (span == null) { @@ -27,46 +27,51 @@ public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareD return; } - span.SetTag("activity.nodeId", activity.NodeId); + span.SetTag("operation.name", "elsa.activity.execution"); + span.SetTag("activity.id", activity.NodeId); + span.SetTag("activity.node.id", activity.NodeId); span.SetTag("activity.type", activity.Type); span.SetTag("activity.name", activity.Name); - span.SetTag("activityInstance.id", context.Id); - span.SetTag("activityExecution.startTimeUtc", span.StartTimeUtc); - span.SetTag("tenantId", context.WorkflowExecutionContext.Workflow.Identity.TenantId); - - span.AddEvent(new("Executing", tags: CreateStatusTags(context))); - + span.SetTag("activity.version", activity.Version); + span.SetTag("activity.instance.id", context.Id); + span.SetTag("activity.tenant.id", context.WorkflowExecutionContext.Workflow.Identity.TenantId); + + span.AddEvent(new("executing")); + await next(context); if (context.Status == ActivityStatus.Faulted) { - span.AddEvent(new("Faulted", tags: CreateStatusTags(context))); + span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("activityInstance.hasIncidents", true); + span.SetTag("activity.incidents", true); var errorSpanHandlers = context.GetServices(); var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception); - - foreach (var handler in errorSpanHandlers) + + foreach (var handler in errorSpanHandlers) handler.Handle(errorSpanHandlerContext); } - else + else if (context.Status == ActivityStatus.Canceled) { - span.AddEvent(new("Executed", tags: CreateStatusTags(context))); + span.AddEvent(new("canceled")); span.SetStatus(ActivityStatusCode.Ok); } - - var now = systemClock.UtcNow; - span.SetTag("activityExecution.endTimeUtc", now); - span.SetTag("activityExecution.durationMs", (now - span.StartTimeUtc).TotalMilliseconds); - } - - private ActivityTagsCollection CreateStatusTags(ActivityExecutionContext context) - { - return new(new Dictionary + else if (context.Status == ActivityStatus.Running) { - ["activityInstance.status"] = context.Status.ToString() - }); + span.AddEvent(new("running")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.Status == ActivityStatus.Completed) + { + span.AddEvent(new("completed")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.Status == ActivityStatus.Pending) + { + span.AddEvent(new("pending")); + span.SetStatus(ActivityStatusCode.Ok); + } } } diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index c0e88ef64..7c61eac04 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -1,10 +1,13 @@ using System.Diagnostics; +using System.Runtime.InteropServices.Marshalling; using System.Text.Json; using Elsa.Common; using Elsa.Expressions.Services; using Elsa.Extensions; using Elsa.OpenTelemetry.Helpers; using Elsa.Workflows; +using Elsa.Workflows.Activities; +using Elsa.Workflows.Models; using Elsa.Workflows.Pipelines.WorkflowExecution; using Elsa.Workflows.Serialization.Converters; using JetBrains.Annotations; @@ -26,7 +29,8 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD { var workflowInstanceId = context.Id; var workflow = context.Workflow; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity("WorkflowExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"execute workflow {workflow.WorkflowMetadata.Name}", ActivityKind.Server, Activity.Current?.Context ?? default); + var now = systemClock.UtcNow; if (span == null) // No listener is registered. { @@ -34,61 +38,79 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD return; } - span.SetTag("workflowInstance.id", workflowInstanceId); - span.SetTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId); - span.SetTag("workflowDefinition.version", workflow.Identity.Version); - span.SetTag("workflowDefinition.name", workflow.WorkflowMetadata.Name); - span.SetTag("workflowExecution.startTimeUtc", span.StartTimeUtc); - span.SetTag("tenantId", workflow.Identity.TenantId); - - if(context.TriggerActivityId != null) + span.SetTag("operation.name", "elsa.workflow.execution"); + span.SetTag("workflow.definition.id", workflow.Identity.DefinitionId); + span.SetTag("workflow.definition.version", workflow.Identity.Version); + span.SetTag("workflow.definition.name", workflow.WorkflowMetadata.Name); + span.SetTag("workflow.definition.tenant.id", workflow.Identity.TenantId); + span.SetTag("workflow.instance.id", workflowInstanceId); + + if (context.TriggerActivityId != null) { - var activity = context.FindActivityById(context.TriggerActivityId) ?? throw new Exception($"Trigger activity with ID {context.TriggerActivityId} not found. This should not happen."); - span.SetTag("workflowExecution.trigger.activityId", activity.Id); - span.SetTag("workflowExecution.trigger.activityName", activity.Name); - span.SetTag("workflowExecution.trigger.activityType", activity.Type); + var activity = context.FindActivityById(context.TriggerActivityId) ?? throw new($"Trigger activity with ID {context.TriggerActivityId} not found. This should not happen."); + span.SetTag("workflow.trigger.activity.id", activity.Id); + span.SetTag("workflow.trigger.activity.name", activity.Name); + span.SetTag("workflow.trigger.activity.type", activity.Type); + span.SetTag("workflow.trigger.activity.version", activity.Version); } - - span.AddEvent(new ActivityEvent("Executing", tags: CreateStatusTags(context))); + + span.AddEvent(new("executing")); await Next(context); if (context.SubStatus == WorkflowSubStatus.Faulted) { - span.AddEvent(new ActivityEvent("Faulted", tags: CreateStatusTags(context))); + span.AddEvent(new("faulted")); span.SetStatus(ActivityStatusCode.Error); - span.SetTag("error", true); } - else + else if (context.SubStatus == WorkflowSubStatus.Finished) { - span.AddEvent(new ActivityEvent("Executed", tags: CreateStatusTags(context))); + span.AddEvent(new("finished")); span.SetStatus(ActivityStatusCode.Ok); } - - if(context.Incidents.Any()) + else if (context.SubStatus == WorkflowSubStatus.Cancelled) { - span.SetStatus(ActivityStatusCode.Error); - span.SetTag("workflowInstance.hasIncidents", true); - span.SetTag("error", true); - - if (context.Incidents.Count > 0) - span.SetTag("error.message", JsonSerializer.Serialize(context.Incidents, _incidentSerializerOptions)); + span.AddEvent(new("canceled")); + span.SetStatus(ActivityStatusCode.Ok); } - - if (!string.IsNullOrWhiteSpace(context.CorrelationId)) - span.SetTag("workflowInstance.correlationId", context.CorrelationId); - - var now = systemClock.UtcNow; - span.SetTag("workflowExecution.endTimeUtc", now); - span.SetTag("workflowExecution.durationMs", (now - span.StartTimeUtc).TotalMilliseconds); - } - - private ActivityTagsCollection CreateStatusTags(WorkflowExecutionContext context) - { - return new ActivityTagsCollection(new Dictionary + else if (context.SubStatus == WorkflowSubStatus.Suspended) { - ["workflowInstance.status"] = context.Status.ToString(), - ["workflowInstance.subStatus"] = context.SubStatus.ToString() + span.AddEvent(new("suspended")); + span.SetStatus(ActivityStatusCode.Ok); + } + else if (context.SubStatus == WorkflowSubStatus.Pending) + { + span.AddEvent(new("pending")); + span.SetStatus(ActivityStatusCode.Ok); + } + + if (context.Incidents.Any()) + { + span.SetTag("workflow.incidents", true); + span.SetTag("workflow.incidents.count", context.Incidents.Count); + + foreach (var incident in context.Incidents) + span.AddEvent(new("incident", incident.Timestamp, CreateIncidentTags(incident))); + } + + if (!string.IsNullOrWhiteSpace(context.CorrelationId)) + span.SetTag("workflow.correlation_id", context.CorrelationId); + } + + private ActivityTagsCollection CreateIncidentTags(ActivityIncident incident) + { + var tags = new ActivityTagsCollection(new Dictionary + { + ["incident.message"] = incident.Message, }); + + if (incident.Exception != null) + { + tags["incident.exception.message"] = incident.Exception.Message; + tags["incident.exception.stackTrace"] = incident.Exception.StackTrace; + tags["incident.exception.type"] = incident.Exception.GetType().FullName; + } + + return tags; } }