Update Docker and k8s configs, enhance OTEL tracing

Updated `docker-compose-datadog.yml` with new environment variables and updated image versions, enhancing trace sampling and service autodiscovery. Added Kubernetes deployment, service, and role files for `elsa-server`, `elsa-studio`, `plant-uml`, `postgres`, and `trace-lens`, improving the project's infrastructure. Enhanced OpenTelemetry middleware for better error handling and detailed tracing of activity and workflow executions.
2024-09-23 23:35:30 +02:00 · 2024-09-23 23:35:30 +02:00 · adf36927dc
parent 79b5bfc378
commit adf36927dc
18 changed files with 478 additions and 35 deletions
--- a/Elsa.sln
+++ b/Elsa.sln
@ -357,6 +357,43 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.MongoDb.UnitTests", "t
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.OpenTelemetry", "src\modules\Elsa.OpenTelemetry\Elsa.OpenTelemetry.csproj", "{25880971-403E-4872-93A1-D33089E07C91}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{FE1AFEC0-7C63-4EF8-8E24-D9703590A778}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "k8s", "k8s", "{924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "elsa-server", "elsa-server", "{A2743EC6-7117-46E2-8AC1-8CDD52C8AAEF}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\k8s\elsa-server\deployment.yaml = scripts\k8s\elsa-server\deployment.yaml
+		scripts\k8s\elsa-server\role-binding.yaml = scripts\k8s\elsa-server\role-binding.yaml
+		scripts\k8s\elsa-server\role.yaml = scripts\k8s\elsa-server\role.yaml
+		scripts\k8s\elsa-server\service-account.yml = scripts\k8s\elsa-server\service-account.yml
+		scripts\k8s\elsa-server\service.yaml = scripts\k8s\elsa-server\service.yaml
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "elsa-studio", "elsa-studio", "{A782913B-142F-4EA8-A2C7-82679CD2ABC1}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\k8s\elsa-studio\deployment.yaml = scripts\k8s\elsa-studio\deployment.yaml
+		scripts\k8s\elsa-studio\service.yaml = scripts\k8s\elsa-studio\service.yaml
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "plant-uml", "plant-uml", "{BFB87B50-3FE9-4AB1-8F18-96BE667C953D}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\k8s\plant-uml\deployment.yaml = scripts\k8s\plant-uml\deployment.yaml
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "postgres", "postgres", "{399C61BC-A51B-4B57-AA32-46BC24D4AE1C}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\k8s\postgres\config-map.yaml = scripts\k8s\postgres\config-map.yaml
+		scripts\k8s\postgres\deployment.yaml = scripts\k8s\postgres\deployment.yaml
+		scripts\k8s\postgres\service.yml = scripts\k8s\postgres\service.yml
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "trace-lens", "trace-lens", "{2FDF67FD-2BD3-4F73-9A24-DA255641ED25}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\k8s\trace-lens\deployment.yaml = scripts\k8s\trace-lens\deployment.yaml
+		scripts\k8s\trace-lens\service.yaml = scripts\k8s\trace-lens\service.yaml
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@ -998,6 +1035,12 @@ Global
 		{90CD37A9-C866-4D90-A3B1-8C87F53B845E} = {CBB515F3-A0EF-43B5-A907-FD4E652DD66E}
 		{56CAA9F2-1882-4EFA-BAC0-9C3D804553F1} = {18453B51-25EB-4317-A4B3-B10518252E92}
 		{25880971-403E-4872-93A1-D33089E07C91} = {5BA4A8FA-F7F4-45B3-AEC8-8886D35AAC79}
+		{924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} = {FE1AFEC0-7C63-4EF8-8E24-D9703590A778}
+		{A2743EC6-7117-46E2-8AC1-8CDD52C8AAEF} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}
+		{A782913B-142F-4EA8-A2C7-82679CD2ABC1} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}
+		{BFB87B50-3FE9-4AB1-8F18-96BE667C953D} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}
+		{399C61BC-A51B-4B57-AA32-46BC24D4AE1C} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}
+		{2FDF67FD-2BD3-4F73-9A24-DA255641ED25} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D4B5CEAA-7D70-4FCB-A68E-B03FBE5E0E5E}
--- a/docker/ElsaServer-Datadog.Dockerfile
+++ b/docker/ElsaServer-Datadog.Dockerfile
@ -18,7 +18,7 @@ COPY *.props ./
 RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj"

 # build and publish (UseAppHost=false creates platform independent binaries).
-WORKDIR /source/src/bundles/Elsa.Server.Web
+WORKDIR /source/src/apps/Elsa.Server.Web
 RUN dotnet build "Elsa.Server.Web.csproj" -c Release -o /app/build
 RUN dotnet publish "Elsa.Server.Web.csproj" -c Release -o /app/publish /p:UseAppHost=false --no-restore -f net8.0

@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    rm -rf /var/lib/apt/lists/*

 # Set PYTHONNET_PYDLL environment variable
-ENV PYTHONNET_PYDLL /usr/lib/aarch64-linux-gnu/libpython3.11.so
+ENV PYTHONNET_PYDLL=/usr/lib/aarch64-linux-gnu/libpython3.11.so

 # Copy the tracer from build target
 COPY --from=build /tmp/datadog-dotnet-apm.deb /tmp/datadog-dotnet-apm.deb
--- a/docker/docker-compose-datadog.yml
+++ b/docker/docker-compose-datadog.yml
@ -12,7 +12,7 @@ services:
      - postgres-data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
-        
+  
  cockroachdb:
    image: cockroachdb/cockroach:v22.1.0
    command: start-single-node --insecure
@ -23,19 +23,20 @@ services:
      - cockroachdb-data:/cockroach/cockroach-data
    environment:
      - COCKROACH_DATABASE=elsa
-    
+  
  rabbitmq:
    image: "rabbitmq:3-management"
    ports:
      - "15672:15672"
      - "5672:5672"
-        
+  
  redis:
    image: redis:latest
    ports:
      - "127.0.0.1:6379:6379"

  elsa-server:
+    pull_policy: always
    build:
      context: ../.
      dockerfile: ./docker/ElsaServer-Datadog.Dockerfile
@ -46,6 +47,11 @@ services:
      - datadog-agent
    environment:
      DD_AGENT_HOST: datadog-agent
+      DD_ENV: development
+      DD_TRACE_DEBUG: true
+      DD_TRACE_OTEL_ENABLED: true
+      DD_SERVICE: "elsa-server-local"
+      DD_VERSION: "3.2.1-blueberry"
      ASPNETCORE_ENVIRONMENT: Development
      PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11
      PYTHONNET_RUNTIME: coreclr
@ -55,11 +61,12 @@ services:
      DISTRIBUTEDLOCKPROVIDER: "Postgres"
    ports:
      - "13000:8080"
-        
+  
  elsa-studio:
+    pull_policy: always
    build:
      context: ../.
-      dockerfile: ./docker/ElsaStudio-Datadog.Dockerfile
+      dockerfile: ./docker/ElsaStudio.Dockerfile
    environment:
      ASPNETCORE_ENVIRONMENT: Development
      ELSASERVER__URL: "http://localhost:13000/elsa/api"
@ -67,16 +74,60 @@ services:
      - "14000:8080"
  
  datadog-agent:
-    image: datadog/agent:7
+    image: datadog/agent:7.42.0
    environment:
      DD_API_KEY: "YOUR_API_KEY"
      DD_SITE: "datadoghq.eu"
      DD_LOGS_ENABLED: "true"
      DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true"
      DD_APM_ENABLED: "true"
+      DD_REMOTE_CONFIGURATION_ENABLED: "true"
      DD_APM_NON_LOCAL_TRAFFIC: "true"
+      
+      # Service autodiscovery
      DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server"
      DD_AC_EXCLUDE: "name:datadog-agent"
+      
+      # Enable priority sampling
+      DD_TRACE_SAMPLING_PRIORITY: "true"
+      
+      # Global rate limiting of traces (number of spans per second)
+      DD_TRACE_RATE_LIMIT: 100
+      
+      # Global sample rate for all traces (applies to spans that do not match a specific sampling rule)
+      DD_TRACE_SAMPLE_RATE: 1.0  # Keep 100% of the traces globally (adjust as needed)
+      
+      # Sampling rules for controlling sampling of specific services and errors
+      DD_TRACE_SAMPLING_RULES: >
+        [
+          {
+            "service": "elsa-server-local",
+            "name": "WorkflowExecution",
+            "sample_rate": 0.5
+          },
+          {
+            "service": "elsa-server-local",
+            "name": "WorkflowExecution",
+            "sample_rate": 1.0,
+            "condition": {"tags": {"hasIncidents": "true"}}
+          }
+          {
+            "service": "elsa-server-local",
+            "name": "ActivityExecution",
+            "sample_rate": 1.0,
+            "condition": {"tags": {"hasIncidents": "true"}}
+          },
+          {
+            "service": "elsa-server-local",
+            "name": "ActivityExecution",
+            "sample_rate": 0.3
+          },
+          {
+            "service": "elsa-server-local",
+            "sample_rate": 1.0,
+            "condition": {"error": true}
+          }
+        ]
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /proc/:/host/proc/:ro
--- a/scripts/k8s/elsa-server/deployment.yaml
+++ b/scripts/k8s/elsa-server/deployment.yaml
@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elsa-server-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elsa-server
+  template:
+    metadata:
+      labels:
+        app: elsa-server
+    spec:
+      serviceAccountName: proto-cluster
+      containers:
+        - name: elsa-server
+          imagePullPolicy: Never
+          image: elsa-server:latest
+          ports:
+            - containerPort: 8080
+          env:
+            - name: ASPNETCORE_ENVIRONMENT
+              value: Development
+            - name: "ProtoActor__AdvertisedHost"
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CONNECTIONSTRINGS__POSTGRESQL
+              value: "Server=postgres;Username=elsa;Database=elsa;Port=5432;Password=elsa;SSLMode=Prefer;MaxPoolSize=2000;Timeout=60"
+            - name: COR_ENABLE_PROFILING
+              value: "1"
+            - name: COR_PROFILER
+              value: "{918728DD-259F-4A6A-AC2B-B85E1B658318}"
+            - name: CORECLR_PROFILER_PATH
+              value: "$INSTALL_DIR/osx-x64/OpenTelemetry.AutoInstrumentation.Native.dylib"
+            - name: DOTNET_ADDITIONAL_DEPS
+              value: "$INSTALL_DIR/AdditionalDeps"
+            - name: DOTNET_EnableDiagnostics
+              value: "1"
+            - name: DOTNET_SHARED_STORE
+              value: "$INSTALL_DIR/store"
+            - name: DOTNET_STARTUP_HOOKS
+              value: "OpenTelemetry.AutoInstrumentation.StartupHook"
+            - name: OTEL_DOTNET_AUTO_HOME
+              value: "$INSTALL_DIR"
+            - name: OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED
+              value: "false"
+            - name: OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED
+              value: "false"
+            - name: OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES
+              value: "Proto.Actor,Elsa.Workflows"
+            - name: OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED
+              value: "false"
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: "http://trace-lens-collector:4317"
+            - name: OTEL_EXPORTER_OTLP_PROTOCOL
+              value: "grpc"
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: "service.name=Elsa Server,service.version=3.3.0"
--- a/scripts/k8s/elsa-server/role-binding.yaml
+++ b/scripts/k8s/elsa-server/role-binding.yaml
@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: proto-cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: proto-cluster
+subjects:
+  - kind: ServiceAccount
+    name: proto-cluster # this is the service account that should have the role applied
--- a/scripts/k8s/elsa-server/role.yaml
+++ b/scripts/k8s/elsa-server/role.yaml
@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: proto-cluster
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
--- a/scripts/k8s/elsa-server/service-account.yml
+++ b/scripts/k8s/elsa-server/service-account.yml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: proto-cluster
+  namespace: default  # Ensure the namespace matches your deployment
--- a/scripts/k8s/elsa-server/service.yaml
+++ b/scripts/k8s/elsa-server/service.yaml
@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elsa-server-service
+spec:
+  type: LoadBalancer
+  sessionAffinity: None
+  ports:
+    - port: 8001
+      targetPort: 8080
+  selector:
+    app: elsa-server
--- a/scripts/k8s/elsa-studio/deployment.yaml
+++ b/scripts/k8s/elsa-studio/deployment.yaml
@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elsa-studio-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elsa-studio
+  template:
+    metadata:
+      labels:
+        app: elsa-studio
+    spec:
+      containers:
+        - name: elsa-studio
+          imagePullPolicy: Never
+          image: elsa-studio:latest
+          ports:
+            - containerPort: 8080
+          env:
+            - name: ASPNETCORE_ENVIRONMENT
+              value: Development
+            - name: "ELSASERVER__URL"
+              value: "http://localhost:8001/elsa/api"
--- a/scripts/k8s/elsa-studio/service.yaml
+++ b/scripts/k8s/elsa-studio/service.yaml
@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elsa-studio-service
+spec:
+  type: LoadBalancer
+  sessionAffinity: None
+  ports:
+    - port: 9001
+      targetPort: 8080
+  selector:
+    app: elsa-studio
--- a/scripts/k8s/plant-uml/deployment.yaml
+++ b/scripts/k8s/plant-uml/deployment.yaml
@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: plant-uml-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: plant-uml
+  template:
+    metadata:
+      labels:
+        app: plant-uml
+    spec:
+      containers:
+        - name: plant-uml
+          imagePullPolicy: Always
+          image: plantuml/plantuml-server:tomcat
+          ports:
+            - containerPort: 8080
--- a/scripts/k8s/postgres/config-map.yaml
+++ b/scripts/k8s/postgres/config-map.yaml
@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: init-db-config
+data:
+  init-db.sh: |
+    #!/bin/bash
+    echo "Starting database initialization"
+  
+    psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    GRANT ALL ON SCHEMA public TO elsa;
+    GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO elsa;
+    GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO elsa;
+    GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO elsa;
+    CREATE USER tracelens WITH PASSWORD 'tracelenspass';
+    CREATE DATABASE tracelens;
+    GRANT ALL PRIVILEGES ON DATABASE tracelens TO tracelens;
+    GRANT ALL ON SCHEMA public TO tracelens;
+    EOSQL
+    
+    echo "Database initialization completed"
--- a/scripts/k8s/postgres/deployment.yaml
+++ b/scripts/k8s/postgres/deployment.yaml
@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          imagePullPolicy: Always
+          image: postgres:latest
+          # command: ["-c", "max_connections=2000"]
+          volumeMounts:
+            - name: postgres-data
+              mountPath: /var/lib/postgresql/data
+            - name: init-db
+              mountPath: /docker-entrypoint-initdb.d/init-db.sh
+              subPath: init-db.sh
+          ports:
+            - containerPort: 5432
+          env:
+            - name: POSTGRES_USER
+              value: "elsa"
+            - name: POSTGRES_PASSWORD
+              value: "elsa"
+            - name: POSTGRES_DB
+              value: "elsa"
+              
+      volumes:
+        - name: postgres-data
+          emptyDir: {}
+        - name: init-db
+          configMap:
+            name: init-db-config
--- a/scripts/k8s/postgres/service.yml
+++ b/scripts/k8s/postgres/service.yml
@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 5432
+      targetPort: 5432
+  selector:
+    app: postgres
--- a/scripts/k8s/trace-lens/deployment.yaml
+++ b/scripts/k8s/trace-lens/deployment.yaml
@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tracelens-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: trace-lens
+  template:
+    metadata:
+      labels:
+        app: trace-lens
+    spec:
+      containers:
+        - name: trace-lens
+          imagePullPolicy: Always
+          image: docker.io/rogeralsing/tracelens:latest
+          ports:
+            - containerPort: 5001
+            - containerPort: 4317
+          env:
+            - name: PlantUml__RemoteUrl
+              value: ""
+            - name: ConnectionStrings__DefaultConnection
+              value: "USER ID=tracelens;PASSWORD=tracelenspass;HOST=postgres;PORT=5432;DATABASE=tracelens;POOLING=true;"
--- a/scripts/k8s/trace-lens/service.yaml
+++ b/scripts/k8s/trace-lens/service.yaml
@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: trace-lens-dashboard
+spec:
+  type: LoadBalancer
+  selector:
+    app: trace-lens
+  ports:
+    - name: dashboard
+      port: 7001
+      targetPort: 5001
+  # OTEL collector port is internal-only, avoid exposing it via LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: trace-lens-collector
+spec:
+  selector:
+    app: trace-lens
+  ports:
+    - name: otel-collector
+      port: 4317
+      targetPort: 4317
--- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs
+++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs
@ -1,10 +1,12 @@
 using System.Diagnostics;
+using Elsa.Common.Contracts;
 using Elsa.OpenTelemetry.Helpers;
 using Elsa.Workflows;
 using Elsa.Workflows.Contracts;
 using Elsa.Workflows.Pipelines.ActivityExecution;
 using Elsa.Workflows.Pipelines.WorkflowExecution;
 using JetBrains.Annotations;
+using Newtonsoft.Json;
 using Activity = System.Diagnostics.Activity;
 using ActivityKind = System.Diagnostics.ActivityKind;

@ -12,22 +14,52 @@ namespace Elsa.OpenTelemetry.Middleware;

 /// <inheritdoc />
 [UsedImplicitly]
-public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next) : IActivityExecutionMiddleware
+public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next, ISystemClock systemClock) : IActivityExecutionMiddleware
 {
    /// <inheritdoc />
    public async ValueTask InvokeAsync(ActivityExecutionContext context)
    {
        var activity = context.Activity;
-        using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"ActivityExecution {context.ActivityDescriptor.TypeName}", ActivityKind.Internal, Activity.Current?.Context ?? default);
-        span?.AddTag("activity.nodeId", activity.NodeId);
-        span?.AddTag("activity.type", activity.Type);
-        span?.AddTag("activity.name", activity.Name);
-        span?.AddTag("activityInstance.id", context.Id);
-        span?.AddTag("activityInstance.originalStatus", context.Status.ToString());
-        span?.AddEvent(new ActivityEvent("Executing"));
+        using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"ActivityExecution", ActivityKind.Internal, Activity.Current?.Context ?? default);
+
+        if (span == null)
+        {
+            await next(context);
+            return;
+        }
+
+        span.SetTag("activity.nodeId", activity.NodeId);
+        span.SetTag("activity.type", activity.Type);
+        span.SetTag("activity.name", activity.Name);
+        span.SetTag("activityInstance.id", context.Id);
+        
+        span.AddEvent(new ActivityEvent("Executing", tags: new ActivityTagsCollection(new Dictionary<string, object?>
+        {
+            ["activityInstance.status"] = context.Status.ToString(),
+        })));
+        
        await next(context);
-        span?.AddEvent(new ActivityEvent("Executed"));
-        span?.AddTag("activityInstance.newStatus", context.Status.ToString());
+
+        if (context.Status == ActivityStatus.Faulted)
+        {
+            span.AddEvent(new ActivityEvent("Faulted"));
+            span.SetStatus(ActivityStatusCode.Error);
+            span.SetTag("error", true);
+            span.SetTag("hasIncidents", true);
+
+            var errorMessage = string.IsNullOrWhiteSpace(context.Exception?.Message) ? "Unknown error" : context.Exception.Message;
+            span.SetTag("error.message", errorMessage);
+            
+            if (!string.IsNullOrEmpty(context.Exception?.StackTrace))
+                span.SetTag("error.stackTrace", context.Exception.StackTrace);
+        }
+        else
+            span.AddEvent(new ActivityEvent("Executed", tags: new ActivityTagsCollection(new Dictionary<string, object?>
+            {
+                ["activityInstance.status"] = context.Status.ToString(),
+            })));
+        
+        span.SetTag("activityExecution.durationMs", (systemClock.UtcNow - span.StartTimeUtc).TotalMilliseconds);
    }
 }

@ -39,4 +71,4 @@ public static class OpenTelemetryTracingActivityExecutionMiddlewareExtensions
 {
    /// Installs the <see cref="OpenTelemetryTracingActivityExecutionMiddleware"/> component in the workflow execution pipeline.
    public static IActivityExecutionPipelineBuilder UseActivityExecutionTracing(this IActivityExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.Insert<OpenTelemetryTracingActivityExecutionMiddleware>(0);
-}
+}
--- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs
+++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs
@ -1,4 +1,6 @@
 using System.Diagnostics;
+using System.Text.Json;
+using Elsa.Common.Contracts;
 using Elsa.OpenTelemetry.Helpers;
 using Elsa.Workflows;
 using Elsa.Workflows.Contracts;
@ -13,28 +15,58 @@ namespace Elsa.OpenTelemetry.Middleware;
 /// Middleware that traces workflow execution using OpenTelemetry.
 /// </summary>
 [UsedImplicitly]
-public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareDelegate next) : WorkflowExecutionMiddleware(next)
+public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareDelegate next, ISystemClock systemClock) : WorkflowExecutionMiddleware(next)
 {
    /// <inheritdoc />
    public override async ValueTask InvokeAsync(WorkflowExecutionContext context)
    {
        var workflowInstanceId = context.Id;
        var workflow = context.Workflow;
-        using var activity = ElsaOpenTelemetry.ActivitySource.StartActivity($"WorkflowExecution {workflow.WorkflowMetadata.Name}", ActivityKind.Internal, Activity.Current?.Context ?? default);
-        
-        if(!string.IsNullOrWhiteSpace(context.CorrelationId))
-            activity?.AddTag("correlationId", context.CorrelationId);
-        
-        activity?.AddTag("workflowInstance.id", workflowInstanceId);
-        activity?.AddTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId);
-        activity?.AddTag("workflowDefinition.version", workflow.Identity.Version);
-        activity?.AddTag("workflowInstance.originalStatus", context.Status.ToString());
-        activity?.AddTag("workflowInstance.originalSubStatus", context.SubStatus.ToString());
-        activity?.AddEvent(new ActivityEvent("Executing"));
+        using var activity = ElsaOpenTelemetry.ActivitySource.StartActivity($"WorkflowExecution", ActivityKind.Internal, Activity.Current?.Context ?? default);
+
+        if (activity == null)
+        {
+            await Next(context);
+            return;
+        }
+
+        if (!string.IsNullOrWhiteSpace(context.CorrelationId))
+            activity.SetTag("correlationId", context.CorrelationId);
+
+        activity.SetTag("workflowInstance.id", workflowInstanceId);
+        activity.SetTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId);
+        activity.SetTag("workflowDefinition.version", workflow.Identity.Version);
+        activity.SetTag("workflowDefinition.name", workflow.WorkflowMetadata.Name);
+        activity.AddEvent(new ActivityEvent("Executing", tags: new ActivityTagsCollection(new Dictionary<string, object?>
+        {
+            ["workflowInstance.status"] = context.Status.ToString(),
+            ["workflowInstance.subStatus"] = context.SubStatus.ToString()
+        })));
        await Next(context);
-        activity?.AddEvent(new ActivityEvent("Executed"));
-        activity?.AddTag("workflowInstance.newStatus", context.Status.ToString());
-        activity?.AddTag("workflowInstance.newSubStatus", context.SubStatus.ToString());
+
+        if (context.SubStatus == WorkflowSubStatus.Faulted)
+        {
+            activity.AddEvent(new ActivityEvent("Faulted"));
+            activity.SetStatus(ActivityStatusCode.Error);
+            activity.SetTag("error", true);
+            activity.SetTag("hasIncidents", true);
+            
+            if (context.Incidents.Count > 0)
+                activity.SetTag("error.message", JsonSerializer.Serialize(context.Incidents));
+        }
+        else
+        {
+            activity.AddEvent(new ActivityEvent("Executed", tags: new ActivityTagsCollection(new Dictionary<string, object?>
+            {
+                ["workflowInstance.status"] = context.Status.ToString(),
+                ["workflowInstance.subStatus"] = context.SubStatus.ToString()
+            })));
+        }
+        
+        if (!string.IsNullOrWhiteSpace(context.CorrelationId))
+            activity.SetTag("correlationId", context.CorrelationId);
+        
+        activity.SetTag("workflowExecution.durationMs", (systemClock.UtcNow - activity.StartTimeUtc).TotalMilliseconds);
    }
 }

@ -46,4 +78,4 @@ public static class OpenTelemetryWorkflowExecutionMiddlewareExtensions
 {
    /// Installs the <see cref="OpenTelemetryTracingWorkflowExecutionMiddleware"/> component in the workflow execution pipeline.
    public static IWorkflowExecutionPipelineBuilder UseWorkflowExecutionTracing(this IWorkflowExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.Insert<OpenTelemetryTracingWorkflowExecutionMiddleware>(0);
-}
+}