From adf36927dcd19a4d8f579eb1b875c27e59cfe8d6 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Mon, 23 Sep 2024 23:35:30 +0200 Subject: [PATCH] Update Docker and k8s configs, enhance OTEL tracing Updated `docker-compose-datadog.yml` with new environment variables and updated image versions, enhancing trace sampling and service autodiscovery. Added Kubernetes deployment, service, and role files for `elsa-server`, `elsa-studio`, `plant-uml`, `postgres`, and `trace-lens`, improving the project's infrastructure. Enhanced OpenTelemetry middleware for better error handling and detailed tracing of activity and workflow executions. --- Elsa.sln | 43 +++++++++++++ docker/ElsaServer-Datadog.Dockerfile | 4 +- docker/docker-compose-datadog.yml | 63 ++++++++++++++++-- scripts/k8s/elsa-server/deployment.yaml | 60 +++++++++++++++++ scripts/k8s/elsa-server/role-binding.yaml | 11 ++++ scripts/k8s/elsa-server/role.yaml | 14 ++++ scripts/k8s/elsa-server/service-account.yml | 5 ++ scripts/k8s/elsa-server/service.yaml | 12 ++++ scripts/k8s/elsa-studio/deployment.yaml | 25 ++++++++ scripts/k8s/elsa-studio/service.yaml | 12 ++++ scripts/k8s/plant-uml/deployment.yaml | 20 ++++++ scripts/k8s/postgres/config-map.yaml | 21 ++++++ scripts/k8s/postgres/deployment.yaml | 41 ++++++++++++ scripts/k8s/postgres/service.yml | 13 ++++ scripts/k8s/trace-lens/deployment.yaml | 26 ++++++++ scripts/k8s/trace-lens/service.yaml | 25 ++++++++ ...metryTracingActivityExecutionMiddleware.cs | 54 ++++++++++++---- ...metryTracingWorkflowExecutionMiddleware.cs | 64 ++++++++++++++----- 18 files changed, 478 insertions(+), 35 deletions(-) create mode 100644 scripts/k8s/elsa-server/deployment.yaml create mode 100644 scripts/k8s/elsa-server/role-binding.yaml create mode 100644 scripts/k8s/elsa-server/role.yaml create mode 100644 scripts/k8s/elsa-server/service-account.yml create mode 100644 scripts/k8s/elsa-server/service.yaml create mode 100644 scripts/k8s/elsa-studio/deployment.yaml create mode 100644 scripts/k8s/elsa-studio/service.yaml create mode 100644 scripts/k8s/plant-uml/deployment.yaml create mode 100644 scripts/k8s/postgres/config-map.yaml create mode 100644 scripts/k8s/postgres/deployment.yaml create mode 100644 scripts/k8s/postgres/service.yml create mode 100644 scripts/k8s/trace-lens/deployment.yaml create mode 100644 scripts/k8s/trace-lens/service.yaml diff --git a/Elsa.sln b/Elsa.sln index 58f882779..8e4c08bb0 100644 --- a/Elsa.sln +++ b/Elsa.sln @@ -357,6 +357,43 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.MongoDb.UnitTests", "t EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elsa.OpenTelemetry", "src\modules\Elsa.OpenTelemetry\Elsa.OpenTelemetry.csproj", "{25880971-403E-4872-93A1-D33089E07C91}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{FE1AFEC0-7C63-4EF8-8E24-D9703590A778}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "k8s", "k8s", "{924BA0AE-46B1-40B5-992C-CFB4EB14EDB0}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "elsa-server", "elsa-server", "{A2743EC6-7117-46E2-8AC1-8CDD52C8AAEF}" + ProjectSection(SolutionItems) = preProject + scripts\k8s\elsa-server\deployment.yaml = scripts\k8s\elsa-server\deployment.yaml + scripts\k8s\elsa-server\role-binding.yaml = scripts\k8s\elsa-server\role-binding.yaml + scripts\k8s\elsa-server\role.yaml = scripts\k8s\elsa-server\role.yaml + scripts\k8s\elsa-server\service-account.yml = scripts\k8s\elsa-server\service-account.yml + scripts\k8s\elsa-server\service.yaml = scripts\k8s\elsa-server\service.yaml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "elsa-studio", "elsa-studio", "{A782913B-142F-4EA8-A2C7-82679CD2ABC1}" + ProjectSection(SolutionItems) = preProject + scripts\k8s\elsa-studio\deployment.yaml = scripts\k8s\elsa-studio\deployment.yaml + scripts\k8s\elsa-studio\service.yaml = scripts\k8s\elsa-studio\service.yaml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "plant-uml", "plant-uml", "{BFB87B50-3FE9-4AB1-8F18-96BE667C953D}" + ProjectSection(SolutionItems) = preProject + scripts\k8s\plant-uml\deployment.yaml = scripts\k8s\plant-uml\deployment.yaml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "postgres", "postgres", "{399C61BC-A51B-4B57-AA32-46BC24D4AE1C}" + ProjectSection(SolutionItems) = preProject + scripts\k8s\postgres\config-map.yaml = scripts\k8s\postgres\config-map.yaml + scripts\k8s\postgres\deployment.yaml = scripts\k8s\postgres\deployment.yaml + scripts\k8s\postgres\service.yml = scripts\k8s\postgres\service.yml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "trace-lens", "trace-lens", "{2FDF67FD-2BD3-4F73-9A24-DA255641ED25}" + ProjectSection(SolutionItems) = preProject + scripts\k8s\trace-lens\deployment.yaml = scripts\k8s\trace-lens\deployment.yaml + scripts\k8s\trace-lens\service.yaml = scripts\k8s\trace-lens\service.yaml + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -998,6 +1035,12 @@ Global {90CD37A9-C866-4D90-A3B1-8C87F53B845E} = {CBB515F3-A0EF-43B5-A907-FD4E652DD66E} {56CAA9F2-1882-4EFA-BAC0-9C3D804553F1} = {18453B51-25EB-4317-A4B3-B10518252E92} {25880971-403E-4872-93A1-D33089E07C91} = {5BA4A8FA-F7F4-45B3-AEC8-8886D35AAC79} + {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} = {FE1AFEC0-7C63-4EF8-8E24-D9703590A778} + {A2743EC6-7117-46E2-8AC1-8CDD52C8AAEF} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} + {A782913B-142F-4EA8-A2C7-82679CD2ABC1} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} + {BFB87B50-3FE9-4AB1-8F18-96BE667C953D} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} + {399C61BC-A51B-4B57-AA32-46BC24D4AE1C} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} + {2FDF67FD-2BD3-4F73-9A24-DA255641ED25} = {924BA0AE-46B1-40B5-992C-CFB4EB14EDB0} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D4B5CEAA-7D70-4FCB-A68E-B03FBE5E0E5E} diff --git a/docker/ElsaServer-Datadog.Dockerfile b/docker/ElsaServer-Datadog.Dockerfile index bd02ed1e8..15e799a1a 100644 --- a/docker/ElsaServer-Datadog.Dockerfile +++ b/docker/ElsaServer-Datadog.Dockerfile @@ -18,7 +18,7 @@ COPY *.props ./ RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj" # build and publish (UseAppHost=false creates platform independent binaries). -WORKDIR /source/src/bundles/Elsa.Server.Web +WORKDIR /source/src/apps/Elsa.Server.Web RUN dotnet build "Elsa.Server.Web.csproj" -c Release -o /app/build RUN dotnet publish "Elsa.Server.Web.csproj" -c Release -o /app/publish /p:UseAppHost=false --no-restore -f net8.0 @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* # Set PYTHONNET_PYDLL environment variable -ENV PYTHONNET_PYDLL /usr/lib/aarch64-linux-gnu/libpython3.11.so +ENV PYTHONNET_PYDLL=/usr/lib/aarch64-linux-gnu/libpython3.11.so # Copy the tracer from build target COPY --from=build /tmp/datadog-dotnet-apm.deb /tmp/datadog-dotnet-apm.deb diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml index 36fb7952f..d92ad76ed 100644 --- a/docker/docker-compose-datadog.yml +++ b/docker/docker-compose-datadog.yml @@ -12,7 +12,7 @@ services: - postgres-data:/var/lib/postgresql/data ports: - "5432:5432" - + cockroachdb: image: cockroachdb/cockroach:v22.1.0 command: start-single-node --insecure @@ -23,19 +23,20 @@ services: - cockroachdb-data:/cockroach/cockroach-data environment: - COCKROACH_DATABASE=elsa - + rabbitmq: image: "rabbitmq:3-management" ports: - "15672:15672" - "5672:5672" - + redis: image: redis:latest ports: - "127.0.0.1:6379:6379" elsa-server: + pull_policy: always build: context: ../. dockerfile: ./docker/ElsaServer-Datadog.Dockerfile @@ -46,6 +47,11 @@ services: - datadog-agent environment: DD_AGENT_HOST: datadog-agent + DD_ENV: development + DD_TRACE_DEBUG: true + DD_TRACE_OTEL_ENABLED: true + DD_SERVICE: "elsa-server-local" + DD_VERSION: "3.2.1-blueberry" ASPNETCORE_ENVIRONMENT: Development PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 PYTHONNET_RUNTIME: coreclr @@ -55,11 +61,12 @@ services: DISTRIBUTEDLOCKPROVIDER: "Postgres" ports: - "13000:8080" - + elsa-studio: + pull_policy: always build: context: ../. - dockerfile: ./docker/ElsaStudio-Datadog.Dockerfile + dockerfile: ./docker/ElsaStudio.Dockerfile environment: ASPNETCORE_ENVIRONMENT: Development ELSASERVER__URL: "http://localhost:13000/elsa/api" @@ -67,16 +74,60 @@ services: - "14000:8080" datadog-agent: - image: datadog/agent:7 + image: datadog/agent:7.42.0 environment: DD_API_KEY: "YOUR_API_KEY" DD_SITE: "datadoghq.eu" DD_LOGS_ENABLED: "true" DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true" DD_APM_ENABLED: "true" + DD_REMOTE_CONFIGURATION_ENABLED: "true" DD_APM_NON_LOCAL_TRAFFIC: "true" + + # Service autodiscovery DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server" DD_AC_EXCLUDE: "name:datadog-agent" + + # Enable priority sampling + DD_TRACE_SAMPLING_PRIORITY: "true" + + # Global rate limiting of traces (number of spans per second) + DD_TRACE_RATE_LIMIT: 100 + + # Global sample rate for all traces (applies to spans that do not match a specific sampling rule) + DD_TRACE_SAMPLE_RATE: 1.0 # Keep 100% of the traces globally (adjust as needed) + + # Sampling rules for controlling sampling of specific services and errors + DD_TRACE_SAMPLING_RULES: > + [ + { + "service": "elsa-server-local", + "name": "WorkflowExecution", + "sample_rate": 0.5 + }, + { + "service": "elsa-server-local", + "name": "WorkflowExecution", + "sample_rate": 1.0, + "condition": {"tags": {"hasIncidents": "true"}} + } + { + "service": "elsa-server-local", + "name": "ActivityExecution", + "sample_rate": 1.0, + "condition": {"tags": {"hasIncidents": "true"}} + }, + { + "service": "elsa-server-local", + "name": "ActivityExecution", + "sample_rate": 0.3 + }, + { + "service": "elsa-server-local", + "sample_rate": 1.0, + "condition": {"error": true} + } + ] volumes: - /var/run/docker.sock:/var/run/docker.sock - /proc/:/host/proc/:ro diff --git a/scripts/k8s/elsa-server/deployment.yaml b/scripts/k8s/elsa-server/deployment.yaml new file mode 100644 index 000000000..22c4f48ef --- /dev/null +++ b/scripts/k8s/elsa-server/deployment.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elsa-server-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: elsa-server + template: + metadata: + labels: + app: elsa-server + spec: + serviceAccountName: proto-cluster + containers: + - name: elsa-server + imagePullPolicy: Never + image: elsa-server:latest + ports: + - containerPort: 8080 + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: "ProtoActor__AdvertisedHost" + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: CONNECTIONSTRINGS__POSTGRESQL + value: "Server=postgres;Username=elsa;Database=elsa;Port=5432;Password=elsa;SSLMode=Prefer;MaxPoolSize=2000;Timeout=60" + - name: COR_ENABLE_PROFILING + value: "1" + - name: COR_PROFILER + value: "{918728DD-259F-4A6A-AC2B-B85E1B658318}" + - name: CORECLR_PROFILER_PATH + value: "$INSTALL_DIR/osx-x64/OpenTelemetry.AutoInstrumentation.Native.dylib" + - name: DOTNET_ADDITIONAL_DEPS + value: "$INSTALL_DIR/AdditionalDeps" + - name: DOTNET_EnableDiagnostics + value: "1" + - name: DOTNET_SHARED_STORE + value: "$INSTALL_DIR/store" + - name: DOTNET_STARTUP_HOOKS + value: "OpenTelemetry.AutoInstrumentation.StartupHook" + - name: OTEL_DOTNET_AUTO_HOME + value: "$INSTALL_DIR" + - name: OTEL_DOTNET_AUTO_LOGS_CONSOLE_EXPORTER_ENABLED + value: "false" + - name: OTEL_DOTNET_AUTO_METRICS_CONSOLE_EXPORTER_ENABLED + value: "false" + - name: OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES + value: "Proto.Actor,Elsa.Workflows" + - name: OTEL_DOTNET_AUTO_TRACES_CONSOLE_EXPORTER_ENABLED + value: "false" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://trace-lens-collector:4317" + - name: OTEL_EXPORTER_OTLP_PROTOCOL + value: "grpc" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "service.name=Elsa Server,service.version=3.3.0" \ No newline at end of file diff --git a/scripts/k8s/elsa-server/role-binding.yaml b/scripts/k8s/elsa-server/role-binding.yaml new file mode 100644 index 000000000..222766da6 --- /dev/null +++ b/scripts/k8s/elsa-server/role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: proto-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: proto-cluster +subjects: + - kind: ServiceAccount + name: proto-cluster # this is the service account that should have the role applied \ No newline at end of file diff --git a/scripts/k8s/elsa-server/role.yaml b/scripts/k8s/elsa-server/role.yaml new file mode 100644 index 000000000..51c2bf9ca --- /dev/null +++ b/scripts/k8s/elsa-server/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: proto-cluster +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - patch \ No newline at end of file diff --git a/scripts/k8s/elsa-server/service-account.yml b/scripts/k8s/elsa-server/service-account.yml new file mode 100644 index 000000000..b639b7a85 --- /dev/null +++ b/scripts/k8s/elsa-server/service-account.yml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proto-cluster + namespace: default # Ensure the namespace matches your deployment \ No newline at end of file diff --git a/scripts/k8s/elsa-server/service.yaml b/scripts/k8s/elsa-server/service.yaml new file mode 100644 index 000000000..cf9ebc193 --- /dev/null +++ b/scripts/k8s/elsa-server/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: elsa-server-service +spec: + type: LoadBalancer + sessionAffinity: None + ports: + - port: 8001 + targetPort: 8080 + selector: + app: elsa-server diff --git a/scripts/k8s/elsa-studio/deployment.yaml b/scripts/k8s/elsa-studio/deployment.yaml new file mode 100644 index 000000000..e56d84afb --- /dev/null +++ b/scripts/k8s/elsa-studio/deployment.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elsa-studio-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: elsa-studio + template: + metadata: + labels: + app: elsa-studio + spec: + containers: + - name: elsa-studio + imagePullPolicy: Never + image: elsa-studio:latest + ports: + - containerPort: 8080 + env: + - name: ASPNETCORE_ENVIRONMENT + value: Development + - name: "ELSASERVER__URL" + value: "http://localhost:8001/elsa/api" \ No newline at end of file diff --git a/scripts/k8s/elsa-studio/service.yaml b/scripts/k8s/elsa-studio/service.yaml new file mode 100644 index 000000000..067c10fca --- /dev/null +++ b/scripts/k8s/elsa-studio/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: elsa-studio-service +spec: + type: LoadBalancer + sessionAffinity: None + ports: + - port: 9001 + targetPort: 8080 + selector: + app: elsa-studio diff --git a/scripts/k8s/plant-uml/deployment.yaml b/scripts/k8s/plant-uml/deployment.yaml new file mode 100644 index 000000000..7c49679c4 --- /dev/null +++ b/scripts/k8s/plant-uml/deployment.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: plant-uml-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: plant-uml + template: + metadata: + labels: + app: plant-uml + spec: + containers: + - name: plant-uml + imagePullPolicy: Always + image: plantuml/plantuml-server:tomcat + ports: + - containerPort: 8080 \ No newline at end of file diff --git a/scripts/k8s/postgres/config-map.yaml b/scripts/k8s/postgres/config-map.yaml new file mode 100644 index 000000000..a2ea928f6 --- /dev/null +++ b/scripts/k8s/postgres/config-map.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: init-db-config +data: + init-db.sh: | + #!/bin/bash + echo "Starting database initialization" + + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + GRANT ALL ON SCHEMA public TO elsa; + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO elsa; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO elsa; + GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO elsa; + CREATE USER tracelens WITH PASSWORD 'tracelenspass'; + CREATE DATABASE tracelens; + GRANT ALL PRIVILEGES ON DATABASE tracelens TO tracelens; + GRANT ALL ON SCHEMA public TO tracelens; + EOSQL + + echo "Database initialization completed" \ No newline at end of file diff --git a/scripts/k8s/postgres/deployment.yaml b/scripts/k8s/postgres/deployment.yaml new file mode 100644 index 000000000..4ec79128b --- /dev/null +++ b/scripts/k8s/postgres/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + imagePullPolicy: Always + image: postgres:latest + # command: ["-c", "max_connections=2000"] + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + - name: init-db + mountPath: /docker-entrypoint-initdb.d/init-db.sh + subPath: init-db.sh + ports: + - containerPort: 5432 + env: + - name: POSTGRES_USER + value: "elsa" + - name: POSTGRES_PASSWORD + value: "elsa" + - name: POSTGRES_DB + value: "elsa" + + volumes: + - name: postgres-data + emptyDir: {} + - name: init-db + configMap: + name: init-db-config \ No newline at end of file diff --git a/scripts/k8s/postgres/service.yml b/scripts/k8s/postgres/service.yml new file mode 100644 index 000000000..ce3bbc467 --- /dev/null +++ b/scripts/k8s/postgres/service.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + type: LoadBalancer + ports: + - port: 5432 + targetPort: 5432 + selector: + app: postgres \ No newline at end of file diff --git a/scripts/k8s/trace-lens/deployment.yaml b/scripts/k8s/trace-lens/deployment.yaml new file mode 100644 index 000000000..2586a152d --- /dev/null +++ b/scripts/k8s/trace-lens/deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tracelens-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: trace-lens + template: + metadata: + labels: + app: trace-lens + spec: + containers: + - name: trace-lens + imagePullPolicy: Always + image: docker.io/rogeralsing/tracelens:latest + ports: + - containerPort: 5001 + - containerPort: 4317 + env: + - name: PlantUml__RemoteUrl + value: "" + - name: ConnectionStrings__DefaultConnection + value: "USER ID=tracelens;PASSWORD=tracelenspass;HOST=postgres;PORT=5432;DATABASE=tracelens;POOLING=true;" \ No newline at end of file diff --git a/scripts/k8s/trace-lens/service.yaml b/scripts/k8s/trace-lens/service.yaml new file mode 100644 index 000000000..dd41b53fa --- /dev/null +++ b/scripts/k8s/trace-lens/service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: trace-lens-dashboard +spec: + type: LoadBalancer + selector: + app: trace-lens + ports: + - name: dashboard + port: 7001 + targetPort: 5001 + # OTEL collector port is internal-only, avoid exposing it via LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + name: trace-lens-collector +spec: + selector: + app: trace-lens + ports: + - name: otel-collector + port: 4317 + targetPort: 4317 \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs index cd389bdb7..6af239905 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs @@ -1,10 +1,12 @@ using System.Diagnostics; +using Elsa.Common.Contracts; using Elsa.OpenTelemetry.Helpers; using Elsa.Workflows; using Elsa.Workflows.Contracts; using Elsa.Workflows.Pipelines.ActivityExecution; using Elsa.Workflows.Pipelines.WorkflowExecution; using JetBrains.Annotations; +using Newtonsoft.Json; using Activity = System.Diagnostics.Activity; using ActivityKind = System.Diagnostics.ActivityKind; @@ -12,22 +14,52 @@ namespace Elsa.OpenTelemetry.Middleware; /// [UsedImplicitly] -public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next) : IActivityExecutionMiddleware +public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next, ISystemClock systemClock) : IActivityExecutionMiddleware { /// public async ValueTask InvokeAsync(ActivityExecutionContext context) { var activity = context.Activity; - using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"ActivityExecution {context.ActivityDescriptor.TypeName}", ActivityKind.Internal, Activity.Current?.Context ?? default); - span?.AddTag("activity.nodeId", activity.NodeId); - span?.AddTag("activity.type", activity.Type); - span?.AddTag("activity.name", activity.Name); - span?.AddTag("activityInstance.id", context.Id); - span?.AddTag("activityInstance.originalStatus", context.Status.ToString()); - span?.AddEvent(new ActivityEvent("Executing")); + using var span = ElsaOpenTelemetry.ActivitySource.StartActivity($"ActivityExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + + if (span == null) + { + await next(context); + return; + } + + span.SetTag("activity.nodeId", activity.NodeId); + span.SetTag("activity.type", activity.Type); + span.SetTag("activity.name", activity.Name); + span.SetTag("activityInstance.id", context.Id); + + span.AddEvent(new ActivityEvent("Executing", tags: new ActivityTagsCollection(new Dictionary + { + ["activityInstance.status"] = context.Status.ToString(), + }))); + await next(context); - span?.AddEvent(new ActivityEvent("Executed")); - span?.AddTag("activityInstance.newStatus", context.Status.ToString()); + + if (context.Status == ActivityStatus.Faulted) + { + span.AddEvent(new ActivityEvent("Faulted")); + span.SetStatus(ActivityStatusCode.Error); + span.SetTag("error", true); + span.SetTag("hasIncidents", true); + + var errorMessage = string.IsNullOrWhiteSpace(context.Exception?.Message) ? "Unknown error" : context.Exception.Message; + span.SetTag("error.message", errorMessage); + + if (!string.IsNullOrEmpty(context.Exception?.StackTrace)) + span.SetTag("error.stackTrace", context.Exception.StackTrace); + } + else + span.AddEvent(new ActivityEvent("Executed", tags: new ActivityTagsCollection(new Dictionary + { + ["activityInstance.status"] = context.Status.ToString(), + }))); + + span.SetTag("activityExecution.durationMs", (systemClock.UtcNow - span.StartTimeUtc).TotalMilliseconds); } } @@ -39,4 +71,4 @@ public static class OpenTelemetryTracingActivityExecutionMiddlewareExtensions { /// Installs the component in the workflow execution pipeline. public static IActivityExecutionPipelineBuilder UseActivityExecutionTracing(this IActivityExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.Insert(0); -} +} \ No newline at end of file diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index ae6d67572..43f40f4c5 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -1,4 +1,6 @@ using System.Diagnostics; +using System.Text.Json; +using Elsa.Common.Contracts; using Elsa.OpenTelemetry.Helpers; using Elsa.Workflows; using Elsa.Workflows.Contracts; @@ -13,28 +15,58 @@ namespace Elsa.OpenTelemetry.Middleware; /// Middleware that traces workflow execution using OpenTelemetry. /// [UsedImplicitly] -public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareDelegate next) : WorkflowExecutionMiddleware(next) +public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareDelegate next, ISystemClock systemClock) : WorkflowExecutionMiddleware(next) { /// public override async ValueTask InvokeAsync(WorkflowExecutionContext context) { var workflowInstanceId = context.Id; var workflow = context.Workflow; - using var activity = ElsaOpenTelemetry.ActivitySource.StartActivity($"WorkflowExecution {workflow.WorkflowMetadata.Name}", ActivityKind.Internal, Activity.Current?.Context ?? default); - - if(!string.IsNullOrWhiteSpace(context.CorrelationId)) - activity?.AddTag("correlationId", context.CorrelationId); - - activity?.AddTag("workflowInstance.id", workflowInstanceId); - activity?.AddTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId); - activity?.AddTag("workflowDefinition.version", workflow.Identity.Version); - activity?.AddTag("workflowInstance.originalStatus", context.Status.ToString()); - activity?.AddTag("workflowInstance.originalSubStatus", context.SubStatus.ToString()); - activity?.AddEvent(new ActivityEvent("Executing")); + using var activity = ElsaOpenTelemetry.ActivitySource.StartActivity($"WorkflowExecution", ActivityKind.Internal, Activity.Current?.Context ?? default); + + if (activity == null) + { + await Next(context); + return; + } + + if (!string.IsNullOrWhiteSpace(context.CorrelationId)) + activity.SetTag("correlationId", context.CorrelationId); + + activity.SetTag("workflowInstance.id", workflowInstanceId); + activity.SetTag("workflowDefinition.definitionId", workflow.Identity.DefinitionId); + activity.SetTag("workflowDefinition.version", workflow.Identity.Version); + activity.SetTag("workflowDefinition.name", workflow.WorkflowMetadata.Name); + activity.AddEvent(new ActivityEvent("Executing", tags: new ActivityTagsCollection(new Dictionary + { + ["workflowInstance.status"] = context.Status.ToString(), + ["workflowInstance.subStatus"] = context.SubStatus.ToString() + }))); await Next(context); - activity?.AddEvent(new ActivityEvent("Executed")); - activity?.AddTag("workflowInstance.newStatus", context.Status.ToString()); - activity?.AddTag("workflowInstance.newSubStatus", context.SubStatus.ToString()); + + if (context.SubStatus == WorkflowSubStatus.Faulted) + { + activity.AddEvent(new ActivityEvent("Faulted")); + activity.SetStatus(ActivityStatusCode.Error); + activity.SetTag("error", true); + activity.SetTag("hasIncidents", true); + + if (context.Incidents.Count > 0) + activity.SetTag("error.message", JsonSerializer.Serialize(context.Incidents)); + } + else + { + activity.AddEvent(new ActivityEvent("Executed", tags: new ActivityTagsCollection(new Dictionary + { + ["workflowInstance.status"] = context.Status.ToString(), + ["workflowInstance.subStatus"] = context.SubStatus.ToString() + }))); + } + + if (!string.IsNullOrWhiteSpace(context.CorrelationId)) + activity.SetTag("correlationId", context.CorrelationId); + + activity.SetTag("workflowExecution.durationMs", (systemClock.UtcNow - activity.StartTimeUtc).TotalMilliseconds); } } @@ -46,4 +78,4 @@ public static class OpenTelemetryWorkflowExecutionMiddlewareExtensions { /// Installs the component in the workflow execution pipeline. public static IWorkflowExecutionPipelineBuilder UseWorkflowExecutionTracing(this IWorkflowExecutionPipelineBuilder pipelineBuilder) => pipelineBuilder.Insert(0); -} +} \ No newline at end of file