From df827b0fda1da3b5d11cfd37db8d01f0346c95a7 Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Tue, 24 Sep 2024 00:27:51 +0200 Subject: [PATCH] Update Datadog config, add incident error serialization Enhanced the Datadog tracing configuration to include priority sampling, rate limits, and sampling rules in the docker-compose-datadog.yml. Updated the agent image version and improved error message serialization in OpenTelemetry middleware to use custom JSON serializer options. Adjusted Dockerfile path to correct bundle location. --- docker/ElsaServer-Datadog.Dockerfile | 2 +- docker/docker-compose-datadog.yml | 69 ++++++++----------- ...metryTracingWorkflowExecutionMiddleware.cs | 7 +- 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/docker/ElsaServer-Datadog.Dockerfile b/docker/ElsaServer-Datadog.Dockerfile index 15e799a1a..5d9d9a4b4 100644 --- a/docker/ElsaServer-Datadog.Dockerfile +++ b/docker/ElsaServer-Datadog.Dockerfile @@ -18,7 +18,7 @@ COPY *.props ./ RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj" # build and publish (UseAppHost=false creates platform independent binaries). -WORKDIR /source/src/apps/Elsa.Server.Web +WORKDIR /source/src/bundles/Elsa.Server.Web RUN dotnet build "Elsa.Server.Web.csproj" -c Release -o /app/build RUN dotnet publish "Elsa.Server.Web.csproj" -c Release -o /app/publish /p:UseAppHost=false --no-restore -f net8.0 diff --git a/docker/docker-compose-datadog.yml b/docker/docker-compose-datadog.yml index d92ad76ed..5ba620230 100644 --- a/docker/docker-compose-datadog.yml +++ b/docker/docker-compose-datadog.yml @@ -52,6 +52,33 @@ services: DD_TRACE_OTEL_ENABLED: true DD_SERVICE: "elsa-server-local" DD_VERSION: "3.2.1-blueberry" + + # Enable priority sampling + DD_TRACE_SAMPLING_PRIORITY: "true" + + # Global rate limiting of traces (number of spans per second) + DD_TRACE_RATE_LIMIT: 100 + + # Global sample rate for all traces (applies to spans that do not match a specific sampling rule) + DD_TRACE_SAMPLE_RATE: 1.0 # Keep 100% of the traces globally (adjust as needed) + + # Sampling rules for controlling sampling of specific services and errors + DD_TRACE_SAMPLING_RULES: > + [ + { + "service": "elsa-server-local", + "name": "WorkflowExecution", + "sample_rate": 1.0, + "condition": {"tags": {"hasIncidents": "true"}} + }, + { + "service": "elsa-server-local", + "name": "ActivityExecution", + "sample_rate": 1.0, + "condition": {"tags": {"hasIncidents": "true"}} + } + ] + ASPNETCORE_ENVIRONMENT: Development PYTHONNET_PYDLL: /opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/bin/python3.11 PYTHONNET_RUNTIME: coreclr @@ -74,7 +101,7 @@ services: - "14000:8080" datadog-agent: - image: datadog/agent:7.42.0 + image: datadog/agent:7.57.1 environment: DD_API_KEY: "YOUR_API_KEY" DD_SITE: "datadoghq.eu" @@ -88,46 +115,6 @@ services: DD_AC_INCLUDE: "name:postgres,name:rabbitmq,name:redis,name:elsa-server" DD_AC_EXCLUDE: "name:datadog-agent" - # Enable priority sampling - DD_TRACE_SAMPLING_PRIORITY: "true" - - # Global rate limiting of traces (number of spans per second) - DD_TRACE_RATE_LIMIT: 100 - - # Global sample rate for all traces (applies to spans that do not match a specific sampling rule) - DD_TRACE_SAMPLE_RATE: 1.0 # Keep 100% of the traces globally (adjust as needed) - - # Sampling rules for controlling sampling of specific services and errors - DD_TRACE_SAMPLING_RULES: > - [ - { - "service": "elsa-server-local", - "name": "WorkflowExecution", - "sample_rate": 0.5 - }, - { - "service": "elsa-server-local", - "name": "WorkflowExecution", - "sample_rate": 1.0, - "condition": {"tags": {"hasIncidents": "true"}} - } - { - "service": "elsa-server-local", - "name": "ActivityExecution", - "sample_rate": 1.0, - "condition": {"tags": {"hasIncidents": "true"}} - }, - { - "service": "elsa-server-local", - "name": "ActivityExecution", - "sample_rate": 0.3 - }, - { - "service": "elsa-server-local", - "sample_rate": 1.0, - "condition": {"error": true} - } - ] volumes: - /var/run/docker.sock:/var/run/docker.sock - /proc/:/host/proc/:ro diff --git a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs index 43f40f4c5..83382d4d9 100644 --- a/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs +++ b/src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingWorkflowExecutionMiddleware.cs @@ -1,10 +1,13 @@ using System.Diagnostics; using System.Text.Json; using Elsa.Common.Contracts; +using Elsa.Expressions.Services; +using Elsa.Extensions; using Elsa.OpenTelemetry.Helpers; using Elsa.Workflows; using Elsa.Workflows.Contracts; using Elsa.Workflows.Pipelines.WorkflowExecution; +using Elsa.Workflows.Serialization.Converters; using JetBrains.Annotations; using Activity = System.Diagnostics.Activity; using ActivityKind = System.Diagnostics.ActivityKind; @@ -17,6 +20,8 @@ namespace Elsa.OpenTelemetry.Middleware; [UsedImplicitly] public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareDelegate next, ISystemClock systemClock) : WorkflowExecutionMiddleware(next) { + private readonly JsonSerializerOptions? _incidentSerializerOptions = new JsonSerializerOptions().WithConverters(new TypeJsonConverter(WellKnownTypeRegistry.CreateDefault())); + /// public override async ValueTask InvokeAsync(WorkflowExecutionContext context) { @@ -52,7 +57,7 @@ public class OpenTelemetryTracingWorkflowExecutionMiddleware(WorkflowMiddlewareD activity.SetTag("hasIncidents", true); if (context.Incidents.Count > 0) - activity.SetTag("error.message", JsonSerializer.Serialize(context.Incidents)); + activity.SetTag("error.message", JsonSerializer.Serialize(context.Incidents, _incidentSerializerOptions)); } else {