Add OpenTelemetry auto-instrumentation support

Introduce OpenTelemetry configuration and dependencies for observability. Replace DataDog tracer with OpenTelemetry auto-instrumentation in the Dockerfile. Adjust project dependencies and log levels to incorporate OpenTelemetry without disrupting existing functionality.
2024-09-25 00:49:21 +02:00 · 2024-09-25 00:49:21 +02:00 · 758b60767b
parent df827b0fda
commit 758b60767b
8 changed files with 115 additions and 42 deletions
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@ -37,7 +37,8 @@
    <PackageVersion Include="Fody" Version="6.8.1" />
    <PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1" />
    <PackageVersion Include="Google.Protobuf" Version="3.27.2" />
-    <PackageVersion Include="Grpc.Tools" Version="2.64.0" />
+    <PackageVersion Include="Grpc.Net.Client" Version="2.66.0" />
+    <PackageVersion Include="Grpc.Tools" Version="2.66.0" />
    <PackageVersion Include="Hangfire" Version="1.8.14" />
    <PackageVersion Include="Hangfire.MemoryStorage" Version="1.8.1.1" />
    <PackageVersion Include="Hangfire.Storage.SQLite" Version="0.4.2" />
@ -65,6 +66,9 @@
    <PackageVersion Include="NuGet.Protocol" Version="6.10.1" />
    <PackageVersion Include="Nuke.Components" Version="8.0.0" />
    <PackageVersion Include="Open.Linq.AsyncExtensions" Version="1.2.0" />
+    <PackageVersion Include="OpenTelemetry.AutoInstrumentation" Version="1.7.0" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.StackExchangeRedis" Version="1.0.0-rc9.15" />
    <PackageVersion Include="Polly.Extensions.Http" Version="3.0.0" />
    <PackageVersion Include="PolySharp" Version="1.14.1" />
    <PackageVersion Include="Proto.Actor" Version="1.6.0" />
--- a/Elsa.sln
+++ b/Elsa.sln
@ -130,6 +130,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{986E54
 		docker\docker-compose.yml = docker\docker-compose.yml
 		docker\docker-compose-datadog.yml = docker\docker-compose-datadog.yml
 		docker\ElsaServer-Datadog.Dockerfile = docker\ElsaServer-Datadog.Dockerfile
+		docker\otel-collector-config.yaml = docker\otel-collector-config.yaml
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Samples.AspNet.RunTaskIntegration", "samples\aspnet\Elsa.Samples.AspNet.RunTaskIntegration\Elsa.Samples.AspNet.RunTaskIntegration.csproj", "{51050209-EC2F-4DC7-8F46-07E22B7811CD}"
--- a/docker/ElsaServer-Datadog.Dockerfile
+++ b/docker/ElsaServer-Datadog.Dockerfile
@ -4,25 +4,20 @@
 FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim AS build
 WORKDIR /source

-# Determine the architecture and download the appropriate version of the tracer
-RUN ARCH=$(if [ "$(uname -m)" = "x86_64" ]; then echo "amd64"; elif [ "$(uname -m)" = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi) \
-    && TRACER_VERSION=$(curl -s https://api.github.com/repos/DataDog/dd-trace-dotnet/releases/latest | grep tag_name | cut -d '"' -f 4 | cut -c2-) \
-    && curl -Lo /tmp/datadog-dotnet-apm.deb https://github.com/DataDog/dd-trace-dotnet/releases/download/v${TRACER_VERSION}/datadog-dotnet-apm_${TRACER_VERSION}_${ARCH}.deb
-
-# copy sources.
+# Copy sources.
 COPY src/. ./src
 COPY ./NuGet.Config ./
 COPY *.props ./

-# restore packages.
+# Restore packages.
 RUN dotnet restore "./src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj"

-# build and publish (UseAppHost=false creates platform independent binaries).
+# Build and publish (UseAppHost=false creates platform independent binaries).
 WORKDIR /source/src/bundles/Elsa.Server.Web
 RUN dotnet build "Elsa.Server.Web.csproj" -c Release -o /app/build
 RUN dotnet publish "Elsa.Server.Web.csproj" -c Release -o /app/publish /p:UseAppHost=false --no-restore -f net8.0

-# move binaries into smaller base image.
+# Move binaries into smaller base image.
 FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim AS base
 WORKDIR /app
 COPY --from=build /app/publish ./
@ -38,21 +33,25 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Set PYTHONNET_PYDLL environment variable
 ENV PYTHONNET_PYDLL=/usr/lib/aarch64-linux-gnu/libpython3.11.so

-# Copy the tracer from build target
-COPY --from=build /tmp/datadog-dotnet-apm.deb /tmp/datadog-dotnet-apm.deb
-# Install the tracer
-RUN mkdir -p /opt/datadog \
-    && mkdir -p /var/log/datadog \
-    && dpkg -i /tmp/datadog-dotnet-apm.deb \
-    && rm /tmp/datadog-dotnet-apm.deb
+# Install dependencies
+RUN apt-get update && apt-get install -y wget unzip curl

-# Enable the tracer
-ENV CORECLR_ENABLE_PROFILING=1
-ENV CORECLR_PROFILER={846F5F1C-F9AE-4B07-969E-05C26BC060D8}
-ENV CORECLR_PROFILER_PATH=/opt/datadog/Datadog.Trace.ClrProfiler.Native.so
-ENV DD_DOTNET_TRACER_HOME=/opt/datadog
-ENV DD_INTEGRATIONS=/opt/datadog/integrations.json
+# Set environment variables for OpenTelemetry Auto-Instrumentation
+ENV OTEL_DOTNET_AUTO_HOME=/otel
+ENV OTEL_LOG_LEVEL="debug"
+
+# Download and extract OpenTelemetry Auto-Instrumentation
+ARG OTEL_VERSION=1.7.0
+RUN mkdir /otel
+RUN curl -L -o /otel/otel-dotnet-install.sh https://github.com/open-telemetry/opentelemetry-dotnet-instrumentation/releases/download/v${OTEL_VERSION}/otel-dotnet-auto-install.sh
+RUN chmod +x /otel/otel-dotnet-install.sh
+RUN /bin/bash /otel/otel-dotnet-install.sh
+
+# Provide necessary permissions for the script to execute
+RUN chmod +x /otel/instrument.sh

 EXPOSE 8080/tcp
 EXPOSE 443/tcp
-ENTRYPOINT ["dotnet", "Elsa.Server.Web.dll"]
+
+# Instrument the application and start it
+ENTRYPOINT ["/bin/bash", "-c", "source /otel/instrument.sh && dotnet Elsa.Server.Web.dll"]
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@ -1,6 +1,4 @@
-version: '3.7'
-
-services:
+services:
  postgres:
    image: postgres:latest
    command: -c 'max_connections=2000'
--- a/docker/otel-collector-config.yaml
+++ b/docker/otel-collector-config.yaml
@ -0,0 +1,79 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+  hostmetrics:
+    collection_interval: 10s
+    scrapers:
+      paging:
+        metrics:
+          system.paging.utilization:
+            enabled: true
+      cpu:
+        metrics:
+          system.cpu.utilization:
+            enabled: true
+      disk:
+      filesystem:
+        metrics:
+          system.filesystem.utilization:
+            enabled: true
+      load:
+      memory:
+      network:
+      processes:
+  docker_stats:
+    metrics:
+      container.network.io.usage.rx_packets:
+        enabled: true
+      container.network.io.usage.tx_packets:
+        enabled: true
+      container.cpu.usage.system:
+        enabled: true
+      container.memory.rss:
+        enabled: true
+      container.blockio.io_serviced_recursive:
+        enabled: true
+
+processors:
+  batch:
+    send_batch_max_size: 100
+    send_batch_size: 10
+    timeout: 1s
+
+connectors:
+  datadog/connector:
+
+exporters:
+  debug:
+    verbosity: detailed
+  datadog:
+    api:
+      site: ${env:DD_SITE}
+      key: ${env:DD_API_KEY}
+
+service:
+  pipelines:
+    metrics:
+      receivers: [ hostmetrics, otlp, datadog/connector ]
+      processors: [ batch ]
+      exporters: [ datadog ]
+    traces:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ datadog/connector ]
+    traces/sampling:
+      # This pipeline has a Datadog connector, a batch processor and a Datadog exporter.
+      # It receivers all traces from the Datadog connector and sends them to Datadog.
+      # Add any sampling here, so that the generated trace metrics account for all traces.
+      receivers: [ datadog/connector ]
+      # Add any sampling here
+      processors: [ ]
+      exporters: [ datadog ]
+    logs:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ datadog ]
--- a/src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj
+++ b/src/bundles/Elsa.Server.Web/Elsa.Server.Web.csproj
@ -51,6 +51,9 @@
        <PackageReference Include="DistributedLock.Postgres"/>
        <PackageReference Include="DistributedLock.Redis"/>
        <PackageReference Include="FluentStorage.Azure.Blobs"/>
+        <PackageReference Include="Grpc.Net.Client" />
+        <PackageReference Include="OpenTelemetry.AutoInstrumentation" />
+        <PackageReference Include="OpenTelemetry.Instrumentation.StackExchangeRedis" />
        <PackageReference Include="Proto.Persistence.Sqlite"/>
        <PackageReference Include="Proto.Persistence.SqlServer"/>
    </ItemGroup>
--- a/src/bundles/Elsa.Server.Web/Program.cs
+++ b/src/bundles/Elsa.Server.Web/Program.cs
@ -384,7 +384,6 @@ services
    });

 services.Configure<CachingOptions>(options => options.CacheDuration = TimeSpan.FromDays(1));
-
 services.AddHealthChecks();
 services.AddControllers();
 services.AddCors(cors => cors.AddDefaultPolicy(policy => policy.AllowAnyHeader().AllowAnyMethod().AllowAnyOrigin().WithExposedHeaders("*")));
@ -392,11 +391,6 @@ services.AddCors(cors => cors.AddDefaultPolicy(policy => policy.AllowAnyHeader()
 // Build the web application.
 var app = builder.Build();

-// app.UseSimulatedLatency(
-//     TimeSpan.FromMilliseconds(1000),
-//     TimeSpan.FromMilliseconds(3000)
-// );
-
 // Configure the pipeline.
 if (app.Environment.IsDevelopment())
    app.UseDeveloperExceptionPage();
--- a/src/bundles/Elsa.Server.Web/appsettings.json
+++ b/src/bundles/Elsa.Server.Web/appsettings.json
@ -1,15 +1,10 @@
 {
  "Logging": {
    "LogLevel": {
-      "Default": "Warning",
-      "Elsa": "Warning",
-      "MassTransit": "Warning",
-      "Microsoft.Extensions.Http": "Warning",
+      "Default": "Debug",
+      "Microsoft": "Warning",
      "Microsoft.Hosting.Lifetime": "Information",
-      "Microsoft.EntityFrameworkCore": "Warning",
-      "Microsoft.AspNetCore": "Warning",
-      "Quartz": "Warning",
-      "System.Net.Http": "Warning"
+      "OpenTelemetry": "Debug"
    }
  },
  "AllowedHosts": "*",