OpenTelemetry: Unified Observability Standard

Traditional observability architecture (fragmented):

Application code
├── Prometheus Client (Metrics)
│   └── → Prometheus → Grafana
├── Logback + Filebeat (Logs)
│   └── → Elasticsearch → Kibana
└── Jaeger Client (Traces)
    └── → Jaeger → Jaeger UI

Problems:
1. Three SDKs, three configurations, three operations stacks
2. No correlation between Metrics / Logs / Traces (TraceID not linked to logs)
3. High cost to switch backends (switching from Prometheus to Datadog requires code changes)
4. Each language needs different client libraries maintained

OpenTelemetry unified architecture:

Application code
└── OpenTelemetry SDK (unified collection)
    ├── Metrics
    ├── Logs (with TraceID)
    └── Traces
            │
            ▼
    OpenTelemetry Collector (unified processing)
    ├── Filter / Aggregate / Sample
    ├── Format conversion
    └── Distribute to backends
            │
    ┌───────┼───────┐
    ▼       ▼       ▼
Prometheus  ELK   Jaeger
(Grafana)  (Kibana)(Jaeger UI)

┌──────────────────────────────────────────────────────────┐
│                OpenTelemetry Three Signals                 │
│                                                          │
│  Traces                                                   │
│  ├── Records the complete path of a request in a         │
│  │   distributed system                                   │
│  ├── Each Span contains: operation name, time, status,   │
│  │   attributes                                           │
│  └── Links Spans across services via TraceID             │
│                                                          │
│  Metrics                                                  │
│  ├── Records aggregatable numeric data                   │
│  ├── Types: Counter / Gauge / Histogram / Summary        │
│  └── Includes Resource and Attributes (labels)           │
│                                                          │
│  Logs                                                     │
│  ├── Records discrete events                             │
│  ├── Correlated with TraceID / SpanID (key feature)      │
│  └── Includes Severity, Body, Attributes                 │
└──────────────────────────────────────────────────────────┘

{
  "trace_id": "7b3cf5b0123456789abcdef012345678",
  "span_id": "0123456789abcdef",
  "parent_span_id": "fedcba9876543210",
  "name": "GET /api/orders",
  "kind": "SERVER",
  "start_time": "2026-07-10T10:00:00.123456789Z",
  "end_time": "2026-07-10T10:00:00.456789123Z",
  "status": {
    "code": "ERROR",
    "description": "Database connection timeout"
  },
  "attributes": {
    "http.method": "GET",
    "http.url": "/api/orders",
    "http.status_code": 500,
    "db.system": "mysql",
    "db.statement": "SELECT * FROM orders"
  },
  "events": [
    {
      "name": "exception",
      "time": "2026-07-10T10:00:00.400Z",
      "attributes": {
        "exception.type": "java.sql.SQLException",
        "exception.message": "Connection timeout"
      }
    }
  ],
  "resource": {
    "service.name": "order-service",
    "service.version": "1.2.3",
    "host.name": "order-pod-abc123"
  }
}

{
  "timestamp": "2026-07-10T10:00:00.500Z",
  "trace_id": "7b3cf5b0123456789abcdef012345678",
  "span_id": "0123456789abcdef",
  "severity_text": "ERROR",
  "severity_number": 17,
  "body": "Failed to process order: Database connection timeout",
  "attributes": {
    "order_id": "ORD-12345",
    "user_id": "USR-67890",
    "retry_count": 3
  },
  "resource": {
    "service.name": "order-service",
    "service.version": "1.2.3"
  }
}

┌──────────────────────────────────────────────────────┐
│              OTel Collector Architecture              │
│                                                      │
│  ┌─────────┐  ┌─────────┐  ┌─────────┐             │
│  │ Receiver │  │ Receiver │  │ Receiver │  ← Receive │
│  │ (OTLP)   │  │ (Jaeger) │  │ (Prom)  │             │
│  └────┬────┘  └────┬────┘  └────┬────┘             │
│       │            │            │                    │
│       └────────────┼────────────┘                   │
│                    ▼                                  │
│              ┌──────────┐                             │
│              │ Processor│  ← Process (filter/sample/enrich) │
│              └────┬─────┘                             │
│                   │                                   │
│       ┌───────────┼───────────┐                      │
│       ▼           ▼           ▼                      │
│  ┌─────────┐ ┌─────────┐ ┌─────────┐                │
│  │ Exporter│ │ Exporter│ │ Exporter│  ← Export       │
│  │ (OTLP)  │ │ (Prom)  │ │ (ES/Loki)│                │
│  └─────────┘ └─────────┘ └─────────┘                │
└──────────────────────────────────────────────────────┘

# otel-collector-config.yaml
receivers:
  # Receive OTLP protocol data (SDK direct)
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  # Receive Jaeger format data
  jaeger:
    protocols:
      grpc:
        endpoint: 0.0.0.0:14250
      thrift_http:
        endpoint: 0.0.0.0:14268

  # Receive Prometheus scrape
  prometheus:
    config:
      scrape_configs:
        - job_name: 'otel-collector'
          scrape_interval: 15s
          static_configs:
            - targets: ['localhost:8888']

processors:
  # Batch processing
  batch:
    timeout: 5s
    send_batch_size: 1000
    send_batch_max_size: 2000

  # Memory limiter
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  # Filtering
  filter:
    traces:
      span:
        - 'attributes["http.route"] == "/health"'  # Filter health check Spans
    metrics:
      metric:
        - 'name == "process.runtime.jvm.gc.time"'

  # Attribute processing
  attributes:
    actions:
      - key: environment
        value: production
        action: upsert
      - key: http.request_header.authorization
        action: delete  # Remove sensitive info

  # Sampling (tail-based)
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow
        type: latency
        latency:
          threshold_ms: 1000
      - name: sample
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

  # Resource processing
  resource:
    attributes:
      - key: deployment.environment
        value: production
        action: upsert

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  zpages:
    endpoint: 0.0.0.0:55679

exporters:
  # Export to Jaeger
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  # Export to Prometheus (via remote_write)
  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write

  # Export to Loki
  loki:
    endpoint: http://loki:3100/loki/api/v1/push

  # Export to Elasticsearch
  elasticsearch:
    endpoints:
      - http://es:9200
    index: otel-logs

service:
  extensions: [health_check, zpages]
  pipelines:
    traces:
      receivers: [otlp, jaeger]
      processors: [memory_limiter, filter, tail_sampling, resource, batch]
      exporters: [otlp/jaeger]

    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, filter, resource, batch]
      exporters: [prometheusremotewrite]

    logs:
      receivers: [otlp]
      processors: [memory_limiter, filter, attributes, resource, batch]
      exporters: [loki, elasticsearch]

Application nodes → Agent Collector → Gateway Collector → Backends
               (local collection)        (centralized processing)

# DaemonSet mode (Agent Collector)
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector-agent
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: otel-collector-agent
  template:
    metadata:
      labels:
        app: otel-collector-agent
    spec:
      containers:
        - name: collector
          image: otel/opentelemetry-collector-contrib:0.103.0
          args: ["--config=/etc/otel/config.yaml"]
          ports:
            - containerPort: 4317  # OTLP gRPC
            - containerPort: 4318  # OTLP HTTP
            - containerPort: 13133  # Health check
          volumeMounts:
            - name: config
              mountPath: /etc/otel
          resources:
            limits:
              cpu: 500m
              memory: 512Mi
            requests:
              cpu: 100m
              memory: 128Mi
      volumes:
        - name: config
          configMap:
            name: otel-collector-config

package main

import (
    "context"
    "log"
    "net/http"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
    "go.opentelemetry.io/otel/trace"
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func initTracer() func() {
    // Create OTLP exporter
    exporter, err := otlptracehttp.New(context.Background(),
        otlptracehttp.WithEndpoint("otel-collector:4318"),
        otlptracehttp.WithInsecure(),
    )
    if err != nil {
        log.Fatalf("Failed to create exporter: %v", err)
    }

    // Create Resource (identifies current service)
    res, _ := resource.New(context.Background(),
        resource.WithAttributes(
            semconv.ServiceName("order-service"),
            semconv.ServiceVersion("1.0.0"),
            semconv.DeploymentEnvironment("production"),
        ),
    )

    // Create TracerProvider
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.TraceIDRatioBased(0.1)), // 10% sampling
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.TraceContext{})

    return func() {
        tp.Shutdown(context.Background())
    }
}

func main() {
    shutdown := initTracer()
    defer shutdown()

    // Use otelhttp for auto-instrumenting HTTP requests
    handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        tracer := otel.Tracer("order-service")
        ctx, span := tracer.Start(r.Context(), "processOrder",
            trace.WithAttributes(
                semconv.HTTPMethod(r.Method),
                semconv.HTTPTarget(r.URL.Path),
            ),
        )
        defer span.End()

        // Simulate business logic
        processOrder(ctx, r.URL.Query().Get("order_id"))

        w.Write([]byte(`{"status": "ok"}`))
    })

    wrappedHandler := otelhttp.NewHandler(handler, "HTTP")
    http.ListenAndServe(":8080", wrappedHandler)
}

func processOrder(ctx context.Context, orderID string) {
    tracer := otel.Tracer("order-service")
    _, span := tracer.Start(ctx, "processOrder",
        trace.WithAttributes(
            attribute.String("order.id", orderID),
        ),
    )
    defer span.End()

    // Simulate database query
    queryDatabase(ctx, orderID)
}

func queryDatabase(ctx context.Context, query string) {
    tracer := otel.Tracer("order-service")
    _, span := tracer.Start(ctx, "db.query",
        trace.WithAttributes(
            attribute.String("db.system", "mysql"),
            attribute.String("db.statement", query),
        ),
    )
    defer span.End()
    // Database query logic...
}

// Add dependency
// build.gradle
dependencies {
    implementation 'io.opentelemetry.instrumentation:opentelemetry-spring-boot-starter:1.32.0'
}

// application.yml
otel:
  exporter:
    otlp:
      endpoint: http://otel-collector:4318
  service:
    name: order-service
    version: 1.0.0
  traces:
    sampler:
      type: parentbased_traceidratio
      arg: 0.1

// Java uses MDC to auto-inject TraceID
import org.slf4j.MDC;

// OTel SDK automatically writes TraceID to MDC
// Log format includes %X{trace_id} and %X{span_id}

// logback.xml
<pattern>
  %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level [%X{trace_id},%X{span_id}] %logger - %msg%n
</pattern>

// Output example:
// 2026-07-10 10:00:00 [http-nio-8080-exec-1] ERROR [7b3cf5b0123456789abcdef012345678,0123456789abcdef] OrderService - Failed to process order

Application code (OTel SDK)
    │
    ▼
OTel Collector
    │
    ├── Traces → Tempo
    ├── Metrics → Mimir / Prometheus
    └── Logs → Loki
            │
            ▼
        Grafana (unified visualization)

exporters:
  # Datadog
  datadog:
    api:
      key: ${DD_API_KEY}
      site: datadoghq.com

  # New Relic
  otlp/newrelic:
    endpoint: otlp.nr-data.net:4317
    headers:
      api-key: ${NEW_RELIC_LICENSE_KEY}

  # Honeycomb
  otlp/honeycomb:
    endpoint: api.honeycomb.io:4317
    headers:
      "x-honeycomb-team": ${HONEYCOMB_API_KEY}

processors:
  tail_sampling:
    decision_wait: 10s        # Wait 10s to collect complete Trace
    num_traces: 50000          # Traces cached in memory
    expected_new_traces_per_sec: 1000

    policies:
      # Policy 1: Keep all error requests
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]

      # Policy 2: Keep slow requests (> 1s)
      - name: slow
        type: latency
        latency:
          threshold_ms: 1000

      # Policy 3: Keep requests from critical services
      - name: critical-service
        type: string_attribute
        string_attribute:
          key: service.name
          values: ["payment-service", "order-service"]

      # Policy 4: Sample 10% of the rest
      - name: sample
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

1000 requests → tail sampling → retained Traces:
  - 5 error requests → all kept
  - 20 slow requests → all kept
  - 100 critical service requests → all kept
  - Remaining 875 × 10% → 87 sampled
  Total: 5 + 20 + 100 + 87 = 212 Traces (21.2%)

Phase 1: Deploy Collector (no app changes)
  → Collector receives existing format data, forwards to existing backends
  → Validate Collector stability

Phase 2: Adopt OTel SDK (Traces first)
  → New services use OTel SDK
  → Gradually replace old services
  → Collector receives both old and new formats simultaneously

Phase 3: Integrate Metrics and Logs
  → Application logs correlate with TraceID
  → Metrics collected via OTel SDK

Phase 4: Unify backends
  → All data processed through Collector
  → Switch backends as needed

receivers:
  # New format: OTel SDK sends
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

  # Old format: Jaeger Client sends
  jaeger:
    protocols:
      thrift_http:
        endpoint: 0.0.0.0:14268

  # Old format: Prometheus scrape
  prometheus:
    config:
      scrape_configs:
        - job_name: 'legacy-apps'
          static_configs:
            - targets: ['app-1:9090']

processors:
  batch:
    timeout: 5s               # Batch timeout
    send_batch_size: 1000     # Batch size
    send_batch_max_size: 2000 # Max batch

  memory_limiter:
    check_interval: 1s        # Check interval
    limit_mib: 1024           # Memory limit
    spike_limit_mib: 256      # Burst memory

service:
  telemetry:
    logs:
      level: info              # Log level
    metrics:
      address: 0.0.0.0:8888   # Self-metrics port

# Resource attribute standards
resource:
  attributes:
    # Required attributes
    service.name: "order-service"           # Service name
    service.version: "1.2.3"               # Version

    # Recommended attributes
    deployment.environment: "production"    # Environment
    host.name: "order-pod-abc"              # Hostname
    k8s.namespace.name: "production"        # K8s namespace
    k8s.pod.name: "order-pod-abc"           # K8s Pod name

    # Optional attributes
    service.instance.id: "uuid-xxxx"       # Instance ID
    cloud.provider: "aws"                  # Cloud provider
    cloud.region: "us-east-1"             # Region

# Prometheus scraping Collector metrics
scrape_configs:
  - job_name: 'otel-collector'
    static_configs:
      - targets: ['otel-collector:8888']

# Key alerts
groups:
  - name: otel
    rules:
      - alert: OTelCollectorDroppingData
        expr: rate(otelcol_processor_refused_spans[5m]) > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "OTel Collector is dropping Span data"

      - alert: OTelCollectorQueueFull
        expr: otelcol_exporter_queue_size / otelcol_exporter_queue_capacity > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "OTel Collector export queue nearly full"