APM 工具选型：从开源到商业的全维度实战指南

# docker-compose-skywalking.yml
version: '3.8'

services:
  oap:
    image: apache/skywalking-oap-server:10.1.0
    ports:
      - "11800:11800"  # gRPC 接收 Agent 数据
      - "12800:12800"  # HTTP REST API
    environment:
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
      SW_CORE_RECORD_DATA_TTL: 7        # 链路数据保留 7 天
      SW_CORE_METRICS_DATA_TTL: 30      # 指标数据保留 30 天
    depends_on:
      - elasticsearch
    restart: unless-stopped

  ui:
    image: apache/skywalking-ui:10.1.0
    ports:
      - "8080:8080"
    environment:
      SW_OAP_ADDRESS: http://oap:12800
    depends_on:
      - oap
    restart: unless-stopped

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
    volumes:
      - es-data:/usr/share/elasticsearch/data
    restart: unless-stopped

volumes:
  es-data:

# 启动 Java 应用时挂载 SkyWalking Agent
java -javaagent:/path/to/skywalking-agent.jar \
     -Dskywalking.agent.service_name=order-service \
     -Dskywalking.collector.backend_service=oap:11800 \
     -jar order-service.jar

package main

import (
    "context"
    "log"
    
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/trace"
)

// 初始化 tracer，将数据发送到 SkyWalking OAP
func initTracer() func() {
    // 实际生产中配置 OTLP exporter 指向 SkyWalking OAP 的 OTLP 接收端口
    // SkyWalking 9.x+ 原生支持 OTLP 协议
    tp, err := initOTLPProvider("oap:11800", "order-service")
    if err != nil {
        log.Fatal(err)
    }
    otel.SetTracerProvider(tp)
    return func() { tp.Shutdown(context.Background()) }
}

// 在 HTTP handler 中创建 span
func handleOrder(ctx context.Context, orderID string) error {
    ctx, span := tracer.Start(ctx, "handleOrder")
    defer span.End()
    
    span.SetAttributes(attribute.String("order.id", orderID))
    
    // 调用下游支付服务，trace context 自动透传
    if err := callPaymentService(ctx, orderID); err != nil {
        span.RecordError(err)
        return err
    }
    return nil
}

# Jaeger all-in-one 部署（仅测试用）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jaeger
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jaeger
  template:
    metadata:
      labels:
        app: jaeger
    spec:
      containers:
      - name: jaeger
        image: jaegertracing/all-in-one:1.60
        ports:
        - containerPort: 16686  # UI
        - containerPort: 4317    # OTLP gRPC
        - containerPort: 4318    # OTLP HTTP
        env:
        - name: COLLECTOR_OTLP_ENABLED
          value: "true"
        - name: SPAN_STORAGE_TYPE
          value: badger
        - name: BADGER_EPHEMERAL
          value: "false"
        - name: BADGER_DIRECTORY_VALUE
          value: /data/values
        - name: BADGER_DIRECTORY_KEY
          value: /data/keys
        volumeMounts:
        - name: data
          mountPath: /data
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: jaeger-pvc

# Tempo + MinIO 部署
server:
  http_listen_port: 3200

distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318

ingester:
  max_block_duration: 5m

compactor:
  compaction:
    block_retention: 48h

storage:
  trace:
    backend: s3
    s3:
      bucket: tempo-traces
      endpoint: minio:9000
      access_key: minioadmin
      secret_key: minioadmin
      insecure: true

┌─────────────────────────────────────────────────────────┐
│                    应用层（Application）                   │
│   OTel SDK 自动/手动埋点 → 生成 Span/Metric/Log           │
└────────────────────────┬────────────────────────────────┘
                         │ OTLP 协议（gRPC :4317 / HTTP :4318）
                         ▼
┌─────────────────────────────────────────────────────────┐
│                  采集层（Collector）                       │
│   接收 → 处理（过滤/采样/批处理）→ 导出                    │
└─────────┬───────────────────────┬───────────────────────┘
          │                       │
          ▼                       ▼
┌──────────────────┐   ┌──────────────────────────────────┐
│  Jaeger / Tempo   │   │  Prometheus / SkyWalking / ES   │
│  (Trace 后端)      │   │  (Metrics / 日志后端)             │
└──────────────────┘   └──────────────────────────────────┘

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  # 批处理，减少导出请求次数
  batch:
    timeout: 5s
    send_batch_size: 1024
  
  # 内存限制器，防止突发流量 OOM
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
  
  # 尾部采样：只保留慢请求和错误请求
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow
        type: latency
        latency:
          threshold_ms: 500
      - name: random_keep
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  # 发送到 Jaeger
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true
  
  # 发送到 Prometheus（指标）
  prometheus:
    endpoint: 0.0.0.0:8889
  
  # 发送到 Loki（日志）
  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, tail_sampling, batch]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

package main

import (
    "context"
    "log"
    "net/http"
    
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
    "go.opentelemetry.io/otel/trace"
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func initTracer(ctx context.Context, serviceName string) func() {
    // 创建 OTLP gRPC exporter，指向 Collector
    exporter, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint("otel-collector:4317"),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil {
        log.Fatalf("failed to create exporter: %v", err)
    }
    
    // 配置资源信息（服务名、实例 ID 等）
    res, _ := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName(serviceName),
            semconv.ServiceVersion("v1.2.0"),
            semconv.DeploymentEnvironment("production"),
        ),
    )
    
    // 创建 TracerProvider
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(res),
        // 头部采样：10% 采样率
        sdktrace.WithSampler(sdktrace.TraceIDRatioBased(0.1)),
    )
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.TraceContext{})
    
    return func() {
        _ = tp.Shutdown(ctx)
    }
}

func main() {
    ctx := context.Background()
    shutdown := initTracer(ctx, "api-gateway")
    defer shutdown()
    
    tracer := otel.Tracer("api-gateway")
    
    // 使用 otelhttp 自动埋点 HTTP 服务
    handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        ctx := r.Context()
        span := trace.SpanFromContext(ctx)
        span.SetAttributes(attribute.String("user.agent", r.UserAgent()))
        
        // 调用下游服务，TraceContext 自动透传
        callDownstream(ctx, "http://order-service:8080/api/orders")
        w.Write([]byte("OK"))
    })
    
    wrapped := otelhttp.NewHandler(handler, "api-gateway")
    http.ListenAndServe(":8080", wrapped)
}

func callDownstream(ctx context.Context, url string) {
    // otelhttp.NewClient 自动注入 Trace Header
    client := http.Client{
        Transport: otelhttp.NewTransport(http.DefaultTransport),
    }
    req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
    resp, err := client.Do(req)
    if err != nil {
        log.Printf("downstream call failed: %v", err)
        return
    }
    defer resp.Body.Close()
}

# 生产环境推荐采样配置
sampling:
  # 正常请求：头部采样 5-10%
  head_based:
    ratio: 0.05
  
  # 慢请求（> 500ms）：全量保留
  # 错误请求：全量保留
  # 通过 OTel Collector 尾部采样实现
  tail_based:
    policies:
      - type: status_code
        status_codes: [ERROR]
      - type: latency
        threshold_ms: 500
      - type: probabilistic
        sampling_percentage: 5

# Kubernetes 部署 OTel Collector（Deployment 模式）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector
spec:
  replicas: 3
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      containers:
      - name: collector
        image: otel/opentelemetry-collector-contrib:0.103.0
        ports:
        - containerPort: 4317  # OTLP gRPC
        - containerPort: 4318  # OTLP HTTP
        - containerPort: 8888  # Metrics
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: 2000m
            memory: 2Gi
        livenessProbe:
          httpGet:
            path: /health
            port: 13133
        readinessProbe:
          httpGet:
            path: /health
            port: 13133
---
apiVersion: v1
kind: Service
metadata:
  name: otel-collector
spec:
  selector:
    app: otel-collector
  ports:
  - name: otlp-grpc
    port: 4317
    targetPort: 4317
  - name: otlp-http
    port: 4318
    targetPort: 4318

// 好的做法：Span 里带上业务信息
func processOrder(ctx context.Context, order *Order) error {
    ctx, span := tracer.Start(ctx, "processOrder")
    defer span.End()
    
    // 关键业务属性
    span.SetAttributes(
        attribute.String("order.id", order.ID),
        attribute.String("order.user_id", order.UserID),
        attribute.Float64("order.amount", order.Amount),
        attribute.String("order.status", order.Status),
    )
    
    // 记录关键事件
    span.AddEvent("payment_initiated", trace.WithAttributes(
        attribute.String("payment.gateway", order.PaymentGateway),
    ))
    
    if err := validateOrder(ctx, order); err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return err
    }
    
    return nil
}

# PromQL：监控 OTel Collector 的数据丢弃率
rate(otelcol_processor_refused_spans_total[5m]) 
/ 
(rate(otelcol_receiver_accepted_spans_total[5m]) + rate(otelcol_processor_refused_spans_total[5m]))