缓存命中率从 58% 拉到 94%：CDN 监控体系搭建与 5 个生产级排障决策

package main

import (
    "fmt"
    "net/http"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

// CDN 指标定义
var (
    // 缓存命中率指标，label 区分请求命中率和字节命中率
    cacheHitRate = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "cdn_cache_hit_rate",
            Help: "CDN cache hit rate, hit_type=request|byte",
        },
        []string{"domain", "hit_type"},
    )
    // 回源 QPS
    originQPS = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "cdn_origin_qps",
            Help: "CDN origin requests per second",
        },
        []string{"domain"},
    )
    // 回源带宽，单位 Mbps
    originBps = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "cdn_origin_bps",
            Help: "CDN origin bandwidth in Mbps",
        },
        []string{"domain"},
    )
    // 边缘节点 HTTP 状态码分布
    edgeStatus = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "cdn_edge_status_total",
            Help: "CDN edge response status codes",
        },
        []string{"domain", "code"},
    )
)

func init() {
    prometheus.MustRegister(cacheHitRate, originQPS, originBps, edgeStatus)
}

// 每分钟拉取一次 CDN 监控数据
// 注意：阿里云 CDN API 有 5 分钟数据延迟，拉取频率低于 5 分钟无意义
func collectCDNMetrics() {
    domains := getCDNDomains() // 从配置文件读取加速域名列表
    for _, domain := range domains {
        // 调用阿里云 CDN API 获取监控数据
        // 时间窗口取过去 5 分钟
        endTime := time.Now().UTC().Format("2006-01-02T15:04:00Z")
        startTime := time.Now().Add(-5 * time.Minute).UTC().Format("2006-01-02T15:04:00Z")

        // 请求命中率
        reqHitRate := callCDNAPI(domain, "RequestHitRate", startTime, endTime)
        cacheHitRate.WithLabelValues(domain, "request").Set(reqHitRate)

        // 字节命中率
        byteHitRate := callCDNAPI(domain, "TrafficHitRate", startTime, endTime)
        cacheHitRate.WithLabelValues(domain, "byte").Set(byteHitRate)

        // 回源 QPS 和带宽
        originQPSVal := callCDNAPI(domain, "OriginQPS", startTime, endTime)
        originQPS.WithLabelValues(domain).Set(originQPSVal)

        originBpsVal := callCDNAPI(domain, "OriginBps", startTime, endTime)
        originBps.WithLabelValues(domain).Set(originBpsVal)
    }
}

func main() {
    // 每 5 分钟采集一次，和 API 数据延迟对齐
    go func() {
        ticker := time.NewTicker(5 * time.Minute)
        for {
            select {
            case <-ticker.C:
                collectCDNMetrics()
            }
        }
    }()
    // 初始采集一次
    collectCDNMetrics()

    http.Handle("/metrics", promhttp.Handler())
    http.ListenAndServe(":9105", nil)
}

# prometheus.yml 拨测配置
scrape_configs:
  - job_name: 'cdn-probe'
    scrape_interval: 30s
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - https://cdn.example.com/health-check.html  # CDN 上的健康检查文件
        - https://www.example.com/                   # 源站直连对比
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

# Nginx 日志中标记 CDN 回源请求
# 通过 $http_x_forwarded_for 判断是否来自 CDN 节点
log_format cdn_origin '$status $body_bytes_sent $request_time '
                      '$http_x_forwarded_for $remote_addr '
                      '$upstream_cache_status $request_uri';

server {
    listen 443 ssl;
    server_name origin.example.com;

    # 只记录来自 CDN 的回源请求
    access_log /var/log/nginx/cdn_origin.log cdn_origin;

    location / {
        proxy_pass http://backend;
        # 给 CDN 回源响应加缓存状态头，方便排查
        add_header X-Origin-Cache-Status $upstream_cache_status;
    }
}

// 字节命中率趋势（过去 1 小时）
cdn_cache_hit_rate{hit_type="byte"}

// 回源带宽占比（回源带宽 / 总带宽）
cdn_origin_bps / (cdn_origin_bps + cdn_edge_bps) * 100

// 回源 QPS 突增检测（当前值 vs 过去 1 小时均值）
cdn_origin_qps > (avg_over_time(cdn_origin_qps[1h]) * 2)

// 5xx 错误率
rate(cdn_edge_status_total{code=~"5.."}[5m]) /
rate(cdn_edge_status_total[5m]) * 100

// 拨测响应时间 P95
histogram_quantile(0.95, rate(probe_duration_seconds_bucket[5m]))

groups:
  - name: cdn_alerts
    rules:
      # 字节命中率低于 80%，持续 10 分钟
      - alert: CDNByteHitRateLow
        expr: cdn_cache_hit_rate{hit_type="byte"} < 80
        for: 10m
        labels:
          severity: P1
          team: sre
        annotations:
          summary: "{{ $labels.domain }} 字节命中率低于 80%"
          description: "当前值 {{ $value }}%，请检查缓存策略和大文件回源情况"

      # 回源带宽超过阈值
      - alert: CDNOriginBandwidthHigh
        expr: cdn_origin_bps > 500
        for: 5m
        labels:
          severity: P1
          team: sre
        annotations:
          summary: "{{ $labels.domain }} 回源带宽超过 500Mbps"
          description: "当前值 {{ $value }}Mbps，检查缓存命中率和回源 URL 分布"

      # 5xx 错误率超标
      - alert: CDNHighErrorRate
        expr: |
          sum(rate(cdn_edge_status_total{code=~"5.."}[5m])) by (domain)
          / sum(rate(cdn_edge_status_total[5m])) by (domain) * 100 > 1          
        for: 5m
        labels:
          severity: P2
          team: sre
        annotations:
          summary: "{{ $labels.domain }} 5xx 错误率超过 1%"
          description: "当前值 {{ $value }}%，检查源站健康状态和边缘节点日志"

      # 拨测失败
      - alert: CDNProbeFailed
        expr: probe_success == 0
        for: 1m
        labels:
          severity: P0
          team: sre
        annotations:
          summary: "CDN 拨测失败: {{ $labels.instance }}"
          description: "CDN 域名不可达，立即排查"

# 检查源站返回的响应头
curl -v -o /dev/null "https://origin.example.com/static/doc.pdf" 2>&1 | grep -i "cache-control"

# 检查 CDN 边缘返回的响应头
curl -v -o /dev/null "https://cdn.example.com/static/doc.pdf" 2>&1 | grep -i "cache-control\|x-cache\|age"

# 源站 Nginx 按 Content-Type 统一缓存策略
location ~* \.(jpg|jpeg|png|gif|webp|svg)$ {
    add_header Cache-Control "public, max-age=604800";  # 7 天
}

location ~* \.(css|js)$ {
    add_header Cache-Control "public, max-age=2592000";  # 30 天
}

location ~* \.(pdf|mp4|zip)$ {
    add_header Cache-Control "public, max-age=2592000";  # 30 天
}

# API 路径不缓存
location /api/ {
    add_header Cache-Control "no-cache, no-store, must-revalidate";
}

# 提取回源请求中 URL，按路径前缀分组统计
awk '{print $7}' /var/log/nginx/cdn_origin.log \
  | sed 's/\?.*//' \
  | sort | uniq -c | sort -rn | head -20

#!/bin/bash
# 发布后批量预热脚本
# 从构建产物中提取需要预热的 URL 列表
DOMAIN="cdn.example.com"
BUILD_DIR="/data/releases/latest"

# 提取所有静态资源 URL
find "$BUILD_DIR" -type f \( -name "*.js" -o -name "*.css" -o -name "*.png" -o -name "*.jpg" \) \
  | sed "s|$BUILD_DIR|https://$DOMAIN|" \
  > /tmp/prefetch_urls.txt

# 调用预热 API（阿里云示例）
while IFS= read -r url; do
    # 每批 50 个 URL，避免 API 限流
    echo "$url" >> /tmp/batch.txt
    if [ $(wc -l < /tmp/batch.txt) -eq 50 ]; then
        aliyun cdn PushObjectCache --ObjectPaths "$(cat /tmp/batch.txt | tr '\n' ',')" --Area mainland
        > /tmp/batch.txt
        sleep 2  # API 限流保护
    fi
done < /tmp/prefetch_urls.txt

# 处理剩余 URL
if [ -s /tmp/batch.txt ]; then
    aliyun cdn PushObjectCache --ObjectPaths "$(cat /tmp/batch.txt | tr '\n' ',')" --Area mainland
fi

echo "预热完成，共 $(wc -l < /tmp/prefetch_urls.txt) 个 URL"

# 源站 Nginx 配置
http {
    # Keep-Alive 超时要比 CDN 侧长 30 秒以上
    # 否则 CDN 会复用已被源站关闭的连接，导致回源 5xx
    keepalive_timeout 90s;
    keepalive_requests 1000;
}

# 1. 按状态码分布统计
awk '{print $9}' cdn_log_20260924.txt | sort | uniq -c | sort -rn

# 2. 按 URL 路径（去掉查询参数）统计回源 Top 20
#    $request 字段格式: "GET /path?query HTTP/1.1"
grep "MISS" cdn_log_20260924.txt \
  | awk '{print $7}' \
  | sed 's/\?.*//' \
  | sort | uniq -c | sort -rn | head -20

# 3. 按小时统计回源量，定位突发回源时间段
grep "MISS" cdn_log_20260924.txt \
  | awk '{print $4}' \
  | cut -d: -f1-2 \
  | sort | uniq -c

# 4. 找回源响应慢的 URL（>2 秒）
awk '$10 > 2.0 {print $7, $10}' cdn_log_20260924.txt | sort -k2 -rn | head -20

# 5. 统计回源请求的 Referer 分布，排查是否有盗链
grep "MISS" cdn_log_20260924.txt \
  | awk '{print $11}' \
  | sort | uniq -c | sort -rn | head -10