监控数据治理：从指标爆炸到精准可观测

# 低基数：3 条时间序列
http_requests_total{method="GET",status="200"} 
http_requests_total{method="POST",status="200"}
http_requests_total{method="GET",status="404"}

# 高基数炸弹：100万用户 = 100万条时间序列
http_requests_total{method="GET",status="200",user_id="12345"}
http_requests_total{method="GET",status="200",user_id="12346"}
...

# metric-catalog.yaml — 指标目录模板
metrics:
  - name: http_requests_total
    type: counter
    owner: platform-team
    purpose: "统计 HTTP 请求总量，用于 QPS 和错误率计算"
    labels:
      - name: method
        cardinality: low  # GET, POST, PUT, DELETE 等 ~10 个值
      - name: status
        cardinality: low  # 200, 404, 500 等 ~20 个值
      - name: handler
        cardinality: medium  # API 路由，约 50-200 个值
    retention: 90d
    status: active

  - name: http_request_duration_seconds
    type: histogram
    owner: platform-team
    purpose: "HTTP 请求延迟分布，用于 P50/P95/P99 计算"
    labels:
      - name: method
        cardinality: low
      - name: handler
        cardinality: medium
    retention: 90d
    status: active

  - name: go_goroutines
    type: gauge
    owner: platform-team
    purpose: "Go 运行时 goroutine 数量"
    labels: []
    retention: 30d
    status: active

<domain>_<subsystem>_<name>_<unit>

# prometheus.yml — 分级采集配置
global:
  scrape_interval: 15s      # 默认 15s
  evaluation_interval: 15s   # 告警规则评估间隔

scrape_configs:
  # 关键业务指标：15s 采集（支付成功率、核心 API 延迟）
  - job_name: 'payment-service'
    scrape_interval: 15s
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ['payment-svc:8080']

  # 基础设施指标：60s 采集（CPU、内存、磁盘）
  - job_name: 'node-exporter'
    scrape_interval: 60s
    static_configs:
      - targets: ['node-1:9100', 'node-2:9100']

  # 调试类指标：300s 或按需（GC 详情、线程栈）
  - job_name: 'debug-metrics'
    scrape_interval: 300s
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: 'go_gc_duration_seconds|go_memstats_*'
        action: keep
    static_configs:
      - targets: ['debug-svc:8080']

scrape_configs:
  - job_name: 'node-exporter'
    scrape_interval: 60s
    metric_relabel_configs:
      # 丢弃磁盘 IO 的细节指标，只保留汇总
      - source_labels: [__name__]
        regex: 'node_disk_.*_bytes_total'
        action: drop

      # 丢弃网络接口的详细统计（只保留 eth0）
      - source_labels: [__name__, 'device']
        regex: 'node_network_.*;(?!eth0).*'
        action: drop

      # 重命名过长的指标名
      - source_labels: [__name__]
        regex: 'node_(.+)'
        target_label: __name__
        replacement: 'node_$1'

metric_relabel_configs:
  # 删除高基数的 instance_ip 标签
  - action: label_drop
    regex: 'instance_ip'

# prometheus.yml — 启动参数配置
# --storage.tsdb.retention.time=15d    短期保留
# --storage.tsdb.retention.size=100GB 磁盘上限（先到先触发清理）

# 推荐的生产配置：时间 + 磁盘双重限制
# 启动命令：
# prometheus --storage.tsdb.retention.time=15d \
#            --storage.tsdb.retention.size=100GB \
#            --query.max-samples=50000000 \
#            --query.timeout=2m

# 找出基数最高的 10 个指标
topk(10, count by (__name__)({__name__=~".+"}))

# 找出基数超过 10000 的指标（需要外部脚本）
# Prometheus 本身不擅长做这种聚合分析，建议用以下 Python 脚本

#!/usr/bin/env python3
"""扫描 Prometheus 中的高基数指标"""

import requests
from collections import defaultdict

PROMETHEUS_URL = "http://localhost:9090"
CARDINALITY_THRESHOLD = 10000  # 单指标时序数阈值

def get_metric_cardinality():
    """获取每个指标的时序数量"""
    # 用 /api/v1/series 接口获取所有时间序列
    resp = requests.get(
        f"{PROMETHEUS_URL}/api/v1/series",
        params={"match[]": "{__name__=~'.+'}"},
        timeout=30
    )
    data = resp.json()["data"]

    cardinality = defaultdict(int)
    for series in data:
        metric_name = series["__name__"]
        cardinality[metric_name] += 1

    return cardinality

def main():
    cardinality = get_metric_cardinality()

    print("=" * 70)
    print(f"{'指标名':<50} {'时序数':>10}")
    print("=" * 70)

    # 按基数排序，输出 Top 20
    sorted_metrics = sorted(cardinality.items(), key=lambda x: x[1], reverse=True)
    for name, count in sorted_metrics[:20]:
        flag = " ⚠️" if count > CARDINALITY_THRESHOLD else ""
        print(f"{name:<50} {count:>10,}{flag}")

    print("=" * 70)
    total = sum(cardinality.values())
    high_card = sum(1 for c in cardinality.values() if c > CARDINALITY_THRESHOLD)
    print(f"总时序数: {total:,}")
    print(f"高基数指标数 (> {CARDINALITY_THRESHOLD:,}): {high_card}")

if __name__ == "__main__":
    main()

# 所有采集目标中失败的比例
count(up == 0) / count(up) * 100

# 按采集任务分组，找出失败最多的 job
count by (job) (up == 0)

# 采集耗时最长的目标
topk(10, scrape_duration_seconds)

# alerts.yml — 采集质量告警
groups:
  - name: scraping-quality
    rules:
      - alert: ScrapeTargetDown
        expr: up == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "采集目标 {{ $labels.instance }} 不可达"
          description: "Job {{ $labels.job }} 中的 {{ $labels.instance }} 已经 5 分钟无法采集"

      - alert: ScrapeSlow
        expr: scrape_duration_seconds > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "采集耗时过长: {{ $labels.instance }}"
          description: "采集耗时 {{ $value }}s，超过 10s 阈值"

      - alert: PrometheusHighCardinality
        # 这个需要通过 recording rule 预计算
        expr: prometheus_tsdb_head_series > 500000
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus 时序数量过高"
          description: "当前活跃时序数 {{ $value }}，超过 50 万阈值，需检查高基数指标"

# prometheus.yml — 开启查询日志
# 启动参数加 --query.log=/var/log/prometheus-query.log
# 之后用脚本分析查询日志，提取被查询过的指标名

#!/usr/bin/env python3
"""分析 Prometheus 查询日志，找出从未被查询的指标"""

import json
import re
from collections import defaultdict

def parse_query_log(log_file):
    """解析查询日志，提取被查询过的指标名"""
    queried_metrics = set()

    with open(log_file) as f:
        for line in f:
            try:
                entry = json.loads(line)
                query = entry.get("params", {}).get("query", "")
                # 提取查询中的指标名
                # 简单正则匹配，实际可能需要解析 PromQL AST
                metrics = re.findall(r'([a-zA-Z_:][a-zA-Z0-9_:]*)\s*(?:\{|$)', query)
                queried_metrics.update(metrics)
            except json.JSONDecodeError:
                continue

    return queried_metrics

def main():
    queried = parse_query_log("/var/log/prometheus-query.log")

    # 获取全量指标名（通过 API）
    import requests
    resp = requests.get("http://localhost:9090/api/v1/label/__name__/values")
    all_metrics = set(resp.json()["data"])

    # 差集：存在但从未被查询的指标
    unused = all_metrics - queried

    print(f"总指标数: {len(all_metrics)}")
    print(f"被查询过: {len(queried)}")
    print(f"从未查询: {len(unused)}")
    print(f"废弃率: {len(unused)/len(all_metrics)*100:.1f}%")

    if unused:
        print("\n废弃指标列表（前 50 个）:")
        for m in sorted(unused)[:50]:
            print(f"  {m}")

if __name__ == "__main__":
    main()

// dashboard 标签示例
{
  "title": "[平台] API 延迟监控",
  "tags": ["team:platform", "env:prod", "status:active"],
  "folderUid": "platform-dashboards"
}

# 使用 Hashmod 分片：将采集目标分散到多个 Prometheus 实例
# prometheus-shard-1.yml
scrape_configs:
  - job_name: 'k8s-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_uid]
        modulus: 3       # 总共 3 个分片
        target_label: __tmp_hashmod
      - source_labels: [__tmp_hashmod]
        regex: "0"       # 这个实例只采集 hashmod=0 的目标
        action: keep