API 网关监控实战：从 Kong 到 APISIX 的指标采集与告警设计

# 启用 Prometheus 插件（全局级别）
curl -X POST http://kong-admin:8001/plugins \
  --data "name=prometheus" \
  --data "config.per_consumer=true" \
  --data "config.status_code_metrics=true" \
  --data "config.latency_metrics=true" \
  --data "config.upstream_health_metrics=true"

# APISIX prometheus 插件配置
plugins:
  - prometheus

# 通过 Admin API 启用
# 全局路由级别
curl http://127.0.0.1:9180/apisix/admin/routes/1 \
  -H 'X-API-KEY: your-key' -X PUT -d '
{
  "uri": "/api/*",
  "plugins": {
    "prometheus": {
      "prefer_name": true
    }
  },
  "upstream": {
    "type": "roundrobin",
    "nodes": {
      "127.0.0.1:8080": 1
    }
  }
}'

# Envoy 配置 - 启用 Prometheus stats sink
stats_sinks:
  - name: envoy.stat_sinks.statsd
    typed_config:
      "@type": type.googleapis.com/envoy.config.metrics.v3.StatsdSink
      address:
        socket_address:
          address: 127.0.0.1
          port_value: 9125
      prefix: envoy.

# 或者用 Prometheus 格式（推荐）
admin:
  address:
    socket_address:
      address: 0.0.0.0
      port_value: 9901

stats_config:
  stats_tags:
    - tag_name: cluster_name
      regex: "^cluster\\.((.+?)\\.)"
  stats_matcher:
    inclusion_list:
      patterns:
        - prefix: "cluster."
        - prefix: "listener."
        - prefix: "server."
        - prefix: "http."

# prometheus.yml - Kong 采集配置
scrape_configs:
  - job_name: 'kong'
    metrics_path: /metrics
    static_configs:
      - targets:
          - 'kong-gateway:8001'  # Kong Admin API 端口
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        replacement: 'kong-prod-01'

# prometheus.yml - APISIX 采集配置
scrape_configs:
  - job_name: 'apisix'
    metrics_path: /apisix/prometheus/metrics
    static_configs:
      - targets:
          - 'apisix:9091'
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        replacement: 'apisix-prod-01'

# prometheus.yml - Envoy 采集配置
scrape_configs:
  - job_name: 'envoy'
    metrics_path: /stats/prometheus
    static_configs:
      - targets:
          - 'envoy-proxy:9901'

# Kubernetes PodMonitor（kube-prometheus-stack）
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: envoy-gateway
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: envoy-gateway
  podMetricsEndpoints:
    - port: admin
      path: /stats/prometheus
      interval: 15s

groups:
  - name: gateway-availability
    rules:
      # 网关进程存活
      - alert: GatewayDown
        expr: up{job=~"kong|apisix|envoy"} == 0
        for: 1m
        labels:
          severity: critical
          team: sre
        annotations:
          summary: "网关 {{ $labels.instance }} 不可达"
          description: "Prometheus 已无法采集 {{ $labels.instance }} 的指标，网关可能已宕机"

      # 网关 5xx 错误率（1分钟内超过 5%）
      - alert: GatewayHighErrorRate
        expr: |
          sum(rate(kong_http_requests_total{status=~"5.."}[1m])) by (service)
          /
          sum(rate(kong_http_requests_total[1m])) by (service)
          > 0.05          
        for: 2m
        labels:
          severity: critical
          team: sre
        annotations:
          summary: "Kong 网关 5xx 错误率超 5%"
          description: "服务 {{ $labels.service }} 的 5xx 错误率当前为 {{ $value | humanizePercentage }}"

  - name: gateway-latency
    rules:
      # Kong P99 延迟超过 500ms
      - alert: GatewayHighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(kong_latency_ms_bucket[5m])) by (service, le)
          ) > 500          
        for: 5m
        labels:
          severity: warning
          team: sre
        annotations:
          summary: "Kong 网关 P99 延迟超 500ms"
          description: "服务 {{ $labels.service }} 的 P99 延迟当前为 {{ $value }}ms"

      # 上游延迟占比过高（Kong 自身延迟 < 总延迟的 20% 说明瓶颈在上游）
      - alert: GatewayUpstreamLatencyDominant
        expr: |
          histogram_quantile(0.95,
            sum(rate(kong_latency_ms_bucket{type="upstream"}[5m])) by (service, le)
          )
          /
          histogram_quantile(0.95,
            sum(rate(kong_latency_ms_bucket{type="total"}[5m])) by (service, le)
          )
          > 0.8          
        for: 10m
        labels:
          severity: info
          team: sre
        annotations:
          summary: "上游延迟占比超 80%，瓶颈在后端服务"
          description: "服务 {{ $labels.service }} 的上游延迟占比 {{ $value | humanizePercentage }}，建议排查后端服务"

  - name: gateway-upstream-health
    rules:
      # 上游节点不健康
      - alert: GatewayUpstreamUnhealthy
        expr: kong_upstream_target_health{state="UNHEALTHY"} == 1
        for: 1m
        labels:
          severity: warning
          team: sre
        annotations:
          summary: "上游目标 {{ $labels.target }} 不健康"
          description: "上游 {{ $labels.upstream }} 的目标 {{ $labels.target }} 当前状态为 UNHEALTHY"

      # Envoy 集群断路器打开
      - alert: EnvoyCircuitBreakerOpen
        expr: envoy_cluster_circuit_breakers_default_cx_pool_open == 1
        for: 30s
        labels:
          severity: critical
          team: sre
        annotations:
          summary: "Envoy 集群 {{ $labels.cluster_name }} 连接池断路器已打开"
          description: "集群连接池已满，新请求被拒绝"

  - name: gateway-traffic-anomaly
    rules:
      # 请求量骤降（可能路由配置有问题或上游全部挂了）
      - alert: GatewayTrafficDrop
        expr: |
          sum(rate(kong_http_requests_total[5m])) by (service)
          <
          sum(rate(kong_http_requests_total[5m] offset 1h)) by (service) * 0.3          
        for: 10m
        labels:
          severity: warning
          team: sre
        annotations:
          summary: "服务 {{ $labels.service }} 流量下降超 70%"
          description: "当前请求量比 1 小时前下降 {{ $value }}"

      # 请求量突增（可能是被刷了或正常流量高峰）
      - alert: GatewayTrafficSpike
        expr: |
          sum(rate(kong_http_requests_total[5m])) by (service)
          >
          sum(rate(kong_http_requests_total[5m] offset 1h)) by (service) * 3          
        for: 5m
        labels:
          severity: info
          team: sre
        annotations:
          summary: "服务 {{ $labels.service }} 流量增长超 200%"
          description: "当前请求量比 1 小时前增长 {{ $value }} 倍，请确认是否为正常流量高峰"

  - name: gateway-resource
    rules:
      # Kong 数据面连接数过高
      - alert: KongHighConnections
        expr: |
          sum by (instance) (
            kong_nginx_http_connections_total{state="active"}
          ) > 10000          
        for: 5m
        labels:
          severity: warning
          team: sre
        annotations:
          summary: "Kong 实例 {{ $labels.instance }} 活跃连接数超 10000"
          description: "当前活跃连接数 {{ $value }}，可能需要扩容"

      # APISIX etcd 连接异常
      - alert: APISIXEtcdUnreachable
        expr: apisix_etcd_reachable == 0
        for: 30s
        labels:
          severity: critical
          team: sre
        annotations:
          summary: "APISIX 无法连接 etcd"
          description: "APISIX 节点 {{ $labels.instance }} 无法连接 etcd，配置变更将无法同步"

┌─────────────────────────────────────────────┐
│  第 1 行：QPS 总量、错误率、P99 延迟、活跃连接  │  ← 一眼看全局
├─────────────────────────────────────────────┤
│  第 2 行：各服务 QPS 对比 + 错误率对比          │  ← 谁有问题
├─────────────────────────────────────────────┤
│  第 3 行：延迟分布热力图 + 延迟分位数趋势       │  ← 延迟详情
├─────────────────────────────────────────────┤
│  第 4 行：上游健康状态 + 断路器状态 + 连接池    │  ← 底层状态
└─────────────────────────────────────────────┘

# 总 QPS（按网关实例聚合）
sum(rate(kong_http_requests_total[5m]))

# 错误率
sum(rate(kong_http_requests_total{status=~"5.."}[5m]))
/
sum(rate(kong_http_requests_total[5m]))

# P99 延迟
histogram_quantile(0.99,
  sum(rate(kong_latency_ms_bucket[5m])) by (le)
)

# 活跃连接数
sum(kong_nginx_http_connections_total{state="active"})

# 延迟热力图数据源
sum(rate(kong_latency_ms_bucket[5m])) by (le, service)

kong_upstream_target_health

# 按服务类型设置不同延迟阈值
- alert: GatewayHighLatencyOnline
  expr: |
    histogram_quantile(0.99,
      sum(rate(kong_latency_ms_bucket{service=~"order-.*|payment-.*"}[5m])) by (service, le)
    ) > 200
    and on(service)
    kong_latency_ms_count{service=~"order-.*|payment-.*"} > 0    
  for: 5m
  labels:
    severity: warning

- alert: GatewayHighLatencyBatch
  expr: |
    histogram_quantile(0.99,
      sum(rate(kong_latency_ms_bucket{service=~"report-.*|export-.*"}[5m])) by (service, le)
    ) > 5000
    and on(service)
    kong_latency_ms_count{service=~"report-.*|export-.*"} > 0    
  for: 10m
  labels:
    severity: warning

#!/usr/bin/env python3
"""API 网关健康巡检脚本
通过 Prometheus API 查询网关关键指标，输出巡检报告。
"""
import requests
import sys
from datetime import datetime

PROMETHEUS_URL = "http://prometheus:9090"

# 巡检项配置：指标名 -> 查询语句 -> 阈值 -> 告警级别
CHECKS = [
    {
        "name": "网关存活",
        "query": 'up{job=~"kong|apisix|envoy"}',
        "expect": 1,
        "severity": "P0"
    },
    {
        "name": "5xx 错误率",
        "query": """
            sum(rate(kong_http_requests_total{status=~"5.."}[5m]))
            / sum(rate(kong_http_requests_total[5m]))
        """,
        "threshold": 0.01,
        "severity": "P1",
        "compare": "gt"
    },
    {
        "name": "P99 延迟",
        "query": """
            histogram_quantile(0.99,
                sum(rate(kong_latency_ms_bucket[5m])) by (le)
            )
        """,
        "threshold": 500,
        "severity": "P2",
        "compare": "gt"
    },
    {
        "name": "上游不健康节点数",
        "query": 'count(kong_upstream_target_health{state="UNHEALTHY"} == 1)',
        "threshold": 0,
        "severity": "P2",
        "compare": "gt"
    },
    {
        "name": "活跃连接数",
        "query": 'sum(kong_nginx_http_connections_total{state="active"})',
        "threshold": 8000,
        "severity": "P3",
        "compare": "gt"
    },
]


def query_prometheus(query):
    """执行 PromQL 查询"""
    resp = requests.get(
        f"{PROMETHEUS_URL}/api/v1/query",
        params={"query": query},
        timeout=10
    )
    resp.raise_for_status()
    data = resp.json()
    if data["status"] != "success":
        raise ValueError(f"查询失败: {data}")
    return data["data"]["result"]


def run_checks():
    """执行所有巡检项"""
    print(f"\n{'='*60}")
    print(f"API 网关健康巡检 - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'='*60}\n")

    issues = []
    for check in CHECKS:
        try:
            results = query_prometheus(check["query"])
            if not results:
                print(f"[SKIP] {check['name']}: 无数据")
                continue

            if check.get("expect") is not None:
                value = float(results[0]["value"][1])
                status = "OK" if value == check["expect"] else "FAIL"
                print(f"[{status}] {check['name']}: {value}")
                if status == "FAIL":
                    issues.append((check["severity"], check["name"], value))
            elif check.get("threshold") is not None:
                value = float(results[0]["value"][1])
                if check.get("compare") == "gt":
                    exceeded = value > check["threshold"]
                else:
                    exceeded = value < check["threshold"]
                status = "OK" if not exceeded else "WARN"
                print(f"[{status}] {check['name']}: {value:.2f} (阈值: {check['threshold']})")
                if exceeded:
                    issues.append((check["severity"], check["name"], value))
        except Exception as e:
            print(f"[ERR] {check['name']}: {e}")
            issues.append(("P0", check["name"], str(e)))

    print(f"\n{'='*60}")
    if issues:
        print(f"发现 {len(issues)} 个问题:")
        for sev, name, val in issues:
            print(f"  [{sev}] {name}: {val}")
    else:
        print("所有巡检项正常")
    print(f"{'='*60}\n")

    return len(issues) == 0


if __name__ == "__main__":
    ok = run_checks()
    sys.exit(0 if ok else 1)