砍掉 80% 年停机时间：可用性从 99.5% 到 99.9% 的 6 个工程决策与踩坑实录

0.995 × 0.995 × 0.995 = 0.9851 ≈ 98.51%

0.999^(1/10) ≈ 0.99990 → 99.99%

# 旧版 SLI：检查服务端口存活
- alert: ServiceDown
  expr: up{job="order-service"} == 0
  for: 1m

# 用户旅程：下单 → 支付 → 订单确认
# SLI 定义：用户视角的成功率

# SLI-1: 下单请求成功率（排除用户主动取消）
sli_order_create_success:
  expr: |
    sum(rate(http_requests_total{
      job="order-service",
      code!~"5..",
      path="/api/v1/orders",
      method="POST"
    }[5m]))
    /
    sum(rate(http_requests_total{
      job="order-service",
      path="/api/v1/orders",
      method="POST"
    }[5m]))    

# SLI-2: 下单请求延迟 P99 < 500ms
sli_order_create_latency_p99:
  expr: |
    histogram_quantile(0.99,
      sum(rate(http_request_duration_seconds_bucket{
        job="order-service",
        path="/api/v1/orders",
        method="POST"
      }[5m])) by (le))    

# SLI-3: 支付回调成功率
sli_payment_callback_success:
  expr: |
    sum(rate(payment_callback_total{status="success"}[5m]))
    /
    sum(rate(payment_callback_total[5m]))    

# SLO 配置：基于 SLI 的目标值
slo_targets:
  order_create_success:
    target: 0.999          # 月度成功率 ≥ 99.9%
    window: 28d            # 28 天滚动窗口
    burn_rate_alerts:
      - threshold: 14.4    # 1小时窗口，2%预算消耗
        window: 1h
      - threshold: 6.0     # 6小时窗口
        window: 6h

  order_create_latency_p99:
    target: 0.999           # 99.9% 的请求 P99 < 500ms
    threshold: 0.5          # 500ms
    window: 28d

  payment_callback_success:
    target: 0.9995          # 支付回调更高要求 99.95%
    window: 28d

# 排除用户主动取消的请求（HTTP 499）
sli_order_create_success_fixed:
  expr: |
    sum(rate(http_requests_total{
      job="order-service",
      code!~"5..|499",       # 排除 5xx 和 499（客户端主动断开）
      path="/api/v1/orders",
      method="POST"
    }[5m]))
    /
    sum(rate(http_requests_total{
      job="order-service",
      code!~"499",            # 分母也排除 499
      path="/api/v1/orders",
      method="POST"
    }[5m]))    

// 用户路由：根据 userID 分片到不同实例组
func GetShardGroup(userID int64) int {
    return int(userID % 10)
}

// 健康检查：自动剔除不健康的分片
func GetHealthyShardGroups() []int {
    var healthy []int
    for i := 0; i < 10; i++ {
        if shardHealth[i].IsHealthy() {
            healthy = append(healthy, i)
        }
    }
    // 如果超过 3 个分片不健康，触发降级
    if len(healthy) < 7 {
        triggerDegradation()
    }
    return healthy
}

SELECT * FROM orders WHERE user_id = ? ORDER BY created_at DESC LIMIT 20;

// 跨分片查询
func QueryUserOrders(userID int64) ([]Order, error) {
    shardGroup := GetShardGroup(userID)
    
    // 先查主分片
    primaryOrders, err := shardDB[shardGroup].QueryOrders(userID)
    if err != nil {
        return nil, err
    }
    
    // 如果用户有迁移历史，可能需要查多个分片
    if userHasMigrationHistory(userID) {
        for _, shard := range getMigrationShards(userID) {
            orders, err := shardDB[shard].QueryOrders(userID)
            if err != nil {
                // 降级：部分分片查不到不报错，返回已有数据
                log.Printf("shard %d query failed: %v, degrading", shard, err)
                continue
            }
            primaryOrders = append(primaryOrders, orders...)
        }
    }
    
    // 排序并截断
    sort.Slice(primaryOrders, func(i, j int) bool {
        return primaryOrders[i].CreatedAt.After(primaryOrders[j].CreatedAt)
    })
    
    return primaryOrders, nil
}

# 错误预算驱动的变更策略
change_gate:
  # 当前错误预算消耗率
  current_burn_rate: 2.3   # 2.3 倍燃烧速率
  
  rules:
    # 预算充足（消耗 < 50%）：自由发布
    - condition: "burn_rate < 0.5 AND budget_remaining > 50%"
      action: "allow"
      description: "错误预算充足，正常发布"
    
    # 预算紧张（消耗 50%-80%）：需要 SRE 签字
    - condition: "burn_rate > 0.8 OR budget_remaining < 50%"
      action: "require_sre_approval"
      description: "错误预算紧张，需 SRE 确认风险"
    
    # 预社耗尽（消耗 > 80%）：冻结非紧急变更
    - condition: "burn_rate > 1.0 OR budget_remaining < 20%"
      action: "freeze"
      description: "错误预算即将耗尽，只允许紧急修复变更"
      exceptions:
        - "P0 故障修复"
        - "安全漏洞修复"

# 金丝雀发布策略
canary:
  stages:
    - name: "canary-5%"
      weight: 5
      duration: 10m
      success_criteria:
        error_rate: "< 0.1%"
        p99_latency: "< 500ms"
        rollback_on_failure: true
    
    - name: "canary-20%"
      weight: 20
      duration: 15m
      success_criteria:
        error_rate: "< 0.1%"
        p99_latency: "< 500ms"
      depends_on: "canary-5%"
    
    - name: "canary-50%"
      weight: 50
      duration: 20m
      success_criteria:
        error_rate: "< 0.1%"
        p99_latency: "< 500ms"
      depends_on: "canary-20%"
    
    - name: "full-rollout"
      weight: 100
      duration: 0
      depends_on: "canary-50%"

# 第一层：告警分级
alert_levels:
  P0:
    description: "核心服务不可用"
    notify: ["phone", "sms", "slack"]
    page: true
    examples:
      - "订单服务 5xx 错误率 > 1%"
      - "支付网关全部实例不可用"
  
  P1:
    description: "服务降级但可用"
    notify: ["slack"]
    page: false
    examples:
      - "P99 延迟 > 1s 但 < 3s"
      - "单实例不可用但有冗余"
  
  P2:
    description: "需关注但不紧急"
    notify: ["daily_report"]
    page: false
    examples:
      - "磁盘使用率 > 70%"
      - "慢查询数量增加"

# 第二层：告警聚合（Alertmanager 路由）
route:
  group_by: ["alertname", "cluster", "service"]
  group_wait: 30s          # 等 30 秒聚合同类告警
  group_interval: 5m       # 同组告警 5 分钟发一次
  repeat_interval: 4h      # 4 小时内不重复发送

  routes:
    - matchers: ["severity=critical"]
      receiver: "oncall-phone"
      group_wait: 0s        # P0 不等待，立即通知
      repeat_interval: 1h
    
    - matchers: ["severity=warning"]
      receiver: "oncall-slack"
      group_wait: 60s       # P1 等 1 分钟聚合
      repeat_interval: 4h

# 第三层：告警抑制
inhibit_rules:
  # 如果服务完全不可用，抑制该服务的子组件告警
  - source_matchers: ['alertname="ServiceUnavailable"', 'severity="critical"']
    target_matchers: ['service=~".+"']
    equal: ["service", "cluster"]
  
  # 如果集群不可达，抑制该集群的所有告警
  - source_matchers: ['alertname="ClusterDown"']
    target_matchers: ['cluster=~".+"']
    equal: ["cluster"]

Availability = MTBF / (MTBF + MTTR)

# 自愈决策引擎
self_healing:
  detection:
    # 检测信号来源
    sources:
      - health_check: "HTTP /health 接口"
      - metrics: "Prometheus 指标异常检测"
      - alert: "Alertmanager 告警触发"
    
    # 检测窗口：连续 3 次失败才触发
    failure_threshold: 3
    failure_window: 30s
  
  diagnosis:
    # 根因分类
    categories:
      - name: "OOM_Killed"
        detection: "container_oom_events > 0"
        action: "restart_pod"
      
      - name: "HighLatency"
        detection: "p99_latency > 2s AND cpu_usage < 80%"
        action: "scale_up"
        
      - name: "HighErrorRate"  
        detection: "error_rate > 1%"
        action: "rollback_last_change"
        
      - name: "DiskFull"
        detection: "disk_usage > 95%"
        action: "clean_logs_and_restart"
  
  remediation:
    # 自动恢复动作
    actions:
      restart_pod:
        command: "kubectl delete pod -n {namespace} {pod_name}"
        cooldown: 60s
        max_retries: 2
        
      scale_up:
        command: "kubectl scale deployment -n {namespace} {deployment} --replicas={current}+2"
        cooldown: 120s
        max_retries: 1
        
      rollback_last_change:
        command: "kubectl rollout undo deployment -n {namespace} {deployment}"
        cooldown: 300s
        max_retries: 1
        require_human_confirm: false  # P0 自动回滚
        
      clean_logs_and_restart:
        command: |
          kubectl exec -n {namespace} {pod_name} -- find /var/log -name "*.log" -mtime +1 -delete
          kubectl delete pod -n {namespace} {pod_name}          
        cooldown: 180s
        max_retries: 1

方案 A：N+1 冗余
  - N 个实例承载正常流量，1 个实例做热备
  - 优点：成本低，架构简单
  - 缺点：故障切换时容量骤降 1/N，高峰期可能雪崩
  - 适用：流量平稳的服务

方案 B：2N 冗余  
  - 双倍实例，主备各承载 50% 流量
  - 优点：故障切换零感知，容量不降
  - 缺点：成本翻倍
  - 适用：核心交易链路

方案 C：N+M 冗余
  - N 个实例承载流量，M 个实例弹性扩容
  - 优点：成本可控，弹性应对突发
  - 缺点：扩容有延迟（1-3 分钟），切换期间有短暂降级
  - 适用：流量有明显波峰波谷的服务

# 容量规划脚本（简化版）
import numpy as np

def calculate_capacity_needed(historical_qps, growth_rate, peak_multiplier=2.5):
    """
    基于历史 QPS 计算所需容量
    historical_qps: 过去 30 天每分钟 QPS 数组
    growth_rate: 月增长率（如 0.1 表示 10%）
    peak_multiplier: 峰值倍数（大促期间通常 2-3 倍）
    """
    # 当前峰值 QPS
    current_peak = np.percentile(historical_qps, 99)
    
    # 预测 3 个月后的峰值
    future_peak = current_peak * (1 + growth_rate) ** 3 * peak_multiplier
    
    # 单实例承载能力（压测数据）
    capacity_per_instance = 500  # QPS
    
    # 需要的实例数（加 20% 安全余量）
    instances_needed = int(np.ceil(future_peak / capacity_per_instance * 1.2))
    
    return {
        'current_peak_qps': int(current_peak),
        'predicted_peak_qps_3mo': int(future_peak),
        'instances_needed': instances_needed,
        'current_instances': len(historical_qps),  # 假设已知
    }

# HPA 配置：缩短扩容等待时间
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: order-service-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  minReplicas: 4
  maxReplicas: 20
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60    # 默认 300，改成 60
      policies:
        - type: Percent
          value: 100                      # 每次最多扩容 100%
          periodSeconds: 30
        - type: Pods
          value: 4                        # 或每次扩容 4 个 Pod
          periodSeconds: 30
      selectPolicy: Max
    scaleDown:
      stabilizationWindowSeconds: 300    # 缩容保持默认，避免抖动
      policies:
        - type: Percent
          value: 10
          periodSeconds: 60

// 改进后的自愈决策：数据库操作需人工确认
func shouldAutoHeal(alert Alert) bool {
    // 数据库相关告警不自动恢复
    if alert.Category == "database" {
        notifyHumanForConfirmation(alert)
        return false
    }
    
    // 检查数据一致性
    if alert.Category == "replication" {
        lag := checkReplicationLag(alert.Instance)
        if lag > 5*time.Second {
            log.Printf("replication lag %v too high, skip auto-heal", lag)
            return false
        }
    }
    
    return true
}

# 分层金丝雀：确保每个用户群体都被覆盖
canary_routing:
  strategy: "stratified"
  groups:
    - name: "small_order"      # 小额订单用户
      weight: 5                  # 金丝雀流量占比
      user_selector: "order_amount < 100"
    
    - name: "large_order"       # 大额订单用户
      weight: 5
      user_selector: "order_amount >= 100"
    
    - name: "new_user"           # 新用户
      weight: 5
      user_selector: "register_days < 7"
    
    - name: "vip_user"           # VIP 用户
      weight: 5
      user_selector: "user_level >= 5"