新服务上线 2 小时就 OOM：生产就绪审查 7 维门禁替代人工审批的工程决策

# PRR 架构韧性检查规则
architecture_resilience:
  - id: ARC-001
    name: "副本数 >= 2"
    check: "kubectl get deploy {svc} -o jsonpath='{.spec.replicas}'"
    assert: ">= 2"
    severity: critical
    description: "生产环境服务至少 2 副本，单副本无冗余"
    
  - id: ARC-002
    name: "配置了 PDB"
    check: "kubectl get pdb {svc} -o name"
    assert: "not_empty"
    severity: critical
    description: "必须配置 PodDisruptionBudget，防止节点维护时 Pod 被全部驱逐"
    
  - id: ARC-003
    name: "拓扑分布约束"
    check: "kubectl get deploy {svc} -o jsonpath='{.spec.template.spec.topologySpreadConstraints}'"
    assert: "not_empty"
    severity: warning
    description: "建议配置 topologySpreadConstraints，避免副本集中在一个可用区"
    
  - id: ARC-004
    name: "反亲和性配置"
    check: "kubectl get deploy {svc} -o jsonpath='{.spec.template.spec.affinity.podAntiAffinity}'"
    assert: "not_empty"
    severity: warning
    description: "建议配置 podAntiAffinity，避免副本调度到同一节点"

import subprocess
import json

def check_resource_quota(service_name, namespace="prod"):
    """PRR 资源配额检查：requests/limits 合理性 + 历史利用率"""
    issues = []
    
    # 1. 检查是否配置了 requests 和 limits
    deploy_json = json.loads(
        subprocess.check_output(
            ["kubectl", "get", "deploy", service_name, 
             "-n", namespace, "-o", "json"]
        )
    )
    
    containers = deploy_json["spec"]["template"]["spec"]["containers"]
    for c in containers:
        resources = c.get("resources", {})
        requests = resources.get("requests", {})
        limits = resources.get("limits", {})
        
        # 必须同时配置 requests 和 limits
        if not requests.get("cpu") or not requests.get("memory"):
            issues.append({
                "id": "RES-001",
                "severity": "critical",
                "msg": f"容器 {c['name']} 缺少 cpu/memory requests"
            })
        if not limits.get("cpu") or not limits.get("memory"):
            issues.append({
                "id": "RES-002",
                "severity": "critical", 
                "msg": f"容器 {c['name']} 缺少 cpu/memory limits"
            })
        
        # 2. 检查 request/limit 比值是否合理
        # request 应在 limit 的 50%-80% 之间
        # request 过低导致调度到资源不足的节点
        # request 过高导致资源浪费
        if requests.get("memory") and limits.get("memory"):
            req_mem = parse_quantity(requests["memory"])
            lim_mem = parse_quantity(limits["memory"])
            ratio = req_mem / lim_mem
            if ratio < 0.5:
                issues.append({
                    "id": "RES-003",
                    "severity": "warning",
                    "msg": f"容器 {c['name']} memory request/limit 比值 "
                           f"{ratio:.0%}，低于 50% 建议值，"
                           f"可能导致节点超卖"
                })
    
    # 3. 查询 Prometheus 历史利用率
    # 如果实际使用量接近 limit，说明 limit 设低了
    utilization = query_prometheus_utilization(service_name)
    for container, stats in utilization.items():
        if stats["memory_p95"] > stats["memory_limit"] * 0.85:
            issues.append({
                "id": "RES-004",
                "severity": "critical",
                "msg": f"容器 {container} 内存 P95 使用量 "
                       f"{stats['memory_p95']:.0f}MB 超过 limit 的 85%，"
                       f"存在 OOM 风险（当前 limit: "
                       f"{stats['memory_limit']:.0f}MB）"
            })
    
    return issues

deployment_safety:
  - id: DEP-001
    name: "配置了滚动更新策略"
    check: "kubectl get deploy {svc} -o jsonpath='{.spec.strategy}'"
    assert: "type == 'RollingUpdate'"
    severity: critical
    
  - id: DEP-002
    name: "maxUnavailable <= 25%"
    check: |
      kubectl get deploy {svc} -o \
      jsonpath='{.spec.strategy.rollingUpdate.maxUnavailable}'      
    assert: "<= 25%"
    severity: warning
    description: "建议 maxUnavailable 设为 0 或低值，确保更新时服务可用"
    
  - id: DEP-003
    name: "配置了健康探针"
    check: |
      kubectl get deploy {svc} -o \
      jsonpath='{.spec.template.spec.containers[0].readinessProbe}'      
    assert: "not_empty"
    severity: critical
    description: "必须配置 readinessProbe，否则流量会被路由到未就绪的 Pod"
    
  - id: DEP-004
    name: "readinessProbe 和 livenessProbe 分离"
    check: |
      检查 readinessProbe.httpGet.path 
      与 livenessProbe.httpGet.path 是否不同      
    assert: "different_paths"
    severity: warning
    description: "就绪探针和存活探针应检查不同端点，避免存活探针误判"

def check_observability(service_name):
    """PRR 可观测性检查：四金信号 + 告警规则 + 链路追踪"""
    issues = []
    
    # 1. 检查四金信号指标是否存在
    golden_signals = {
        "latency": f'histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{{service="{service_name}"}}[5m]))',
        "traffic": f'rate(http_requests_total{{service="{service_name}"}}[5m])',
        "errors": f'rate(http_requests_total{{service="{service_name}",status=~"5.."}}[5m])',
        "saturation": f'container_cpu_usage_seconds_total{{pod=~"{service_name}-.*"}}'
    }
    
    for signal, query in golden_signals.items():
        result = query_prometheus(query)
        if not result:
            issues.append({
                "id": "OBS-001",
                "severity": "critical",
                "msg": f"缺少 {signal} 指标，四金信号不完整"
            })
    
    # 2. 检查告警规则是否存在
    alerts = query_alertmanager_rules(service_name)
    if not alerts:
        issues.append({
            "id": "OBS-002",
            "severity": "critical",
            "msg": "该服务没有任何告警规则，故障无法被自动发现"
        })
    
    # 3. 检查是否有 Runbook 链接
    # 告警必须关联 Runbook，否则值班人员不知道怎么处理
    for alert in alerts:
        if not alert.get("annotations", {}).get("runbook_url"):
            issues.append({
                "id": "OBS-003",
                "severity": "warning",
                "msg": f"告警 {alert['name']} 缺少 runbook_url 注解，"
                       f"值班人员无法快速找到处理指南"
            })
    
    # 4. 检查链路追踪是否启用
    traces = check_jaeger_service(service_name)
    if not traces:
        issues.append({
            "id": "OBS-004",
            "severity": "warning",
            "msg": "未接入链路追踪，跨服务调用链不可见"
        })
    
    return issues

recovery:
  - id: REC-001
    name: "数据库备份频率"
    check: "查询备份系统 API，确认最近一次成功备份时间"
    assert: "last_backup_age < 24h"
    severity: critical
    description: "数据库至少每天备份一次"
    
  - id: REC-002
    name: "备份恢复测试"
    check: "查询恢复测试记录，确认最近一次成功恢复测试"
    assert: "last_restore_test_age < 30d"
    severity: warning
    description: "建议每月至少做一次恢复演练，验证备份可用性"
    
  - id: REC-003
    name: "配置了优雅终止"
    check: |
      kubectl get deploy {svc} -o \
      jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}'      
    assert: ">= 30"
    severity: warning
    description: "建议 terminationGracePeriodSeconds >= 30，给服务足够时间优雅退出"
    
  - id: REC-004
    name: "preStop hook 配置"
    check: |
      kubectl get deploy {svc} -o \
      jsonpath='{.spec.template.spec.containers[0].lifecycle.preStop}'      
    assert: "not_empty"
    severity: warning
    description: "建议配置 preStop hook，确保 Pod 退出前从注册中心摘除"

def check_security_compliance(service_name, namespace="prod"):
    """PRR 安全合规检查"""
    issues = []
    
    # 1. NetworkPolicy 是否配置
    np = kubectl_get_networkpolicy(service_name, namespace)
    if not np:
        issues.append({
            "id": "SEC-001",
            "severity": "warning",
            "msg": "未配置 NetworkPolicy，Pod 间网络未隔离"
        })
    
    # 2. 镜像是否扫描
    image = get_deploy_image(service_name, namespace)
    scan_result = query_harbor_scan(image)
    if not scan_result or scan_result.get("scan_status") != "success":
        issues.append({
            "id": "SEC-002",
            "severity": "critical",
            "msg": f"镜像 {image} 未通过安全扫描"
        })
    elif scan_result.get("vulnerabilities", {}).get("critical", 0) > 0:
        issues.append({
            "id": "SEC-003",
            "severity": "critical",
            "msg": f"镜像 {image} 有 "
                   f"{scan_result['vulnerabilities']['critical']} 个严重漏洞"
        })
    
    # 3. ServiceAccount 权限检查
    sa = get_service_account(service_name, namespace)
    if sa == "default":
        issues.append({
            "id": "SEC-004",
            "severity": "critical",
            "msg": "使用 default ServiceAccount，应创建专用 SA 并绑定最小权限"
        })
    
    # 4. Pod Security Standards
    pod_security = check_pod_security(service_name, namespace)
    if pod_security.get("privileged"):
        issues.append({
            "id": "SEC-005",
            "severity": "critical",
            "msg": "容器以 privileged 模式运行，安全隔离被绕过"
        })
    
    return issues

ownership:
  - id: OWN-001
    name: "服务目录注册"
    check: "查询服务目录 API，确认服务已注册"
    assert: "registered"
    severity: critical
    description: "服务必须在服务目录中注册，包含 owner 信息"
    
  - id: OWN-002
    name: "Owner 明确"
    check: "查询服务目录，确认 owner 字段非空"
    assert: "owner_not_empty"
    severity: critical
    
  - id: OWN-003
    name: "Runbook 链接"
    check: "查询服务目录，确认 runbook_url 非空"
    assert: "runbook_not_empty"
    severity: critical
    description: "每条告警必须关联 Runbook（参考告警策略设计）"
    
  - id: OWN-004
    name: "On-Call 排班"
    check: "查询排班系统，确认服务已纳入排班"
    assert: "in_rotation"
    severity: critical
    description: "服务必须有明确的 On-Call 负责人"

package main

import (
    "encoding/json"
    "fmt"
    "os"
)

// Severity 检查项严重级别
type Severity string

const (
    Critical Severity = "critical"
    Warning  Severity = "warning"
    Info     Severity = "info"
)

// CheckResult 检查结果
type CheckResult struct {
    ID       string   `json:"id"`
    Name     string   `json:"name"`
    Severity Severity `json:"severity"`
    Passed   bool     `json:"passed"`
    Message  string   `json:"message"`
    Evidence string   `json:"evidence"`
}

// PRRGate 生产就绪审查门禁
type PRRGate struct {
    ServiceName string
    Namespace   string
    Checks      []Check
}

// Check 检查接口
type Check interface {
    ID() string
    Name() string
    Severity() Severity
    Execute(svc, ns string) (CheckResult, error)
}

// Run 执行所有检查项
func (g *PRRGate) Run() (*GateReport, error) {
    report := &GateReport{
        Service: g.ServiceName,
        Total:   len(g.Checks),
    }
    
    for _, check := range g.Checks {
        result, err := check.Execute(g.ServiceName, g.Namespace)
        if err != nil {
            result = CheckResult{
                ID:       check.ID(),
                Name:     check.Name(),
                Severity: check.Severity(),
                Passed:   false,
                Message:  fmt.Sprintf("检查执行失败: %v", err),
            }
        }
        report.Results = append(report.Results, result)
        
        if !result.Passed {
            if result.Severity == Critical {
                report.CriticalFail++
            } else {
                report.WarningFail++
            }
        }
    }
    
    // 门禁策略：critical 不通过则阻断发布
    report.GatePassed = report.CriticalFail == 0
    return report, nil
}

// GateReport 门禁报告
type GateReport struct {
    Service      string        `json:"service"`
    Total        int           `json:"total"`
    CriticalFail int           `json:"critical_fail"`
    WarningFail  int           `json:"warning_fail"`
    GatePassed   bool          `json:"gate_passed"`
    Results      []CheckResult `json:"results"`
}

func main() {
    gate := &PRRGate{
        ServiceName: os.Getenv("SERVICE_NAME"),
        Namespace:   os.Getenv("NAMESPACE"),
        Checks: registerChecks(),
    }
    
    report, err := gate.Run()
    if err != nil {
        fmt.Printf("PRR 门禁执行失败: %v\n", err)
        os.Exit(2)
    }
    
    output, _ := json.MarshalIndent(report, "", "  ")
    fmt.Println(string(output))
    
    if !report.GatePassed {
        fmt.Printf("\n❌ PRR 门禁未通过: %d 项 critical 检查失败\n",
            report.CriticalFail)
        os.Exit(1)
    }
    
    if report.WarningFail > 0 {
        fmt.Printf("\n⚠️ PRR 门禁通过，但有 %d 项 warning 需关注\n",
            report.WarningFail)
    } else {
        fmt.Println("\n✅ PRR 门禁全部通过")
    }
}

# GitLab CI / Jenkins Pipeline 中的 PRR Gate 位置
stages:
  - build
  - test
  - scan          # 安全扫描
  - prr-gate      # ← 生产就绪审查门禁
  - deploy-staging
  - integration-test
  - deploy-prod   # 只有 PRR 通过才到这步
  - verify

prr-gate:
  stage: prr-gate
  image: prr-gate:latest
  variables:
    SERVICE_NAME: $CI_PROJECT_NAME
    NAMESPACE: prod
  script:
    - /app/prr-gate
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: on_success
  artifacts:
    reports:
      json: prr-report.json