切流量切到一半数据库连接池炸了：蓝绿与金丝雀发布的 6 个生产级决策与流量治理实录

# Blue 环境（当前生产）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-blue
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: blue
  template:
    metadata:
      labels:
        app: myapp
        version: blue
    spec:
      containers:
      - name: myapp
        image: myapp:v1
        ports:
        - containerPort: 8080
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
---
# Green 环境（新版本）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-green
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: green
  template:
    metadata:
      labels:
        app: myapp
        version: green
    spec:
      containers:
      - name: myapp
        image: myapp:v2
        ports:
        - containerPort: 8080
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
---
# Service 指向 Blue
apiVersion: v1
kind: Service
metadata:
  name: myapp-svc
spec:
  selector:
    app: myapp
    version: blue   # 切换到 green 即完成蓝绿切换
  ports:
  - port: 80
    targetPort: 8080

# CI 流水线中的连接池配置检查脚本（简化版）
import yaml
import sys

def check_pool_config(old_yaml, new_yaml):
    old = yaml.safe_load(old_yaml)
    new = yaml.safe_load(new_yaml)
    old_pool = old.get('spring', {}).get('datasource', {}).get('hikari', {}).get('maximum-pool-size', 'N/A')
    new_pool = new.get('spring', {}).get('datasource', {}).get('hikari', {}).get('maximum-pool-size', 'N/A')
    if old_pool != new_pool:
        print(f"WARNING: DB pool size changed from {old_pool} to {new_pool}")
        print(f"Blue-green deployment may cause connection overflow")
        sys.exit(1)
    print("OK: DB pool size unchanged")

# 稳定版 Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-stable
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-stable-svc
            port:
              number: 80
---
# 金丝雀 Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-canary
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "10"   # 10% 流量到金丝雀
spec:
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-canary-svc
            port:
              number: 80

┌──────────────────────────────────────────────────────┐
│                    Argo Rollouts Controller            │
│  (监听 Rollout CRD 变化，协调 ReplicaSet 和流量路由)     │
└──────────────┬───────────────────────┬───────────────┘
               │                       │
               ▼                       ▼
    ┌──────────────┐        ┌──────────────┐
    │  Stable RS   │        │  Canary RS   │
    │  (v1, 90%)   │        │  (v2, 10%)   │
    └──────┬───────┘        └──────┬───────┘
           │                       │
           ▼                       ▼
    ┌──────────────┐        ┌──────────────┐
    │ stable-svc   │        │ canary-svc   │
    └──────┬───────┘        └──────┬───────┘
           │                       │
           ▼                       ▼
    ┌────────────────────────────────────┐
    │       Istio VirtualService          │
    │  (按 weight 分配流量到两个 Service)    │
    └────────────────────────────────────┘

# 创建命名空间
kubectl create namespace argo-rollouts

# 安装控制器
kubectl apply -n argo-rollouts -f \
  https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml

# 安装 kubectl 插件
# 方式一：直接下载二进制
curl -sSL -o /usr/local/bin/kubectl-argo-rollouts \
  https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64
chmod +x /usr/local/bin/kubectl-argo-rollouts

# 方式二：用 Krew
kubectl krew install argo-rollouts

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-rollout
spec:
  replicas: 10
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - name: myapp
        image: myapp:v2
        ports:
        - name: http
          containerPort: 8080
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 512Mi
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          failureThreshold: 3
  strategy:
    canary:
      canaryService: myapp-canary      # 金丝雀版本 Service
      stableService: myapp-stable      # 稳定版本 Service
      trafficRouting:
        istio:
          virtualService:
            name: myapp-vsvc            # VirtualService 名称
            routes:
            - primary                   # 路由名称
      steps:
      - setWeight: 5                    # 5% 流量到金丝雀
      - pause: { duration: 5m }        # 等 5 分钟观察
      - analysis:                       # 自动分析指标
          templates:
          - templateName: success-rate  # 引用 AnalysisTemplate
      - setWeight: 20                   # 20% 流量
      - pause: { duration: 5m }
      - setWeight: 50                   # 50% 流量
      - pause: { duration: 10m }
      - analysis:
          templates:
          - templateName: success-rate
      - setWeight: 100                  # 全量切换

apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: myapp-vsvc
spec:
  gateways:
  - myapp-gateway
  http:
  - name: primary          # 必须和 Rollout 中的 routes 名一致
    route:
    - destination:
        host: myapp-stable
        port:
          number: 80
      weight: 100          # 初始 100% 到稳定版
    - destination:
        host: myapp-canary
        port:
          number: 80
      weight: 0             # 初始 0% 到金丝雀

apiVersion: v1
kind: Service
metadata:
  name: myapp-stable
spec:
  selector:
    app: myapp
  ports:
  - port: 80
    targetPort: 8080
---
apiVersion: v1
kind: Service
metadata:
  name: myapp-canary
spec:
  selector:
    app: myapp
  ports:
  - port: 80
    targetPort: 8080

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
spec:
  args:
  - name: service-name
  metrics:
  - name: success-rate
    interval: 30s
    count: 5                    # 采样 5 次
    successCondition: result[0] >= 0.99
    failureLimit: 2             # 允许失败 2 次，超过则中止
    provider:
      prometheus:
        address: http://prometheus-server:9090
        query: |
          sum(rate(http_requests_total{service="{{args.service-name}}",code!~"5.."}[2m]))
          /
          sum(rate(http_requests_total{service="{{args.service-name}}"}[2m]))          
  - name: p99-latency
    interval: 30s
    count: 5
    successCondition: result[0] < 200
    failureLimit: 2
    provider:
      prometheus:
        address: http://prometheus-server:9090
        query: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_ms_bucket{service="{{args.service-name}}"}[2m])) by (le))          

# 触发新版本发布（修改 image）
kubectl argo rollouts set image myapp-rollout myapp=myapp:v2

# 实时查看发布状态
kubectl argo rollouts get rollout myapp-rollout --watch

# 手动 promote（跳过当前 pause）
kubectl argo rollouts promote myapp-rollout

# 手动中止并回滚
kubectl argo rollouts abort myapp-rollout

# 查看发布历史
kubectl argo rollouts history myapp-rollout

-- 步骤1：扩展（发布前执行，v1 v2 共存期间安全）
ALTER TABLE orders ADD COLUMN payment_method_v2 VARCHAR(32) DEFAULT 'unknown';
CREATE INDEX CONCURRENTLY idx_orders_status_v2 ON orders(status, payment_method_v2);

-- 步骤2：v2 代码读写新列，v1 代码读写旧列，两者兼容
-- 应用层双写：v2 同时写旧列和新列

-- 步骤3：收缩（v1 下线后执行，通常在下一个发布周期）
ALTER TABLE orders DROP COLUMN payment_method;
DROP INDEX idx_orders_status;

-- 原表结构
CREATE TABLE orders (
    id BIGINT PRIMARY KEY,
    amount_cents INT,        -- 旧字段：金额以分为单位
    status VARCHAR(16)
);

-- 变更目标：amount_cents 改成 amount_yuan DECIMAL(10,2)
-- 步骤1：加新列
ALTER TABLE orders ADD COLUMN amount_yuan DECIMAL(10,2);
UPDATE orders SET amount_yuan = amount_cents / 100.0;

-- 步骤2：创建视图，旧列名映射到新列
CREATE OR REPLACE VIEW orders_legacy AS
SELECT id, amount_yuan * 100 AS amount_cents, status FROM orders;

-- v1 应用读 orders_legacy 视图，v2 应用读 orders 表
-- 两套应用共存期间，数据一致性由视图保证

// v2 代码：同时写新旧字段
func (r *OrderRepo) Create(ctx context.Context, order *Order) error {
    // 写新字段
    order.AmountYuan = order.AmountCents / 100.0
    return r.db.WithContext(ctx).Create(order).Error
}

// v2 代码：读时优先新字段，回退旧字段
func (r *OrderRepo) GetByID(ctx context.Context, id int64) (*Order, error) {
    var order Order
    if err := r.db.WithContext(ctx).First(&order, id).Error; err != nil {
        return nil, err
    }
    // 兼容逻辑：如果新字段为空，用旧字段
    if order.AmountYuan == 0 && order.AmountCents > 0 {
        order.AmountYuan = float64(order.AmountCents) / 100.0
    }
    return &order, nil
}

# feature_flags.yaml
use_new_payment_flow: false   # v2 部署后先关闭

if config.GetBool("use_new_payment_flow") {
    // 新逻辑
} else {
    // 旧逻辑
}

spec:
  metrics:
  - name: success-rate
    interval: 30s          # 每 30 秒查一次
    count: 10              # 共查 10 次（5 分钟窗口）
    successCondition: result[0] >= 0.99
    failureLimit: 3        # 连续失败 3 次触发回滚

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: production-gate
spec:
  args:
  - name: service-name
  metrics:
  # 指标1：错误率
  - name: error-rate
    interval: 30s
    count: 10
    successCondition: result[0] < 0.01    # 错误率 < 1%
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(rate(http_requests_total{service="{{args.service-name}}",code=~"5.."}[2m]))
          /
          sum(rate(http_requests_total{service="{{args.service-name}}"}[2m]))          

  # 指标2：P99 延迟
  - name: p99-latency
    interval: 30s
    count: 10
    successCondition: result[0] < 200     # P99 < 200ms
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_ms_bucket{service="{{args.service-name}}"}[2m])) by (le))          

  # 指标3：CPU 使用率（防止资源耗尽）
  - name: cpu-usage
    interval: 30s
    count: 10
    successCondition: result[0] < 0.85    # CPU < 85%
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(rate(container_cpu_usage_seconds_total{pod=~"{{args.service-name}}-.*"}[2m]))
          /
          sum(kube_pod_container_resource_limits{pod=~"{{args.service-name}}-.*",resource="cpu"})          

# 在 VirtualService 中配置流量镜像
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: myapp-vsvc
spec:
  http:
  - name: primary
    route:
    - destination:
        host: myapp-stable
        port:
          number: 80
      weight: 100
    mirror:
      host: myapp-canary          # 镜像流量到金丝雀
      port:
        number: 80
    mirrorPercentage:
      value: 100.0                # 镜像 100% 流量

spec:
  revisionHistoryLimit: 3   # 只保留 3 个历史版本

# 发布前预检脚本
#!/bin/bash
set -e

# 1. 检查 Prometheus 指标可用
SERVICE="myapp"
PROM_URL="http://prometheus:9090"

echo "=== 检查 Prometheus 指标 ==="
ERROR_RATE=$(curl -s "${PROM_URL}/api/v1/query?query=sum(rate(http_requests_total{service=\"${SERVICE}\",code=~\"5..\"}[2m]))/sum(rate(http_requests_total{service=\"${SERVICE}\"}[2m]))" | jq -r '.data.result[0].value[1]')
echo "当前错误率: ${ERROR_RATE}"
if [ "$(echo "${ERROR_RATE} > 0.01" | bc)" = 1 ]; then
  echo "ERROR: 基线错误率已超过 1%，不建议发布"
  exit 1
fi

# 2. 检查集群资源余量
echo "=== 检查集群资源 ==="
ALLOCATABLE=$(kubectl get nodes -o jsonpath='{.items[*].status.allocatable.cpu}' | awk '{sum+=$1} END {print sum}')
REQUESTED=$(kubectl get pods --all-namespaces -o jsonpath='{.items[*].spec.containers[*].resources.requests.cpu}' | awk '{sum+=$1} END {print sum}')
echo "CPU: ${REQUESTED}/${ALLOCATABLE} 已分配"
if [ "$(echo "${REQUESTED} > ${ALLOCATABLE} * 0.7" | bc)" = 1 ]; then
  echo "ERROR: 集群 CPU 分配率已超 70%，无法安全发布"
  exit 1
fi

# 3. 检查数据库连接数
echo "=== 检查数据库连接数 ==="
CURRENT_CONNS=$(kubectl exec -n data postgres-0 -- psql -U admin -c "SELECT count(*) FROM pg_stat_activity;" -t)
MAX_CONNS=$(kubectl exec -n data postgres-0 -- psql -U admin -c "SHOW max_connections;" -t)
echo "数据库连接: ${CURRENT_CONNS}/${MAX_CONNS}"
if [ "$(echo "${CURRENT_CONNS} > ${MAX_CONNS} * 0.7" | bc)" = 1 ]; then
  echo "ERROR: 数据库连接数已超 70%，蓝绿部署可能导致连接耗尽"
  exit 1
fi

echo "=== 预检通过 ==="

spec:
  progressDeadlineSeconds: 300   # 5 分钟无进展判定超时

apiVersion: v1
kind: Pod
metadata:
  annotations:
    proxy.istio.io/config: '{"holdApplicationUntilProxyStarts": true}'