Kubernetes 集群升级策略：从规划到零故障落地

版本发布 ──────────────────────────────────────── EOL
│ ←──────────── 约 14 个月 ──────────────────→ │
│                                                │
├── 首月：社区验证期                                │
├── 2-6月：主流采用期                               │
├── 7-12月：成熟稳定期                              │
└── 13-14月：仅安全补丁，进入 EOL 倒计时              │

控制平面版本: v1.36
    │
    ├── kubelet: 最高 v1.36，最低 v1.35 (低 1 个小版本)
    ├── kube-proxy: 与 kubelet 相同版本
    └── kubectl: 最高 v1.37，最低 v1.35 (±1 个小版本)

## 升级前评估清单

### 集群状态
- [ ] 当前版本号（kubeadm version + kubectl version）
- [ ] 目标版本号
- [ ] 集群节点数量与分布
- [ ] 运行的工作负载数量与类型
- [ ] 已有的自定义资源（CRD）列表

### 兼容性检查
- [ ] 目标版本的 API 弃用检查
- [ ] 第三方组件兼容性（CNI、CSI、Ingress Controller）
- [ ] 应用使用的 API 版本是否将被移除
- [ ] kubelet 版本偏差在允许范围内
- [ ] etcd 版本兼容性

### 备份与恢复
- [ ] etcd 数据完整备份
- [ ] 集群配置文件备份
- [ ] 证书过期时间检查
- [ ] 备份恢复演练已验证

### 资源与容量
- [ ] 节点资源余量充足（至少 20%）
- [ ] PodDisruptionBudget 配置正确
- [ ] 多副本部署确认
- [ ] Pod 反亲和规则确认

### 运维保障
- [ ] 升级窗口已确定
- [ ] 回滚方案已准备
- [ ] 监控告警已配置
- [ ] 应急联系人已就位

# 方法1：使用 deprecation 命令（kubeadm 自带）
# 检查当前集群中即将被移除的 API 资源
kubectl get --raw /metrics | grep apiserver_requested_deprecated_apis

# 方法2：使用 kube-no-trouble 工具
# 安装 kube-no-trouble（kubent）
# GitHub: https://github.com/doitintl/kube-no-trouble

# 下载并运行
wget https://github.com/doitintl/kube-no-trouble/releases/latest/download/kubent-linux-amd64.tar.gz
tar xzf kubent-linux-amd64.tar.gz
./kubent

# 输出示例：
# [ERROR] Deployment "my-app" in namespace "default": uses deprecated API v1beta1
# [ERROR] ClusterRole "my-role" in namespace "": uses deprecated API v1beta1
# 建议在升级前将所有 deprecated API 迁移到新版本

# 方法3：直接查询 API server
# 查看即将移除的 API（目标版本）
kubectl get --raw='/api/v1' | jq '.resources[] | select(.deprecationWarning != null) | {name, deprecationWarning}'

#!/bin/bash
# etcd_backup.sh - 生产级 etcd 备份脚本

ETCD_ENDPOINTS="https://127.0.0.1:2379"
ETCD_CERT="/etc/etcd/pki/etcd-peer.crt"
ETCD_KEY="/etc/etcd/pki/etcd-peer.key"
ETCD_CACERT="/etc/etcd/pki/ca.crt"
BACKUP_DIR="/var/backups/etcd"
DATE=$(date +%Y%m%d_%H%M%S)

mkdir -p "${BACKUP_DIR}"

# 执行快照
ETCDCTL_API=3 etcdctl snapshot save "${BACKUP_DIR}/etcd-snapshot-${DATE}.db" \
    --endpoints="${ETCD_ENDPOINTS}" \
    --cert="${ETCD_CERT}" \
    --key="${ETCD_KEY}" \
    --cacert="${ETCD_CACERT}"

# 验证快照完整性
ETCDCTL_API=3 etcdctl snapshot status "${BACKUP_DIR}/etcd-snapshot-${DATE}.db" \
    --write-out=table

# 清理 7 天前的旧备份
find "${BACKUP_DIR}" -name "etcd-snapshot-*.db" -mtime +7 -delete

echo "Backup completed: etcd-snapshot-${DATE}.db"

# 检查所有 K8s 证书过期时间
sudo kubeadm certs check-expiration

# 输出示例：
# CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY
# admin.conf                 Jul 11, 2027 10:00 UTC   364d            ca
# apiserver                 Jul 11, 2027 10:00 UTC   364d            ca
# apiserver-etcd-client      Jul 11, 2027 10:00 UTC   364d            etcd-ca
# ...

# 如果证书即将过期（<90 天），先续期
sudo kubeadm certs renew all

# 重启相关组件
sudo systemctl restart kube-apiserver kube-controller-manager kube-scheduler

# ============================================
# 步骤1：升级 kubeadm
# ============================================

# 添加 Kubernetes APT 仓库（如果使用 APT）
sudo apt-mark unhold kubeadm
sudo apt-get update && sudo apt-get install -y kubeadm=1.36.0-00
sudo apt-mark hold kubeadm

# 验证版本
kubeadm version
# kubeadm version: &version.Info{Major:"1", Minor:"36", ...}

# ============================================
# 步骤2：升级计划检查
# ============================================

# 检查升级计划
sudo kubeadm upgrade plan

# 输出示例：
# Components that must be upgraded manually after you have upgraded the control plane:
# COMPONENT   CURRENT    TARGET
# kubelet     v1.35.0    v1.36.0
#
# Upgrade to the latest version in the v1.36 series:
# COMPONENT                 CURRENT    TARGET
# kube-apiserver            v1.35.0    v1.36.0
# kube-controller-manager   v1.35.0    v1.36.0
# kube-scheduler            v1.35.0    v1.36.0
# kube-proxy                v1.35.0    v1.36.0
# etcd                      3.5.16     3.7.0

# ============================================
# 步骤3：应用升级
# ============================================

# 在第一个控制平面节点执行
sudo kubeadm upgrade apply v1.36.0

# kubeadm 会自动执行以下操作：
# 1. 预检查集群状态
# 2. 升级 etcd（如有需要）
# 3. 逐个升级控制平面组件（API server, controller manager, scheduler）
# 4. 更新 kube-proxy ConfigMap
# 5. 更新 kubelet 配置

# 在其他控制平面节点执行（不需要版本号）
sudo kubeadm upgrade node

# ============================================
# 步骤4：升级 CNI 插件
# ============================================

# CNI 插件需要手动升级！kubeadm 不会管理 CNI
# 检查当前 CNI 版本
kubectl get pods -n kube-system -l k8s-app=calico-node -o yaml | grep image

# 升级 Calico
kubectl apply -f https://docs.projectcalico.org/manifests/calico-v3.28.yaml
# 或升级 Cilium
helm upgrade cilium cilium/cilium --namespace kube-system \
    --version 1.16.0 --reuse-values

# ============================================
# 步骤5：升级 kubelet（所有节点）
# ============================================

# 在每个节点上执行
sudo apt-mark unhold kubelet kubectl
sudo apt-get update && sudo apt-get install -y \
    kubelet=1.36.0-00 kubectl=1.36.0-00
sudo apt-mark hold kubelet kubectl

# 重启 kubelet
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# 验证节点状态
kubectl get nodes
# NAME          STATUS   ROLES           AGE    VERSION
# master-01     Ready    control-plane   365d   v1.36.0
# worker-01     Ready    <none>          365d   v1.35.0  <- 尚未升级
# worker-02     Ready    <none>          365d   v1.35.0

# ============================================
# 步骤6：排空并升级工作节点（逐个执行）
# ============================================

# 排空节点（将 Pod 迁移到其他节点）
kubectl drain worker-01 --ignore-daemonsets --delete-emptydir-data

# 升级 kubeadm（已升级则跳过）
# sudo apt-get install -y kubeadm=1.36.0-00

# 在工作节点上执行升级
sudo kubeadm upgrade node

# 升级 kubelet
sudo apt-get install -y kubelet=1.36.0-00 kubectl=1.36.0-00
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# 恢复调度
kubectl uncordon worker-01

# 验证
kubectl get nodes

# 验证所有节点版本一致
kubectl get nodes -o wide

# 验证核心组件状态
kubectl get pods -n kube-system -o wide

# 验证组件版本
kubectl version --short
# Client Version: v1.36.0
# Server Version: v1.36.0

# 检查集群事件
kubectl get events -n kube-system --sort-by='.lastTimestamp' | tail -20

# 验证关键工作负载
kubectl get pods --all-namespaces --field-selector=status.phase!=Running

                    ┌───────────────┐
                    │  Global LB    │
                    │  / DNS        │
                    └───────┬───────┘
                            │
              ┌─────────────┴──────────────┐
              │                              │
    ┌─────────▼─────────┐       ┌──────────▼──────────┐
    │  Blue Cluster      │       │  Green Cluster      │
    │  (v1.35 - Current)  │       │  (v1.36 - Target)   │
    │                     │       │                      │
    │  ┌───┐  ┌───┐      │       │  ┌───┐  ┌───┐      │
    │  │CP1│  │CP2│      │       │  │CP1│  │CP2│      │
    │  └───┘  └───┘      │       │  └───┘  └───┘      │
    │  ┌───┐  ┌───┐      │       │  ┌───┐  ┌───┐      │
    │  │W1 │  │W2 │      │       │  │W1 │  │W2 │      │
    │  └───┘  └───┘      │       │  └───┘  └───┘      │
    └─────────────────────┘       └──────────────────────┘

    流量: 100% ─────────────────── 流量: 0%
                            ↑
                    切换流量后: 0%     切换后: 100%

#!/bin/bash
# blue_green_upgrade.sh - 蓝绿升级流程

set -euo pipefail

OLD_CLUSTER="prod-blue"
NEW_CLUSTER="prod-green"
LB_ENDPOINT="lb.example.com"

# ============================================
# 阶段1：部署新集群（Green）
# ============================================

echo "=== Phase 1: Deploy Green Cluster ==="

# 使用 Terraform 或 Ansible 部署新集群
# 新集群使用目标版本 v1.36
terraform apply -var="cluster_name=${NEW_CLUSTER}" \
                -var="k8s_version=v1.36.0" \
                -auto-approve

# 等待集群就绪
echo "Waiting for cluster to be ready..."
until kubectl --context=${NEW_CLUSTER} get nodes | grep -c "Ready" | grep -q "3"; do
    echo "  Waiting... $(date)"
    sleep 30
done

# ============================================
# 阶段2：部署工作负载
# ============================================

echo "=== Phase 2: Deploy Workloads to Green ==="

# 应用所有 manifests
kubectl --context=${NEW_CLUSTER} apply -f manifests/namespaces/
kubectl --context=${NEW_CLUSTER} apply -f manifests/crds/
kubectl --context=${NEW_CLUSTER} apply -f manifests/deployments/
kubectl --context=${NEW_CLUSTER} apply -f manifests/services/
kubectl --context=${NEW_CLUSTER} apply -f manifests/ingress/

# 等待所有 Deployment 就绪
kubectl --context=${NEW_CLUSTER} wait --for=condition=available \
    --all --timeout=600s deployment -A

# ============================================
# 阶段3：数据同步
# ============================================

echo "=== Phase 3: Data Migration ==="

# 对于有状态服务，需要同步数据
# 例如：使用 Velero 迁移或数据库复制

# Velero 迁移示例
velero backup create pre-upgrade-backup \
    --include-namespaces app,monitoring,logging \
    --kube-context=${OLD_CLUSTER}

velero restore create --from-backup pre-upgrade-backup \
    --kube-context=${NEW_CLUSTER}

# ============================================
# 阶段4：金丝雀验证（10%流量）
# ============================================

echo "=== Phase 4: Canary - 10% traffic to Green ==="

# 使用 Weighted DNS 或 Service Mesh 路由 10% 流量到新集群
# Istio VirtualService 示例
cat <<EOF | kubectl apply -f -
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: app-routing
  namespace: istio-system
spec:
  gateways:
  - mesh
  - gateway
  hosts:
  - "app.example.com"
  http:
  - route:
    - destination:
        host: app.blue.svc.cluster.local
        port:
          number: 80
      weight: 90
    - destination:
        host: app.green.svc.cluster.local
        port:
          number: 80
      weight: 10
EOF

# 监控 15 分钟
echo "Monitoring for 15 minutes..."
sleep 900

# 检查错误率
ERROR_RATE=$(curl -s "http://prometheus:9090/api/v1/query?query=rate(http_requests_total{cluster=\"green\",code=~\"5..\"}[5m])/rate(http_requests_total{cluster=\"green\"}[5m])" | jq -r '.data.result[0].value[1]')

if (( $(echo "${ERROR_RATE} > 0.01" | bc -l) )); then
    echo "ERROR: Error rate too high (${ERROR_RATE}), rolling back!"
    exit 1
fi

# ============================================
# 阶段5：全量切换
# ============================================

echo "=== Phase 5: Full cutover - 100% to Green ==="

# 切换全部流量到新集群
cat <<EOF | kubectl apply -f -
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: app-routing
  namespace: istio-system
spec:
  gateways:
  - mesh
  - gateway
  hosts:
  - "app.example.com"
  http:
  - route:
    - destination:
        host: app.green.svc.cluster.local
        port:
          number: 80
      weight: 100
EOF

# 更新 DNS 记录（如果使用 DNS 路由）
# nsupdate -k /etc/bind/Kexample.com.key << EOF
# server dns.example.com
# zone example.com
# update delete app.example.com A
# update add app.example.com 300 A ${NEW_CLUSTER_IP}
# send
# EOF

echo "=== Upgrade complete. Old cluster (Blue) kept for 72h rollback window ==="

集群: 100 个工作节点

升级批次:
├── 批次1: 3 个节点（3%）—— 最小验证集
│   └── 观察 30 分钟，验证核心功能
├── 批次2: 7 个节点（10%）—— 扩大验证
│   └── 观察 30 分钟，验证扩缩容
├── 批次3: 20 个节点（30%）—— 大规模验证
│   └── 观察 1 小时，验证性能指标
├── 批次4: 30 个节点（60%）—— 主要流量
│   └── 观察 1 小时
└── 批次5: 40 个节点（100%）—— 全量完成
    └── 最终验证

#!/bin/bash
# canary_node_upgrade.sh - 金丝雀节点升级

set -euo pipefail

TARGET_VERSION="1.36.0-00"
BATCH_SIZES=(3 7 20 30 40)
OBSERVE_TIME_SEC=1800  # 30 分钟观察期

# 获取待升级节点列表（按批次）
get_nodes_for_batch() {
    local batch_num=$1
    local batch_size=${BATCH_SIZES[$((batch_num - 1))]}
    kubectl get nodes -l '!node-role.kubernetes.io/control-plane' \
        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \
        | grep "v1.35" \
        | head -n "${batch_size}"
}

# 升级单个节点
upgrade_node() {
    local node=$1
    echo "  Upgrading node: ${node}"

    # 1. 排空节点
    kubectl drain "${node}" --ignore-daemonsets --delete-emptydir-data \
        --grace-period=30 --timeout=300s

    # 2. SSH 到节点执行升级
    ssh "${node}" << EOF
        sudo apt-mark unhold kubeadm kubelet kubectl
        sudo apt-get update
        sudo apt-get install -y kubeadm=${TARGET_VERSION} kubelet=${TARGET_VERSION} kubectl=${TARGET_VERSION}
        sudo apt-mark hold kubeadm kubelet kubectl
        sudo kubeadm upgrade node
        sudo systemctl daemon-reload
        sudo systemctl restart kubelet
EOF

    # 3. 恢复调度
    kubectl uncordon "${node}"

    # 4. 等待节点 Ready
    kubectl wait --for=condition=Ready "node/${node}" --timeout=300s
}

# 健康检查
health_check() {
    echo "  Running health checks..."

    # 检查节点状态
    local not_ready=$(kubectl get nodes | grep -v "Ready" | wc -l)
    if [ "${not_ready}" -gt 0 ]; then
        echo "  FAIL: ${not_ready} nodes not ready"
        return 1
    fi

    # 检查 Pod 重启率
    local restarts=$(kubectl get pods -A -o json \
        | jq '[.items[] | select(.status.containerStatuses != null) | .status.containerStatuses[] | select(.restartCount > 5)] | length')
    if [ "${restarts}" -gt 3 ]; then
        echo "  FAIL: ${restarts} pods with excessive restarts"
        return 1
    fi

    # 检查 CrashLoopBackOff
    local crashes=$(kubectl get pods -A --field-selector=status.phase=Running \
        -o json | jq '[.items[] | select(.status.containerStatuses[]? | .state.waiting.reason == "CrashLoopBackOff")] | length')
    if [ "${crashes}" -gt 0 ]; then
        echo "  FAIL: ${crashes} pods in CrashLoopBackOff"
        return 1
    fi

    echo "  Health check PASSED"
    return 0
}

# 主循环
for batch_idx in "${!BATCH_SIZES[@]}"; do
    batch_num=$((batch_idx + 1))
    echo "============================================"
    echo "Batch ${batch_num}/${#BATCH_SIZES[@]}"
    echo "============================================"

    nodes=$(get_nodes_for_batch "${batch_num}")
    if [ -z "${nodes}" ]; then
        echo "No more nodes to upgrade"
        break
    fi

    for node in ${nodes}; do
        upgrade_node "${node}"
    done

    echo "Waiting ${OBSERVE_TIME_SEC}s for observation..."
    sleep "${OBSERVE_TIME_SEC}"

    if ! health_check; then
        echo "BATCH ${batch_num} FAILED! Stopping upgrade."
        echo "Already upgraded nodes remain on v1.36, pending investigation."
        exit 1
    fi
done

echo "All batches completed successfully!"
kubectl get nodes -o wide

# EKS 集群升级流程

# 1. 检查可用版本
aws eks describe-cluster --name prod-cluster \
    --query 'cluster.version' --output text
# 1.35

aws eks describe-addon-versions \
    --kubernetes-version 1.36 \
    --query 'addons[].addonName' --output text

# 2. 升级控制平面
aws eks update-cluster-version \
    --name prod-cluster \
    --version 1.36

# 3. 等待升级完成
aws eks wait cluster-active --name prod-cluster

# 4. 升级节点组
aws eks update-nodegroup-version \
    --cluster-name prod-cluster \
    --nodegroup-name prod-ng-1 \
    --kubernetes-version 1.36

# 5. 升级 managed node group 的 AMI
aws eks update-nodegroup-version \
    --cluster-name prod-cluster \
    --nodegroup-name prod-ng-1 \
    --release-version 1.36.0-20260711

# 6. 监控升级进度
aws eks describe-update \
    --name prod-cluster \
    --update-id <update-id>

# GKE 集群升级

# 1. 检查可用版本
gcloud container get-server-config

# 2. 升级控制平面
gcloud container clusters upgrade prod-cluster \
    --master --cluster-version 1.36 \
    --region asia-east1

# 3. 升级节点池
gcloud container clusters upgrade prod-cluster \
    --node-pool default-pool \
    --cluster-version 1.36 \
    --region asia-east1

# 4. 设置维护窗口（自动升级）
gcloud container clusters update prod-cluster \
    --maintenance-window-start=2026-07-11T02:00:00Z \
    --region asia-east1

# 5. 启用 surge upgrade（并行升级节点）
gcloud container clusters update prod-cluster \
    --max-surge-upgrade=3 \
    --max-unavailable-upgrade=1 \
    --region asia-east1

# AKS 集群升级

# 1. 检查可用版本
az aks get-upgrades \
    --resource-group prod-rg \
    --name prod-aks \
    --output table

# 2. 升级控制平面
az aks upgrade \
    --resource-group prod-rg \
    --name prod-aks \
    --kubernetes-version 1.36 \
    --no-wait

# 3. 查看升级状态
az aks show \
    --resource-group prod-rg \
    --name prod-aks \
    --query "provisioningState"

# 4. 升级节点池
az aks nodepool upgrade \
    --resource-group prod-rg \
    --cluster-name prod-aks \
    --name nodepool1 \
    --kubernetes-version 1.36

# 5. 设置节点自动升级通道
az aks update \
    --resource-group prod-rg \
    --name prod-aks \
    --auto-upgrade-channel node-image \
    --node-os-upgrade-channel SecurityPatch

# kubeadm 不支持自动回滚！
# 回滚需要手动降级组件

# 场景：从 v1.36 回滚到 v1.35

# 1. 降级 kubeadm
sudo apt-mark unhold kubeadm
sudo apt-get install -y kubeadm=1.35.0-00
sudo apt-mark hold kubeadm

# 2. 恢复 etcd 备份（关键步骤！）
sudo ETCDCTL_API=3 etcdctl snapshot restore \
    /var/backups/etcd/etcd-snapshot-pre-upgrade.db \
    --data-dir=/var/lib/etcd-restored

# 3. 替换 etcd 数据
sudo systemctl stop etcd
sudo mv /var/lib/etcd /var/lib/etcd-broken
sudo mv /var/lib/etcd-restored /var/lib/etcd
sudo chown -R etcd:etcd /var/lib/etcd
sudo systemctl start etcd

# 4. 降级控制平面组件
sudo apt-mark unhold kubelet kubectl
sudo apt-get install -y kubelet=1.35.0-00 kubectl=1.35.0-00
sudo apt-mark hold kubelet kubectl
sudo systemctl restart kubelet

# 5. 验证
kubectl get nodes
kubectl get pods -A

# 秒级回滚
cat <<EOF | kubectl apply -f -
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: app-routing
  namespace: istio-system
spec:
  gateways:
  - mesh
  - gateway
  hosts:
  - "app.example.com"
  http:
  - route:
    - destination:
        host: app.blue.svc.cluster.local
        port:
          number: 80
      weight: 100
    - destination:
        host: app.green.svc.cluster.local
        port:
          number: 80
      weight: 0
EOF

echo "Traffic switched back to Blue cluster"

# 使用 maintenance window 自动化升级
# 建议在业务低峰期执行

# 查看集群负载趋势（根据历史数据确定窗口）
# 典型窗口：凌晨 2:00-6:00（北京时间）

# 使用 kubeadm 配置维护窗口
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
metadata:
  name: config
kubernetesVersion: v1.36.0
---
# 节点升级时设置维护窗口
apiVersion: apps/v1
kind: MaintenanceWindow
metadata:
  name: upgrade-window
spec:
  schedule: "0 2 * * 6"  # 每周六凌晨 2 点
  duration: 4h

# 为关键应用设置 PDB
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-server-pdb
  namespace: production
spec:
  minAvailable: 2          # 至少保持 2 个副本可用
  # 或使用 maxUnavailable: 1
  selector:
    matchLabels:
      app: api-server
---
# 数据库类应用的严格 PDB
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: redis-pdb
  namespace: production
spec:
  maxUnavailable: 0          # 不允许任何副本不可用（使用滚动更新）
  selector:
    matchLabels:
      app: redis

# 升级期间重点关注的 Prometheus 指标
# 监控规则示例

groups:
- name: upgrade-monitoring
  rules:
  # 节点 NotReady 告警
  - alert: NodeNotReady
    expr: kube_node_status_condition{condition="Ready",status!="true"} == 1
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Node {{ $labels.node }} is not ready"

  # Pod 重启告警
  - alert: PodRestarting
    expr: increase(kube_pod_container_status_restarts_total[30m]) > 3
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "Pod {{ $labels.pod }} restarting frequently"

  # API Server 延迟告警
  - alert: APIServerLatencyHigh
    expr: histogram_quantile(0.99, rate(apiserver_request_duration_seconds_bucket[5m])) > 1
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "API server P99 latency > 1s"

  # 调度失败告警
  - alert: SchedulingFailed
    expr: kube_pod_status_unscheduled == 1
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "Pod {{ $labels.pod }} cannot be scheduled"

# argocd-app-cluster-config.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: cluster-config
  namespace: argocd
spec:
  source:
    repoURL: https://github.com/org/cluster-config
    targetRevision: release/v1.36
    path: manifests
  destination:
    server: https://kubernetes.default.svc
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
    - CreateNamespace=true
    - ApplyOutOfSyncOnly=true

# 检查 kubelet 状态
sudo systemctl status kubelet

# 检查 kubelet 日志
sudo journalctl -u kubelet --since "10 minutes ago" | tail -50

# 常见原因：
# 1. CNI 插件未升级导致网络不通
# 2. 容器运行时版本不兼容
# 3. 配置文件格式变更

# 修复 CNI
kubectl get pods -n kube-system -l k8s-app=calico-node
# 如果 CNI Pod 未运行，手动升级 CNI
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml

# CoreDNS 升级后 ConfigMap 格式可能变化
kubectl get configmap coredns -n kube-system -o yaml

# 检查 CoreDNS Pod
kubectl get pods -n kube-system -l k8s-app=kube-dns

# 如果 CoreDNS 不断重启，检查配置
kubectl logs -n kube-system -l k8s-app=kube-dns --tail=50

# 修复：更新 CoreDNS ConfigMap
kubectl edit configmap coredns -n kube-system
# 确保插件配置与 CoreDNS 版本兼容

# 如果升级后发现某些资源消失，可能是 API 被移除了

# 检查 API server 日志
sudo journalctl -u kube-apiserver | grep "deprecated"

# 查看被移除的 API
kubectl api-resources --verbs=list -o name | xargs -n1 kubectl get -A 2>&1 | grep "not found"

# 恢复方法：
# 1. 临时降级回旧版本
# 2. 将资源 YAML 中的 apiVersion 更新为新版本
# 3. 重新 apply
# 4. 再次升级

# Cluster API 升级示例
# 通过修改 MachineDeployment 的 version 字段触发自动升级

apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: prod-cluster
  namespace: default
spec:
  clusterNetwork:
    pods:
      cidrBlocks: ["10.244.0.0/16"]
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: KubeadmControlPlane
    name: prod-cluster-cp
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    kind: AWSCluster
    name: prod-cluster-infra
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
  name: prod-cluster-cp
  namespace: default
spec:
  version: v1.36.0   # 修改版本号即可触发升级
  replicas: 3
  machineTemplate:
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
      kind: AWSMachineTemplate
      name: prod-cluster-cp-template
  kubeadmConfigSpec:
    initConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          feature-gates: "..."
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
  name: prod-cluster-md-0
  namespace: default
spec:
  clusterName: prod-cluster
  replicas: 5
  template:
    spec:
      version: v1.36.0   # 工作节点也同步升级
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
          kind: KubeadmConfigTemplate
          name: prod-cluster-md-0
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
        kind: AWSMachineTemplate
        name: prod-cluster-md-0