Kubernetes 自动扩缩容：HPA/VPA/CA 深度解析

1. 从指标 API 获取 Pod 的当前指标值（CPU/内存/自定义）
2. 计算期望副本数 = ceil(当前副本数 * (当前指标值 / 目标指标值))
3. 与当前副本数比较，决定扩容或缩容
4. 调用 Deployment/ReplicaSet 的 Scale API 修改副本数

期望副本数 = ceil(当前副本数 × (当前指标值 ÷ 目标指标值))

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: myapp-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myapp
  minReplicas: 3                    # 最小副本数
  maxReplicas: 50                   # 最大副本数
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization           # 利用率百分比
        averageUtilization: 60      # 目标CPU使用率60%
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 70      # 目标内存使用率70%

Pod 指标 → cAdvisor → kubelet → Metrics Server (resource metrics)
                                    ↓
应用指标 → Prometheus → Prometheus Adapter → K8s API (custom metrics)
                                    ↓
                                 HPA 查询

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install prometheus-adapter prometheus-community/prometheus-adapter \
  --namespace monitoring \
  --create-namespace \
  --set prometheus.url=http://prometheus-server.monitoring.svc.cluster.local \
  --set prometheus.port=80

# Prometheus Adapter 配置
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-adapter
  namespace: monitoring
data:
  config.yaml: |
    rules:
    - seriesQuery: 'http_requests_total{namespace!="",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^(.*)_total"
        as: "${1}_per_second"
      metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'    

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: myapp-hpa-custom
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myapp
  minReplicas: 3
  maxReplicas: 50
  metrics:
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1000"       # 每个Pod目标1000 QPS
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70

metrics:
- type: Resource
  resource:
    name: cpu
    target:
      type: Utilization
      averageUtilization: 60
- type: Resource
  resource:
    name: memory
    target:
      type: Utilization
      averageUtilization: 70
- type: Pods
  pods:
    metric:
      name: http_requests_per_second
    target:
      type: AverageValue
      averageValue: "1000"
# HPA 会分别计算每个指标的期望副本数，取最大值

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: myapp-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myapp
  minReplicas: 3
  maxReplicas: 50
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 60
  behavior:
    # 扩容策略
    scaleUp:
      stabilizationWindowSeconds: 0    # 扩容不需要稳定窗口，立即执行
      policies:
      - type: Percent
        value: 100                      # 每次最多扩容100%（翻倍）
        periodSeconds: 30               # 30秒内最多扩容一次
      - type: Pods
        value: 4                        # 或每次最多扩4个Pod
        periodSeconds: 30
      selectPolicy: Max                 # 取两个策略中的最大值
    # 缩容策略
    scaleDown:
      stabilizationWindowSeconds: 300   # 缩容需要5分钟稳定窗口
      policies:
      - type: Percent
        value: 10                       # 每次最多缩容10%
        periodSeconds: 60               # 60秒内最多缩容一次

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: myapp-vpa
  namespace: production
spec:
  targetRef:
    apiVersion: "apps/v1"
    kind: Deployment
    name: myapp
  updatePolicy:
    updateMode: "Initial"            # 只在Pod创建时应用推荐值
  resourcePolicy:
    containerPolicies:
    - containerName: '*'
      minAllowed:
        cpu: 100m
        memory: 128Mi
      maxAllowed:
        cpu: 4
        memory: 8Gi
      controlledResources: ["cpu", "memory"]

# 查看 VPA 推荐值
kubectl describe vpa myapp-vpa -n production

# 输出示例
# Recommendation:
#   Target:
#     CPU: 250m
#     Memory: 500Mi
#   Lower Bound:
#     CPU: 100m
#     Memory: 200Mi
#   Upper Bound:
#     CPU: 500m
#     Memory: 1Gi
#   Uncapped Target:
#     CPU: 250m
#     Memory: 500Mi

HPA 扩容 → Pod Pending（资源不足）→ CA 新增节点 → Pod 调度成功
流量下降 → HPA 缩容 → 节点利用率低 → CA 删除节点

# Cluster Autoscaler 配置示例（AWS EKS）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-autoscaler
  namespace: kube-system
spec:
  template:
    spec:
      containers:
      - image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.0
        name: cluster-autoscaler
        command:
        - ./cluster-autoscaler
        - --scale-down-unneeded-time=10m        # 节点低利用率持续10分钟才缩容
        - --scale-down-delay-after-add=10m      # 扩容后10分钟内不缩容
        - --scale-down-unempty-time=30m         # 排空节点后等30分钟再删除
        - --max-node-provision-time=15m         # 节点创建最大等待时间
        - --balance-similar-node-groups=true    # 平衡相似节点组
        - --expander=least-waste               # 扩容策略：最小浪费
        - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/my-cluster
        env:
        - name: AWS_REGION
          value: us-east-1

事件源(Kafka/Redis/...) → KEDA Scaler → KEDA Operator → HPA → Deployment
                                            ↓
                                      ScaledObject (CRD)
                                      ScaledJob (CRD)

helm repo add kedacore https://kedacore.github.io/charts
helm install keda kedacore/keda \
  --namespace keda-system \
  --create-namespace

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: kafka-consumer-scaledobject
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: kafka-consumer
  minReplicaCount: 0                  # 没消息时缩到0
  maxReplicaCount: 50                 # 最多50个消费者
  pollingInterval: 30                 # 30秒检查一次
  cooldownPeriod: 300                 # 缩容到0前等待300秒
  triggers:
  - type: kafka
    metadata:
      bootstrapServers: kafka-broker:9092
      consumerGroup: my-consumer-group
      topic: orders
      lagThreshold: "100"             # 每个分区滞后超过100条就扩容
      offsetResetPolicy: latest
      partitionLimitation: "0,1,2,3"  # 只监控指定分区

apiVersion: keda.sh/v1alpha1
kind: ScaledJob
metadata:
  name: image-processor
  namespace: production
spec:
  jobTargetRef:
    template:
      spec:
        containers:
        - name: processor
          image: myapp/processor:v1
          command: ["./process"]
        restartPolicy: Never
  maxReplicaCount: 10
  pollingInterval: 30
  triggers:
  - type: aws-sqs
    metadata:
      queueURL: https://sqs.us-east-1.amazonaws.com/123456789012/image-queue
      queueLength: "5"              # 每5条消息创建一个Job
      awsRegion: us-east-1

# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-server
  namespace: production
spec:
  replicas: 3
  selector:
    matchLabels:
      app: api-server
  template:
    metadata:
      labels:
        app: api-server
    spec:
      containers:
      - name: api
        image: myapp/api:v1
        resources:
          requests:
            cpu: 200m
            memory: 256Mi
          limits:
            cpu: 1000m
            memory: 1Gi
        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5

---
# HPA
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-server-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-server
  minReplicas: 3
  maxReplicas: 30
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 60
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "500"
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 0
      policies:
      - type: Percent
        value: 100
        periodSeconds: 30
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 10
        periodSeconds: 60

---
# PDB（配合 HPA 缩容时的优雅驱逐）
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-server-pdb
  namespace: production
spec:
  minAvailable: 2            # 至少保持2个Pod可用
  selector:
    matchLabels:
      app: api-server

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-server-pdb
spec:
  minAvailable: 50%          # 至少保持50%可用
  # 或 maxUnavailable: 1     # 最多不可用1个
  selector:
    matchLabels:
      app: api-server

# Prometheus 告警规则
groups:
- name: hpa-alerts
  rules:
  # HPA 达到最大副本数
  - alert: HPAAtMaxReplicas
    expr: kube_hpa_status_condition{condition="ScalingLimited",status="true"} == 1
    for: 10m
    annotations:
      summary: "HPA {{ $labels.hpa }} 达到扩容上限"

  # HPA 无法获取指标
  - alert: HPAMetricsUnavailable
    expr: kube_hpa_status_condition{condition="ScalingActive",status="false"} == 1
    for: 5m
    annotations:
      summary: "HPA {{ $labels.hpa }} 无法获取指标"

  # CA 无法扩容
  - alert: ClusterAutoscalerUnschedulable
    expr: cluster_autoscaler_unschedulable_pods_count > 0
    for: 10m
    annotations:
      summary: "{{ $value }} 个 Pod 无法调度"

# 压测触发 HPA 扩容
kubectl run load-generator --image=busybox:latest --restart=Never \
  -- /bin/sh -c "while true; do wget -q -O- http://api-server.production.svc.cluster.local:8080/; done"

# 观察 HPA 状态
kubectl get hpa -n production -w

# 观察 Pod 扩容过程
kubectl get pods -n production -l app=api-server -w

# 查看 HPA 详细状态
kubectl describe hpa myapp-hpa -n production

# 查看 HPA 指标
kubectl get --raw "/apis/autoscaling/v2/namespaces/production/horizontalpodautoscalers/myapp-hpa" | jq .

# 查看自定义指标
kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/production/pods/*/http_requests_per_second" | jq .

# 查看 CA 日志
kubectl logs -n kube-system -l app=cluster-autoscaler --tail=50

# 查看节点扩容事件
kubectl get events --field-selector reason=TriggeredScaleUp