Kubernetes 调度器原理与调优

Pod 创建 → API Server → etcd → 调度器 Watch → 调度决策 → 绑定到节点 → kubelet 创建容器

1. 过滤（Filter）：排除不满足条件的节点 → 候选节点集
2. 打分（Score）：对候选节点打分 → 选择最高分节点

                    ┌──────────────┐
                    │  Pod 入队     │
                    └──────┬───────┘
                           ▼
                    ┌──────────────┐
                    │  调度周期开始  │
                    └──────┬───────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：PreFilter       │  ← 预过滤（检查 Pod 是否可调度）
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：Filter          │  ← 过滤不满足条件的节点
              │  （排除不可行节点）       │
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：PostFilter      │  ← 过滤后无节点？（触发抢占）
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：Score           │  ← 对候选节点打分
              │  （选择最优节点）         │
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：Reserve          │  ← 预留资源
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：Permit          │  ← 允许/延迟/拒绝
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：PreBind         │  ← 绑定前处理
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：Bind            │  ← 绑定到节点
              └────────────┬────────────┘
                           ▼
              ┌────────────────────────┐
              │  扩展点：PostBind        │  ← 绑定后处理
              └────────────────────────┘

Pod 进入 → activeQueue → 调度成功 → Bind
               ↓ 调度失败
          backoffQueue → 等待退避时间 → activeQueue（重试）
               ↓ 多次失败
          unschedulableQueue → 条件变化时 → activeQueue

# 通过调度器配置指定
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: default-scheduler
  pluginConfig:
  - name: NodeResourcesFit
    args:
      scoringStrategy:
        type: LeastAllocated    # 打分策略
        resources:
        - name: cpu
          weight: 1
        - name: memory
          weight: 1

# LeastAllocated（默认）：Pod 分散到不同节点
# 适合：通用场景，负载均匀

# MostAllocated：Pod 集中到同一节点
# 适合：节省节点（缩容场景）

spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/arch
            operator: In
            values:
            - amd64
          - key: node.kubernetes.io/instance-type
            operator: In
            values:
            - c5.2xlarge
            - c5.4xlarge
          - key: topology.kubernetes.io/zone
            operator: NotIn
            values:
            - us-east-1a    # 排除该可用区

spec:
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100            # 权重 1-100
        preference:
          matchExpressions:
          - key: ssd
            operator: In
            values:
            - "true"
      - weight: 50
        preference:
          matchExpressions:
          - key: topology.kubernetes.io/zone
            operator: In
            values:
            - us-east-1b

# 添加标签
kubectl label nodes node-1 disktype=ssd
kubectl label nodes node-1 zone=east
kubectl label nodes node-1 gpu=true

# 查看标签
kubectl get nodes --show-labels
kubectl get nodes -l disktype=ssd

# 删除标签
kubectl label nodes node-1 disktype-

spec:
  affinity:
    podAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values:
            - cache
        topologyKey: kubernetes.io/hostname
        # Pod 要和 app=cache 的 Pod 在同一节点

spec:
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values:
            - web
        topologyKey: kubernetes.io/hostname
        # web Pod 之间不能在同一节点（高可用）
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchExpressions:
            - key: app
              operator: In
              values:
              - web
          topologyKey: topology.kubernetes.io/zone
          # 尽量让 web Pod 分布在不同可用区

spec:
  topologySpreadConstraints:
  - maxSkew: 1                           # 各拓扑域之间的最大差值
    topologyKey: kubernetes.io/hostname   # 拓扑域：节点
    whenUnsatisfiable: DoNotSchedule       # 不满足时拒绝调度
    labelSelector:
      matchLabels:
        app: web
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone  # 拓扑域：可用区
    whenUnsatisfiable: ScheduleAnyway         # 尽量满足
    labelSelector:
      matchLabels:
        app: web

# 添加污点
kubectl taint nodes node-1 dedicated=gpu:NoSchedule

# 查看污点
kubectl describe node node-1 | grep Taint

# 删除污点
kubectl taint nodes node-1 dedicated=gpu:NoSchedule-

spec:
  tolerations:
  - key: "dedicated"
    operator: "Equal"
    value: "gpu"
    effect: "NoSchedule"

  # 简写：只匹配 key 和 effect
  - key: "dedicated"
    operator: "Exists"
    effect: "NoSchedule"

  # 容忍所有污点（慎用）
  - operator: "Exists"

  # 容忍 NoExecute 并设置驱逐延迟
  - key: "node.kubernetes.io/not-ready"
    operator: "Exists"
    effect: "NoExecute"
    tolerationSeconds: 300    # 节点NotReady后300秒才驱逐

# 1. GPU 专用节点
kubectl taint nodes gpu-node-1 nvidia.com/gpu=:NoSchedule
# 只有容忍该污点的 Pod（GPU 任务）才会调度

# 2. 维护模式
kubectl taint nodes node-1 maintenance=true:NoExecute
# 所有 Pod 被驱逐（除了容忍的），用于节点维护

# 3. 特殊节点
kubectl taint nodes special-node-1 dedicated=special-team:NoSchedule
kubectl label nodes special-node-1 team=special-team

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000
globalDefault: false
description: "高优先级工作负载"

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: low-priority
value: 100
globalDefault: false
description: "低优先级工作负载"

apiVersion: apps/v1
kind: Deployment
metadata:
  name: critical-app
spec:
  template:
    spec:
      priorityClassName: high-priority
      containers:
      - name: app
        image: myapp:v1

1. 高优先级 Pod 无法调度（资源不足）
2. 调度器寻找可以驱逐的低优先级 Pod
3. 驱逐低优先级 Pod，释放资源
4. 高优先级 Pod 调度成功
5. 低优先级 Pod 进入 Pending 状态

# 低优先级 Pod 配置 PDB 防止被驱逐
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: low-priority-pdb
spec:
  minAvailable: 1                    # 至少保持1个可用
  selector:
    matchLabels:
      app: low-priority-app

spec:
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: DoNotSchedule
    labelSelector:
      matchLabels:
        app: web
    minDomains: 2                   # 至少2个拓扑域才生效
    matchLabelKeys:
    - pod-template-hash             # 配合滚动更新

3个可用区，6个 web Pod，maxSkew=1

zone-a: [web-0, web-1]
zone-b: [web-2, web-3]
zone-c: [web-4, web-5]
最大差值 = 0 ≤ 1 ✓

如果 zone-c 不可用：
zone-a: [web-0, web-1, web-2]
zone-b: [web-3, web-4, web-5]
最大差值 = 0 ≤ 1 ✓

# 部署自定义调度器
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-scheduler
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: my-scheduler
  template:
    metadata:
      labels:
        app: my-scheduler
    spec:
      containers:
      - name: my-scheduler
        image: registry.k8s.io/kube-scheduler:v1.30.0
        command:
        - kube-scheduler
        - --config=/etc/kubernetes/my-scheduler-config.yaml
        - --scheduler-name=my-scheduler
        volumeMounts:
        - name: config
          mountPath: /etc/kubernetes
      volumes:
      - name: config
        configMap:
          name: my-scheduler-config

---
# Pod 使用自定义调度器
apiVersion: v1
kind: Pod
metadata:
  name: my-pod
spec:
  schedulerName: my-scheduler    # 指定调度器
  containers:
  - name: app
    image: myapp:v1

# /etc/kubernetes/my-scheduler-config.yaml
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: my-scheduler
  plugins:
    filter:
      enabled:
      - name: NodeResourcesFit
      - name: NodeAffinity
      - name: TaintToleration
      disabled:
      - name: VolumeBinding     # 禁用某个默认插件
    score:
      enabled:
      - name: NodeResourcesFit
        weight: 10
      - name: InterPodAffinity
        weight: 5
  pluginConfig:
  - name: NodeResourcesFit
    args:
      scoringStrategy:
        type: LeastAllocated
        resources:
        - name: cpu
          weight: 2
        - name: memory
          weight: 1

扩展点：
PreFilter → Filter → PostFilter → PreScore → Score → Reserve → Permit → PreBind → Bind → PostBind

// 自定义 Filter 插件示例
package myplugin

import (
    "context"
    v1 "k8s.io/api/core/v1"
    "k8s.io/kubernetes/pkg/scheduler/framework"
)

const Name = "MyPlugin"

type MyPlugin struct {
    handle framework.Handle
}

func (p *MyPlugin) Name() string { return Name }

func (p *MyPlugin) Filter(
    ctx context.Context,
    state *framework.CycleState,
    pod *v1.Pod,
    nodeInfo *framework.NodeInfo,
) *framework.Status {
    // 自定义过滤逻辑
    if nodeInfo.Node().Labels["custom-label"] != "allowed" {
        return framework.NewStatus(framework.Unschedulable, "node not allowed")
    }
    return framework.NewStatus(framework.Success, "")
}

func (p *MyPlugin) Score(
    ctx context.Context,
    state *framework.CycleState,
    pod *v1.Pod,
    nodeName string,
) (int64, *framework.Status) {
    // 自定义打分逻辑
    return 100, framework.NewStatus(framework.Success, "")
}

// New 初始化插件
func New(
    ctx context.Context,
    configuration runtime.Object,
    f framework.Handle,
) (framework.Plugin, error) {
    return &MyPlugin{handle: f}, nil
}

# 注册自定义插件
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: my-scheduler
  plugins:
    filter:
      enabled:
      - name: MyPlugin
    score:
      enabled:
      - name: MyPlugin
        weight: 10

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 6
  template:
    metadata:
      labels:
        app: web
    spec:
      # 跨节点 + 跨可用区分布
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/zone
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: web
      - maxSkew: 2
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app: web
      # 节点亲和性：选择合适的节点
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node-role.kubernetes.io/worker
                operator: Exists
              - key: kubernetes.io/arch
                operator: In
                values:
                - amd64
      # 容忍污点
      tolerations:
      - key: "node.kubernetes.io/not-ready"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 30
      - key: "node.kubernetes.io/unreachable"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 30
      containers:
      - name: web
        image: myapp:v1
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: 1000m
            memory: 1Gi

# GPU 节点打污点
kubectl taint nodes gpu-node nvidia.com/gpu=:NoSchedule
kubectl label nodes gpu-node accelerator=nvidia

---
# GPU Pod
apiVersion: v1
kind: Pod
metadata:
  name: gpu-task
spec:
  tolerations:
  - key: "nvidia.com/gpu"
    operator: "Exists"
    effect: "NoSchedule"
  nodeSelector:
    accelerator: nvidia
  containers:
  - name: gpu-container
    image: tensorflow/tensorflow:latest-gpu
    resources:
      limits:
        nvidia.com/gpu: 2       # 请求2块GPU

# 创建专用节点池
kubectl label nodes node-pool-a dedicated=team-a
kubectl taint nodes node-pool-a dedicated=team-a:NoSchedule

# Team A 的 Pod
spec:
  nodeSelector:
    dedicated: team-a
  tolerations:
  - key: "dedicated"
    operator: "Equal"
    value: "team-a"
    effect: "NoSchedule"

# 查看 Pod 调度失败原因
kubectl describe pod <pod-name> -n <namespace>

# 关注 Events 部分：
# Events:
#   Type     Reason            Age   From               Message
#   ----     ------            ----  ----               -------
#   Warning  FailedScheduling  10s   default-scheduler  0/10 nodes are available: 
#     3 Insufficient cpu, 2 Insufficient memory, 
#     3 node(s) had untolerated taint, 
#     2 node(s) didn't match Pod's node affinity

# 查看调度器日志
kubectl logs -n kube-system -l component=kube-scheduler --tail=100

# 查看调度决策详情
# 需要开启调度器详细日志
kube-scheduler --v=5  # 调高日志级别

# 使用 kubectl debug 模拟调度
kubectl debug node/<node-name> -it --image=busybox

# 使用 scheduler simulator 测试调度策略
# 参考 https://github.com/kubernetes-sigs/kube-scheduler-simulator