Kubernetes Scheduler Principles and Tuning

Pod created → API Server → etcd → Scheduler Watch → Scheduling decision → Bind to node → kubelet creates container

1. Filter: Exclude nodes that don't meet conditions → Candidate node set
2. Score: Score candidate nodes → Select highest-scoring node

                    ┌──────────────┐
                    │  Pod enqueued │
                    └──────┬───────┘
                           ▼
                    ┌──────────────┐
                    │ Scheduling    │
                    │ cycle starts  │
                    └──────┬───────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: PreFilter   │  ← Pre-filter (check if Pod is schedulable)
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: Filter       │  ← Filter out infeasible nodes
               │  (exclude infeasible)    │
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: PostFilter  │  ← No nodes after filter? (trigger preemption)
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: Score        │  ← Score candidate nodes
               │  (select best node)     │
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: Reserve      │  ← Reserve resources
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: Permit       │  ← Allow/delay/reject
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: PreBind      │  ← Pre-bind processing
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: Bind         │  ← Bind to node
               └────────────┬────────────┘
                           ▼
               ┌────────────────────────┐
               │  Extension: PostBind     │  ← Post-bind processing
               └────────────────────────┘

Pod enters → activeQueue → Scheduling succeeds → Bind
                ↓ Scheduling fails
           backoffQueue → Wait backoff time → activeQueue (retry)
                ↓ Multiple failures
           unschedulableQueue → When conditions change → activeQueue

# Specify via scheduler configuration
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: default-scheduler
  pluginConfig:
  - name: NodeResourcesFit
    args:
      scoringStrategy:
        type: LeastAllocated    # Scoring strategy
        resources:
        - name: cpu
          weight: 1
        - name: memory
          weight: 1

# LeastAllocated (default): Pods spread across different nodes
# Suitable: General scenarios, even load

# MostAllocated: Pods concentrate on same node
# Suitable: Saving nodes (scale-down scenarios)

spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/arch
            operator: In
            values:
            - amd64
          - key: node.kubernetes.io/instance-type
            operator: In
            values:
            - c5.2xlarge
            - c5.4xlarge
          - key: topology.kubernetes.io/zone
            operator: NotIn
            values:
            - us-east-1a    # Exclude this AZ

spec:
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100            # Weight 1-100
        preference:
          matchExpressions:
          - key: ssd
            operator: In
            values:
            - "true"
      - weight: 50
        preference:
          matchExpressions:
          - key: topology.kubernetes.io/zone
            operator: In
            values:
            - us-east-1b

# Add labels
kubectl label nodes node-1 disktype=ssd
kubectl label nodes node-1 zone=east
kubectl label nodes node-1 gpu=true

# View labels
kubectl get nodes --show-labels
kubectl get nodes -l disktype=ssd

# Delete label
kubectl label nodes node-1 disktype-

spec:
  affinity:
    podAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values:
            - cache
        topologyKey: kubernetes.io/hostname
        # Pod should be on same node as app=cache Pods

spec:
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values:
            - web
        topologyKey: kubernetes.io/hostname
        # web Pods can't be on same node (high availability)
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchExpressions:
            - key: app
              operator: In
              values:
              - web
          topologyKey: topology.kubernetes.io/zone
          # Try to spread web Pods across different AZs

spec:
  topologySpreadConstraints:
  - maxSkew: 1                           # Max skew between topology domains
    topologyKey: kubernetes.io/hostname   # Topology domain: node
    whenUnsatisfiable: DoNotSchedule       # Reject if not satisfiable
    labelSelector:
      matchLabels:
        app: web
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone  # Topology domain: AZ
    whenUnsatisfiable: ScheduleAnyway         # Best effort
    labelSelector:
      matchLabels:
        app: web

# Add taint
kubectl taint nodes node-1 dedicated=gpu:NoSchedule

# View taints
kubectl describe node node-1 | grep Taint

# Remove taint
kubectl taint nodes node-1 dedicated=gpu:NoSchedule-

spec:
  tolerations:
  - key: "dedicated"
    operator: "Equal"
    value: "gpu"
    effect: "NoSchedule"

  # Shorthand: only match key and effect
  - key: "dedicated"
    operator: "Exists"
    effect: "NoSchedule"

  # Tolerate all taints (use with caution)
  - operator: "Exists"

  # Tolerate NoExecute with eviction delay
  - key: "node.kubernetes.io/not-ready"
    operator: "Exists"
    effect: "NoExecute"
    tolerationSeconds: 300    # Evict 300s after node becomes NotReady

# 1. GPU dedicated nodes
kubectl taint nodes gpu-node-1 nvidia.com/gpu=:NoSchedule
# Only Pods tolerating this taint (GPU tasks) will be scheduled

# 2. Maintenance mode
kubectl taint nodes node-1 maintenance=true:NoExecute
# All Pods evicted (except those tolerating), for node maintenance

# 3. Special nodes
kubectl taint nodes special-node-1 dedicated=special-team:NoSchedule
kubectl label nodes special-node-1 team=special-team

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000
globalDefault: false
description: "High priority workloads"

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: low-priority
value: 100
globalDefault: false
description: "Low priority workloads"

apiVersion: apps/v1
kind: Deployment
metadata:
  name: critical-app
spec:
  template:
    spec:
      priorityClassName: high-priority
      containers:
      - name: app
        image: myapp:v1

1. High-priority Pod can't be scheduled (insufficient resources)
2. Scheduler looks for lower-priority Pods to evict
3. Evicts lower-priority Pods, releases resources
4. High-priority Pod scheduled successfully
5. Lower-priority Pods enter Pending state

# Configure PDB for low-priority Pods to prevent eviction
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: low-priority-pdb
spec:
  minAvailable: 1                    # Keep at least 1 available
  selector:
    matchLabels:
      app: low-priority-app

spec:
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: DoNotSchedule
    labelSelector:
      matchLabels:
        app: web
    minDomains: 2                   # Requires at least 2 topology domains
    matchLabelKeys:
    - pod-template-hash             # For rolling updates

3 AZs, 6 web Pods, maxSkew=1

zone-a: [web-0, web-1]
zone-b: [web-2, web-3]
zone-c: [web-4, web-5]
Max skew = 0 ≤ 1 ✓

If zone-c is unavailable:
zone-a: [web-0, web-1, web-2]
zone-b: [web-3, web-4, web-5]
Max skew = 0 ≤ 1 ✓

# Deploy custom scheduler
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-scheduler
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: my-scheduler
  template:
    metadata:
      labels:
        app: my-scheduler
    spec:
      containers:
      - name: my-scheduler
        image: registry.k8s.io/kube-scheduler:v1.30.0
        command:
        - kube-scheduler
        - --config=/etc/kubernetes/my-scheduler-config.yaml
        - --scheduler-name=my-scheduler
        volumeMounts:
        - name: config
          mountPath: /etc/kubernetes
      volumes:
      - name: config
        configMap:
          name: my-scheduler-config

---
# Pod using custom scheduler
apiVersion: v1
kind: Pod
metadata:
  name: my-pod
spec:
  schedulerName: my-scheduler    # Specify scheduler
  containers:
  - name: app
    image: myapp:v1

# /etc/kubernetes/my-scheduler-config.yaml
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: my-scheduler
  plugins:
    filter:
      enabled:
      - name: NodeResourcesFit
      - name: NodeAffinity
      - name: TaintToleration
      disabled:
      - name: VolumeBinding     # Disable a default plugin
    score:
      enabled:
      - name: NodeResourcesFit
        weight: 10
      - name: InterPodAffinity
        weight: 5
  pluginConfig:
  - name: NodeResourcesFit
    args:
      scoringStrategy:
        type: LeastAllocated
        resources:
        - name: cpu
          weight: 2
        - name: memory
          weight: 1

Extension points:
PreFilter → Filter → PostFilter → PreScore → Score → Reserve → Permit → PreBind → Bind → PostBind

// Custom Filter plugin example
package myplugin

import (
    "context"
    v1 "k8s.io/api/core/v1"
    "k8s.io/kubernetes/pkg/scheduler/framework"
)

const Name = "MyPlugin"

type MyPlugin struct {
    handle framework.Handle
}

func (p *MyPlugin) Name() string { return Name }

func (p *MyPlugin) Filter(
    ctx context.Context,
    state *framework.CycleState,
    pod *v1.Pod,
    nodeInfo *framework.NodeInfo,
) *framework.Status {
    // Custom filtering logic
    if nodeInfo.Node().Labels["custom-label"] != "allowed" {
        return framework.NewStatus(framework.Unschedulable, "node not allowed")
    }
    return framework.NewStatus(framework.Success, "")
}

func (p *MyPlugin) Score(
    ctx context.Context,
    state *framework.CycleState,
    pod *v1.Pod,
    nodeName string,
) (int64, *framework.Status) {
    // Custom scoring logic
    return 100, framework.NewStatus(framework.Success, "")
}

// New initializes the plugin
func New(
    ctx context.Context,
    configuration runtime.Object,
    f framework.Handle,
) (framework.Plugin, error) {
    return &MyPlugin{handle: f}, nil
}

# Register custom plugin
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: my-scheduler
  plugins:
    filter:
      enabled:
      - name: MyPlugin
    score:
      enabled:
      - name: MyPlugin
        weight: 10

apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 6
  template:
    metadata:
      labels:
        app: web
    spec:
      # Cross-node + cross-AZ distribution
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/zone
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: web
      - maxSkew: 2
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app: web
      # Node affinity: select appropriate nodes
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node-role.kubernetes.io/worker
                operator: Exists
              - key: kubernetes.io/arch
                operator: In
                values:
                - amd64
      # Tolerate taints
      tolerations:
      - key: "node.kubernetes.io/not-ready"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 30
      - key: "node.kubernetes.io/unreachable"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 30
      containers:
      - name: web
        image: myapp:v1
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: 1000m
            memory: 1Gi

# Taint GPU nodes
kubectl taint nodes gpu-node nvidia.com/gpu=:NoSchedule
kubectl label nodes gpu-node accelerator=nvidia

---
# GPU Pod
apiVersion: v1
kind: Pod
metadata:
  name: gpu-task
spec:
  tolerations:
  - key: "nvidia.com/gpu"
    operator: "Exists"
    effect: "NoSchedule"
  nodeSelector:
    accelerator: nvidia
  containers:
  - name: gpu-container
    image: tensorflow/tensorflow:latest-gpu
    resources:
      limits:
        nvidia.com/gpu: 2       # Request 2 GPUs

# Create dedicated node pool
kubectl label nodes node-pool-a dedicated=team-a
kubectl taint nodes node-pool-a dedicated=team-a:NoSchedule

# Team A's Pods
spec:
  nodeSelector:
    dedicated: team-a
  tolerations:
  - key: "dedicated"
    operator: "Equal"
    value: "team-a"
    effect: "NoSchedule"

# View Pod scheduling failure reason
kubectl describe pod <pod-name> -n <namespace>

# Focus on Events section:
# Events:
#   Type     Reason            Age   From               Message
#   ----     ------            ----  ----               -------
#   Warning  FailedScheduling  10s   default-scheduler  0/10 nodes are available:
#     3 Insufficient cpu, 2 Insufficient memory,
#     3 node(s) had untolerated taint,
#     2 node(s) didn't match Pod's node affinity

# View scheduler logs
kubectl logs -n kube-system -l component=kube-scheduler --tail=100

# View scheduling decision details
# Need to enable verbose scheduler logging
kube-scheduler --v=5  # Increase log level

# Use kubectl debug to simulate scheduling
kubectl debug node/<node-name> -it --image=busybox

# Use scheduler simulator to test scheduling policies
# Reference: https://github.com/kubernetes-sigs/kube-scheduler-simulator