K8s 安全扫描与 CIS Benchmark 合规：从 kube-bench 到生产级加固的完整实战

# 下载最新版本（截至 2026 年 7 月，最新版本为 v0.10.0）
curl -L https://github.com/aquasecurity/kube-bench/releases/download/v0.10.0/kube-bench_0.10.0_linux_amd64.tar.gz -o kube-bench.tar.gz
tar -zxvf kube-bench.tar.gz
sudo mv kube-bench /usr/local/bin/

# 验证
kube-bench version

apiVersion: batch/v1
kind: Job
metadata:
  name: kube-bench
  namespace: kube-system
spec:
  template:
    spec:
      hostPID: true
      containers:
        - name: kube-bench
          image: aquasec/kube-bench:0.10.0
          command: ["kube-bench"]
          args: ["--benchmark", "cis-1.10"]
          volumeMounts:
            - name: var-lib-etcd
              mountPath: /var/lib/etcd
              readOnly: true
            - name: etc-kubernetes
              mountPath: /etc/kubernetes
              readOnly: true
            - name: usr-bin
              mountPath: /usr/local/mount-from-host/bin
              readOnly: true
            - name: var-lib-kubelet
              mountPath: /var/lib/kubelet
              readOnly: true
            - name: etc-systemd
              mountPath: /etc/systemd
              readOnly: true
      restartPolicy: Never
      volumes:
        - name: var-lib-etcd
          hostPath:
            path: /var/lib/etcd
        - name: etc-kubernetes
          hostPath:
            path: /etc/kubernetes
        - name: usr-bin
          hostPath:
            path: /usr/bin
        - name: var-lib-kubelet
          hostPath:
            path: /var/lib/kubelet
        - name: etc-systemd
          hostPath:
            path: /etc/systemd

# 在 master 节点上跑，检查控制平面
kube-bench --benchmark cis-1.10 --targets master,node

# 只检查某个组件
kube-bench --benchmark cis-1.10 --targets etcd

# 输出 JSON 格式报告（方便程序化处理）
kube-bench --benchmark cis-1.10 --targets master,node --output json | jq . > kube-bench-report.json

[FAIL] 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated)
[FAIL] 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated)
[PASS] 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated)
[WARN] 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated)

Remediation:
Edit the API Server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml
on the master node and set the below parameter.
--authorization-mode=Node,RBAC

# 检查当前配置
ps -ef | grep kube-apiserver | grep anonymous-auth

# 修复：编辑 /etc/kubernetes/manifests/kube-apiserver.yaml
# 添加参数
- --anonymous-auth=false

# 检查
ps -ef | grep etcd | grep client-cert-auth

# 修复：编辑 etcd 配置
- --client-cert-auth=true
- --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
- --cert-file=/etc/kubernetes/pki/etcd/server.crt
- --key-file=/etc/kubernetes/pki/etcd/server.key

# 检查
ps -ef | grep kubelet | grep read-only-port

# 修复
# 在 kubelet 配置文件 /var/lib/kubelet/config.yaml 中设置
readOnlyPort: 0

# 检查
ps -ef | grep kube-apiserver | grep authorization-mode

# 修复：确保 authorization-mode 包含 RBAC
- --authorization-mode=Node,RBAC

# /etc/kubernetes/manifests/kube-apiserver.yaml
- --audit-log-path=/var/log/kubernetes/audit/audit.log
- --audit-log-maxage=30
- --audit-log-maxbackup=10
- --audit-log-maxsize=100
- --audit-policy-file=/etc/kubernetes/audit-policy.yaml

# /etc/kubernetes/audit-policy.yaml
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  # 记录所有 Secret 操作
  - level: RequestResponse
    resources:
      - group: ""
        resources: ["secrets"]
  # 记录所有认证授权操作
  - level: Metadata
    resources:
      - group: ""
        resources: ["serviceaccounts", "rolebindings", "clusterrolebindings"]
  # 记录所有写操作
  - level: Request
    verbs: ["create", "update", "patch", "delete"]
  # 其余只记录元数据
  - level: Metadata

# 给 namespace 打标签，强制 restricted 级别
kubectl label namespace production \
  pod-security.kubernetes.io/enforce=restricted \
  pod-security.kubernetes.io/audit=restricted \
  pod-security.kubernetes.io/warn=restricted

# 检查
ls -la /var/lib/etcd/

# 修复
chown -R etcd:etcd /var/lib/etcd/
chmod 700 /var/lib/etcd/

# 检查
ls -la /etc/kubernetes/pki/

# 修复：CA 证书 644，CA 私钥 600
chmod 644 /etc/kubernetes/pki/ca.crt
chmod 600 /etc/kubernetes/pki/ca.key
# 各组件证书 644，私钥 600
chmod 644 /etc/kubernetes/pki/apiserver.crt
chmod 600 /etc/kubernetes/pki/apiserver.key

# /var/lib/kubelet/config.yaml
authentication:
  anonymous:
    enabled: false
  webhook:
    enabled: true

# 确保 API Server 配置了 TLS
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key

# 方式一：二进制
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin/

# 方式二：apt（Debian/Ubuntu）
sudo apt-get install trivy

# 方式三：Docker
docker run --rm aquasec/trivy:latest image nginx:latest

# 扫描单个镜像
trivy image nginx:1.27

# 扫描结果按严重级别过滤
trivy image --severity HIGH,CRITICAL nginx:1.27

# 只输出存在漏洞的包
trivy image --severity CRITICAL --format json nginx:1.27 | jq '.Results[0].Vulnerabilities[] | {VulnerabilityID, PkgName, InstalledVersion, FixedVersion}'

# 扫描镜像并生成 HTML 报告
trivy image --format template --template "@html.tpl" -o report.html nginx:1.27

# 列出集群中所有运行中的 Pod 使用的镜像
kubectl get pods --all-namespaces -o jsonpath="{range .items[*]}{range .spec.containers[*]}{.image}{'\n'}{end}{end}" | sort -u > images.txt

# 批量扫描
while read image; do
  echo "=== Scanning $image ==="
  trivy image --severity HIGH,CRITICAL "$image"
done < images.txt

# 扫描集群 KBOM
trivy k8s cluster --format kubebom -o kbom.json

# 扫描集群配置错误（类似 kube-bench 但集成在 Trivy 里）
trivy k8s cluster --scanners misconfig

# 扫描集群中的漏洞
trivy k8s cluster --scanners vuln

# 安装
pip install kube-hunter

# 在集群外扫描（模拟外部攻击者）
kube-hunter --remote <cluster-ip>

# 在节点上扫描（模拟拿到节点后的横向移动）
kube-hunter --pod

# 在集群内扫描
kubectl run kube-hunter --rm -it --image=aquasec/kube-hunter:latest -- --pod

Vulnerability                | Severity    | Description
-----------------------------|-------------|--------------------------------------
CAP_NET_RAW enabled          | low         | Pods have CAP_NET_RAW capability
Dashboard exposed            | high        | Kubernetes Dashboard is exposed
Metrics Server insecure      | medium      | Metrics Server allows anonymous access

# .github/workflows/security-scan.yml
name: Container Image Security Scan
on:
  pull_request:
    paths:
      - 'deploy/**'  # 只在部署配置变更时触发

jobs:
  trivy-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Extract images from manifests
        id: images
        run: |
          IMAGES=$(grep -ohP 'image:\s*\K.*' deploy/*.yaml | sort -u)
          echo "images<<EOF" >> $GITHUB_OUTPUT
          echo "$IMAGES" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT          

      - name: Run Trivy scan
        run: |
          for image in ${{ steps.images.outputs.images }}; do
            echo "=== Scanning $image ==="
            trivy image --severity CRITICAL --exit-code 1 "$image" || {
              echo "::error::Critical vulnerabilities found in $image"
              exit 1
            }
          done          

apiVersion: batch/v1
kind: CronJob
metadata:
  name: kube-bench-scan
  namespace: kube-system
spec:
  schedule: "0 2 * * 1"  # 每周一凌晨 2 点
  jobTemplate:
    spec:
      template:
        spec:
          hostPID: true
          containers:
            - name: kube-bench
              image: aquasec/kube-bench:0.10.0
              command:
                - /bin/sh
                - -c
                - |
                  kube-bench --benchmark cis-1.10 --targets master,node \
                    --output json > /tmp/report.json
                  # 把报告推到监控系统或 Slack
                  curl -X POST "$SLACK_WEBHOOK" \
                    -H "Content-Type: application/json" \
                    -d "{\"text\":\"Weekly kube-bench scan completed. $(jq '[.Controls[].Checks[] | select(.State == \"FAIL\")] | length') FAIL items.\"}"                  
              env:
                - name: SLACK_WEBHOOK
                  valueFrom:
                    secretKeyRef:
                      name: slack-webhook
                      key: url
              volumeMounts:
                - name: etc-kubernetes
                  mountPath: /etc/kubernetes
                  readOnly: true
                - name: var-lib-kubelet
                  mountPath: /var/lib/kubelet
                  readOnly: true
          restartPolicy: OnFailure
          volumes:
            - name: etc-kubernetes
              hostPath:
                path: /etc/kubernetes
            - name: var-lib-kubelet
              hostPath:
                path: /var/lib/kubelet

#!/usr/bin/env python3
"""kube-bench results exporter for Prometheus"""
import json
import subprocess
from http.server import HTTPServer, BaseHTTPRequestHandler

class MetricsHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path != '/metrics':
            self.send_error(404)
            return

        result = subprocess.run(
            ['kube-bench', '--benchmark', 'cis-1.10',
             '--targets', 'master,node', '--output', 'json'],
            capture_output=True, text=True
        )
        data = json.loads(result.stdout)

        metrics = []
        total_pass = 0
        total_fail = 0
        total_warn = 0

        for control in data.get('Controls', []):
            for check in control.get('Checks', []):
                status = 1 if check['State'] == 'PASS' else 0
                metrics.append(
                    f'kube_bench_check{{id="{check["ID"]}",'
                    f'title="{check["Text"]}",'
                    f'state="{check["State"]}"}} {status}'
                )
                if check['State'] == 'PASS':
                    total_pass += 1
                elif check['State'] == 'FAIL':
                    total_fail += 1
                else:
                    total_warn += 1

        total = total_pass + total_fail + total_warn
        compliance = (total_pass / total * 100) if total > 0 else 0

        metrics.append(f'kube_bench_total_checks {total}')
        metrics.append(f'kube_bench_pass_total {total_pass}')
        metrics.append(f'kube_bench_fail_total {total_fail}')
        metrics.append(f'kube_bench_warn_total {total_warn}')
        metrics.append(f'kube_bench_compliance_percent {compliance}')

        self.send_response(200)
        self.send_header('Content-Type', 'text/plain')
        self.end_headers()
        self.wfile.write('\n'.join(metrics).encode())

if __name__ == '__main__':
    HTTPServer(('0.0.0.0', 9099), MetricsHandler).serve_forever()

# 合规率
kube_bench_compliance_percent

# FAIL 项数量趋势
kube_bench_fail_total

# 按检查项查看状态
kube_bench_check{state="FAIL"}

# 禁止特权容器的 ConstraintTemplate
apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
  name: k8sdisallowprivileged
spec:
  crd:
    spec:
      names:
        kind: K8sDisallowPrivileged
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8sdisallowprivileged

        violation[{"msg": msg}] {
          container := input.review.object.spec.containers[_]
          container.securityContext.privileged == true
          msg := sprintf("Privileged container %v is not allowed", [container.name])
        }

        violation[{"msg": msg}] {
          container := input.review.object.spec.initContainers[_]
          container.securityContext.privileged == true
          msg := sprintf("Privileged init container %v is not allowed", [container.name])
        }        
---
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sDisallowPrivileged
metadata:
  name: no-privileged-containers
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Pod"]
    excludedNamespaces: ["kube-system"]

# Falco 规则示例：检测容器内 Shell 执行
- rule: Shell Spawned in Container
  desc: Detect shell spawned in container (possible attack)
  condition: >
    container.id != "" and
    proc.name in (bash, sh, zsh, ksh)    
  output: >
    Shell spawned in container
    (user=%user.name container_id=%container.id
    container_name=%container.name
    shell=%proc.name cmdline=%proc.cmdline)    
  priority: WARNING
  tags: [container, shell, mitre_execution]

helm repo add falcosecurity https://falcosecurity.github.io/charts
helm repo update
helm install falco falcosecurity/falco \
  --namespace falco \
  --create-namespace \
  --set driver.kind=ebpf \
  --set falcosidekick.enabled=true \
  --set falcosidekick.config.slack.webhookurl=$SLACK_WEBHOOK

步骤 1: kube-bench 扫描 → 获取 FAIL 清单
步骤 2: 按决策矩阵排优先级
步骤 3: 修复 P0/P1 项（API Server 参数、etcd 认证、RBAC、审计日志）
步骤 4: Trivy 扫描所有运行镜像 → 修复 CRITICAL 漏洞镜像
步骤 5: 部署 PSA 标签 → 限制 Pod 安全级别
步骤 6: 部署 Gatekeeper → 拦截不安全配置（先 warn 模式）
步骤 7: 部署 Falco → 运行时监控
步骤 8: 把 kube-bench + Trivy 扫描加到 CronJob + CI/CD
步骤 9: Grafana 建合规率大盘 → 持续监控

# 在有网环境下载数据库
trivy --download-db-only

# 把数据库拷到离线环境
# 默认路径：~/.cache/trivy/db/

# 离线环境扫描时指定 --skip-db-update
trivy image --skip-db-update nginx:1.27