从 Docker Machine 退役说起：GitLab CI Runner 弹性架构与缓存治理的 7 个生产决策

# 步骤1：部署新的 K8s Runner，注册到同一个 GitLab 实例
# 用不同的 tag 区分新旧 Runner
# 旧 Runner tag: docker-machine
# 新 Runner tag: kubernetes

# 步骤2：在 .gitlab-ci.yml 中，逐步将 job 的 tags 从 docker-machine 切到 kubernetes
# 先切非核心 job（lint、文档生成），再切构建 job，最后切部署 job

# 步骤3：观察 1-2 周，确认无兼容性问题后，下线 Docker Machine Runner

def calculate_manager_resources(concurrent_jobs, avg_log_mb=4):
    """GitLab Runner Manager Pod 资源规划"""
    # CPU: ~6m per concurrent job + 10m base
    base_cpu = 0.01
    cpu_per_job = 0.006
    total_cpu = base_cpu + (concurrent_jobs * cpu_per_job)

    # Memory: ~2.5MB per job + 50MB base
    base_memory = 50
    memory_per_job = 2.5 * (avg_log_mb / 4)
    total_memory = base_memory + (concurrent_jobs * memory_per_job)

    return {
        'cpu_request': f"{int(total_cpu * 1000)}m",
        'cpu_limit': f"{int(total_cpu * 1.5 * 1000)}m",      # 50% 余量
        'memory_request': f"{int(total_memory)}Mi",
        'memory_limit': f"{int(total_memory * 2.0)}Mi"        # 100% 余量
    }

# 50 并发 Job 的 Manager Pod 资源配置
resources:
  requests:
    cpu: "310m"    # 10m + (50 × 6m) = 310m
    memory: "175Mi" # 50 + (50 × 2.5) = 175MB
  limits:
    cpu: "465m"    # 50% 余量
    memory: "350Mi" # 100% 余量

┌─────────────────────────────────────────────────────────┐
│ 第一层：Runner Manager 并发控制                           │
│ config.toml: concurrent = 50                            │
│ 作用：单个 Manager Pod 能同时处理多少 Job                 │
├─────────────────────────────────────────────────────────┤
│ 第二层：HPA 水平扩展 Manager Pod                          │
│ 作用：根据 CPU/内存利用率自动增减 Manager Pod 数量         │
│ minReplicas: 2  maxReplicas: 5                          │
├─────────────────────────────────────────────────────────┤
│ 第三层：Cluster Autoscaler 扩容 K8s 节点                  │
│ 作用：当 Job Pod pending 时自动新增节点                   │
└─────────────────────────────────────────────────────────┘

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: gitlab-runner-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: gitlab-runner
  minReplicas: 2        # 至少 2 个，保证高可用
  maxReplicas: 5        # 最多 5 个，每个 50 并发 = 250 总并发
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70   # CPU 超过 70% 触发扩容
    - type: Resource
      resource:
        name: memory
        target:
          type: Utilization
          averageUtilization: 80   # 内存超过 80% 触发扩容

# config.toml 关键配置
concurrent = 50

[[runners]]
  limit = 50
  request_concurrency = 10   # 默认1，调高后 Job 拉取更快

  [runners.kubernetes]
    namespace = "gitlab-runner"
    poll_interval = "5s"       # 默认3s，降低K8s API压力可调到5s
    poll_timeout = "180s"

# config.toml 分布式缓存配置
[runners.cache]
  Type = "s3"
  Shared = true    # 关键！开启后所有 Runner 共享同一个缓存桶

  [runners.cache.s3]
    ServerAddress = "minio.internal.example.com"
    BucketName = "gitlab-runner-cache"
    Insecure = false
    AuthenticationType = "access-key"
    # AccessKey 和 SecretKey 通过环境变量注入，不写在配置文件里

# .gitlab-ci.yml 缓存配置

# 策略1：按依赖文件 hash 缓存（推荐）
# 依赖文件不变就用缓存，变了就重新下载
cache:
  key:
    files:
      - go.sum          # Go 依赖锁文件
      - package-lock.json # Node 依赖锁文件
  paths:
    - .cache/go-build/
    - node_modules/

# 策略2：按分支缓存（适合不同分支依赖差异大的场景）
cache:
  key: "$CI_COMMIT_REF_SLUG"
  paths:
    - .m2/repository/

# 策略3：按分支 + 依赖文件组合缓存（最精细）
cache:
  key:
    key: "$CI_COMMIT_REF_SLUG"
    files:
      - go.sum
  paths:
    - .cache/

# 修复前（错误）
cache:
  key: "myproject"
  paths:
    - vendor/

# 修复后（正确）
cache:
  key:
    files:
      - go.mod
  paths:
    - vendor/

# .gitlab-ci.yml（单个微服务项目）
include:
  - project: 'devops/ci-templates'
    file: '/go-service.yml'
    ref: 'main'

# 项目特定变量
variables:
  SERVICE_NAME: "user-service"
  REGISTRY: "registry.example.com"

# 只需要覆盖项目特有的配置
# 通用构建、测试、部署逻辑都在 go-service.yml 模板里

# go-service.yml（共享模板）
stages:
  - lint
  - test
  - build
  - deploy

variables:
  # 默认值，各项目可覆盖
  GO_VERSION: "1.22"

lint:
  stage: lint
  image: golang:${GO_VERSION}
  script:
    - go fmt ./...
    - go vet ./...
    - golangci-lint run

test:
  stage: test
  image: golang:${GO_VERSION}
  cache:
    key:
      files: [go.sum]
    paths: [.cache/]
  script:
    - go test -race -coverprofile=coverage.out ./...

build:
  stage: build
  image: docker:24.0
  services: [docker:24.0-dind]
  script:
    - docker build -t $REGISTRY/$SERVICE_NAME:$CI_COMMIT_SHORT_SHA .
    - docker push $REGISTRY/$SERVICE_NAME:$CI_COMMIT_SHORT_SHA

# .gitlab-ci.yml（父流水线）
stages:
  - generate
  - trigger

generate-config:
  stage: generate
  script:
    - |
      cat > generated-config.yml << 'EOF'
      include:
        - project: 'devops/ci-templates'
          file: '/go-service.yml'
      EOF
      # 如果 docs/ 有变更，追加文档部署 job
      if git diff --name-only HEAD~1 | grep -q "^docs/"; then
        cat >> generated-config.yml << 'EOF'
        deploy-docs:
          stage: deploy
          script:
            - make docs-deploy
        EOF
      fi      
  artifacts:
    paths: [generated-config.yml]

trigger-child:
  stage: trigger
  trigger:
    include:
      - artifact: generated-config.yml
        job: generate-config
  strategy: depend    # 父流水线等待子流水线完成

# 推荐：rules
build:
  stage: build
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never       # MR 流水线不跑构建
    - if: '$CI_COMMIT_BRANCH == "main"'
      changes:
        - src/**/*
      when: on_success   # main 分支 src/ 有变更才构建
    - when: manual        # 其他情况手动触发
  script:
    - make build

# 给 Manager 节点打 taint 和 label
kubectl taint nodes node-pool-manager runner.gitlab.com/manager=:NoExecute
kubectl label nodes node-pool-manager runner.gitlab.com/workload-type=manager

# 给 Worker 节点打 taint 和 label
kubectl taint nodes node-pool-worker runner.gitlab.com/job=:NoExecute
kubectl label nodes node-pool-worker runner.gitlab.com/workload-type=job

# Manager Pod 配置 nodeSelector 和 tolerations
spec:
  nodeSelector:
    runner.gitlab.com/workload-type: manager
  tolerations:
    - key: runner.gitlab.com/manager
      operator: Exists
      effect: NoExecute

[runners.kubernetes.node_selector]
"runner.gitlab.com/workload-type" = "job"

[runners.kubernetes.node_tolerations]
"runner.gitlab.com/job=" = "NoExecute"

# Manager Pod CPU 持续高利用率
groups:
  - name: gitlab-runner
    rules:
      - alert: RunnerManagerHighCPU
        expr: |
          rate(container_cpu_usage_seconds_total{pod=~"gitlab-runner.*"}[5m]) * 1000
          / kube_pod_container_resource_limits{pod=~"gitlab-runner.*", resource="cpu"} * 1000
          > 70          
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Runner Manager CPU 使用率持续超过 70%"

      - alert: RunnerManagerOOM
        expr: |
          container_memory_working_set_bytes{pod=~"gitlab-runner.*"}
          / kube_pod_container_resource_limits{pod=~"gitlab-runner.*", resource="memory"}
          > 0.8          
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Runner Manager 内存使用率超过 80%，可能即将 OOM"

      - alert: RunnerJobQueueSaturation
        expr: |
          gitlab_runner_jobs / gitlab_runner_limit > 0.9          
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Runner Job 队列饱和度超过 90%"

      - alert: RunnerErrors
        expr: increase(gitlab_runner_errors_total[5m]) > 0
        labels:
          severity: critical
        annotations:
          summary: "Runner 5 分钟内出现错误"

# 查看 Manager Pod 当前资源使用
kubectl top pods --containers -l app=gitlab-runner

# 检查最近 2 小时的错误日志
kubectl logs -l app=gitlab-runner --since=2h | grep -E "(error|timeout|failed)"

# 查看 pending 的 Job Pod（如果持续 pending 说明资源不够）
kubectl get pods -n gitlab-runner --field-selector=status.phase=Pending

# 检查 Manager Pod 是否被 OOMKill 过
kubectl get pods -l app=gitlab-runner -o jsonpath='{.items[].status.containerStatuses[].lastState.terminated.reason}'