CI/CD Deployment Speed Optimization in Practice: From 90 Minutes to 5 Minutes

# GitLab CI: Duration tracking is built-in
# No extra config needed—GitLab records started_at and finished_at for each job
# Fetch detailed data via API:

# Get duration of each job in the latest pipeline
# curl --header "PRIVATE-TOKEN: <token>" \
#   "https://gitlab.example.com/api/v4/projects/<project_id>/pipelines/<pipeline_id>/jobs" \
#   | jq '.[] | {name: .name, duration: .duration, status: .status}'

// Jenkins: Record stage durations in Pipeline
pipeline {
    agent any
    stages {
        stage('Build') {
            steps {
                script {
                    def startTime = currentBuild.startTimeInMillis
                    echo "Build started at: ${new Date(startTime)}"
                }
                // ... build steps
            }
            post {
                always {
                    script {
                        def duration = (System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000
                        echo "Total duration: ${duration}s"
                    }
                }
            }
        }
    }
}

# Query P95 build duration trend over the last 7 days
histogram_quantile(0.95, 
  sum(rate(ci_build_duration_seconds_bucket[7d])) by (le, stage)
)

# .gitlab-ci.yml
variables:
  # overlay2 driver is 3-5x faster than vfs
  DOCKER_DRIVER: overlay2
  CACHE_DIR: "${CI_PROJECT_DIR}/.cache"

# Global cache config
cache:
  # Generate cache key from lockfiles to avoid cross-branch pollution
  key:
    files:
      - package-lock.json
      - go.sum
    prefix: ${CI_COMMIT_REF_SLUG}
  paths:
    - ${CACHE_DIR}/npm/       # npm cache
    - ${CACHE_DIR}/go-build/  # Go build cache
    - ${CACHE_DIR}/go-pkg/    # Go module cache
    - node_modules/           # Node.js dependencies

# Go project build job
build_go:
  stage: build
  image: golang:1.22-alpine
  variables:
    GOPATH: ${CACHE_DIR}/go-pkg
    GOCACHE: ${CACHE_DIR}/go-build
  script:
    # Set Go proxy (essential for China mainland)
    - export GOPROXY=https://goproxy.cn,direct
    - go mod download          # Skips network download on cache hit
    - go build -o bin/app ./cmd/server
  artifacts:
    paths:
      - bin/
    expire_in: 1 hour         # Retain build artifacts for 1 hour

// Jenkinsfile: Kubernetes agent with persistent cache volumes
pipeline {
    agent {
        kubernetes {
            yaml '''
apiVersion: v1
kind: Pod
spec:
  containers:
  - name: golang
    image: golang:1.22-alpine
    command: ["sleep", "infinity"]
    volumeMounts:
    - name: go-cache
      mountPath: /go/pkg       # Go module cache
    - name: go-build-cache
      mountPath: /root/.cache/go-build  # Go compile cache
    env:
    - name: GOPROXY
      value: "https://goproxy.cn,direct"
  volumes:
  - name: go-cache
    persistentVolumeClaim:
      claimName: go-module-cache    # PVC persistence
  - name: go-build-cache
    persistentVolumeClaim:
      claimName: go-build-cache     # PVC persistence
'''
        }
    }
    stages {
        stage('Build') {
            steps {
                sh 'go mod download'
                sh 'go build -o bin/app ./cmd/server'
            }
        }
    }
}

<!-- pom.xml: Enable incremental compilation -->
<plugin>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-compiler-plugin</artifactId>
    <version>3.11.0</version>
    <configuration>
        <useIncrementalCompilation>true</useIncrementalCompilation>
        <parallelCompilation>true</parallelCompilation>
    </configuration>
</plugin>

# .gitlab-ci.yml: Maven cache
build_java:
  image: maven:3.9-eclipse-temurin-17
  cache:
    key:
      files:
        - pom.xml
    paths:
      - .m2/repository/        # Maven local repo
  script:
    # Offline mode first (skip download on cache hit)
    - mvn package -o -DskipTests || mvn package -DskipTests
    # Try offline compile first, fall back to online if deps are missing
  artifacts:
    paths:
      - target/*.jar

# .gitlab-ci.yml: Parallel builds for multiple microservices
stages:
  - build
  - test
  - package

# Parallel builds for multiple services
build_user_service:
  stage: build
  script:
    - cd services/user && go build -o ../../bin/user ./cmd/server
  rules:
    - changes:
        - services/user/**/*        # Only build if user service code changed

build_order_service:
  stage: build
  script:
    - cd services/order && go build -o ../../bin/order ./cmd/server
  rules:
    - changes:
        - services/order/**/*

build_payment_service:
  stage: build
  script:
    - cd services/payment && go build -o ../../bin/payment ./cmd/server
  rules:
    - changes:
        - services/payment/**/*

# .gitlab-ci.yml: Test parallelization
test_unit:
  stage: test
  parallel: 4               # Split into 4 parallel jobs
  script:
    # Go tests have built-in parallelism, -parallel controls concurrency
    - go test -parallel 4 -count=1 -coverprofile=coverage.out ./...
    # GitLab automatically distributes test cases across 4 parallel jobs

// Jenkinsfile: Parallel tests
stage('Test') {
    parallel {
        stage('Unit Tests') {
            steps {
                sh 'go test -short ./...'
            }
        }
        stage('Integration Tests') {
            steps {
                sh 'go test -run Integration ./test/integration/...'
            }
        }
        stage('Lint') {
            steps {
                sh 'golangci-lint run ./...'
            }
        }
    }
}

# .gitlab-ci.yml: Using needs to break stage dependencies
stages:
  - build
  - test
  - deploy

build:
  stage: build
  script: go build -o bin/app ./cmd/server

test_unit:
  stage: test
  needs: ["build"]          # Only depends on build, not other test jobs
  script: go test -short ./...

test_integration:
  stage: test
  needs: ["build"]          # Runs in parallel with test_unit
  script: go test -run Integration ./...

deploy_staging:
  stage: deploy
  needs: ["test_unit"]      # Deploy once unit tests pass, don't wait for integration
  script: kubectl apply -f k8s/staging/

# Dockerfile: Multi-stage build (Go project example)

# Stage 1: Build environment (full Go toolchain)
FROM golang:1.22-alpine AS builder

WORKDIR /app

# Key: Copy go.mod and go.sum first, then go mod download
# This way, as long as deps haven't changed, this layer's cache is hit
COPY go.mod go.sum ./
RUN go mod download

# Then copy source code (source changes often, but dep layer cache is unaffected)
COPY . .

# Compile (CGO disabled, static binary)
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /app/bin/server ./cmd/server

# Stage 2: Runtime environment (minimal image, no Go toolchain)
FROM gcr.io/distroless/static-debian12:nonroot

COPY --from=builder /app/bin/server /server

USER nonroot:nonroot
EXPOSE 8080
ENTRYPOINT ["/server"]

# .gitlab-ci.yml: Enable BuildKit
variables:
  DOCKER_BUILDKIT: "1"
  BUILDX_BUILDER: "default"

build_image:
  stage: package
  image: docker:24.0
  services:
    - docker:24.0-dind
  script:
    # Enable BuildKit
    - export DOCKER_BUILDKIT=1
    - docker build 
        --build-arg BUILDKIT_INLINE_CACHE=1 
        --cache-from type=registry,ref=$CI_REGISTRY_IMAGE:cache
        --cache-to type=registry,ref=$CI_REGISTRY_IMAGE:cache,mode=max
        -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA 
        -t $CI_REGISTRY_IMAGE:latest
        .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
    - docker push $CI_REGISTRY_IMAGE:latest

# K8s config hot reload (no Pod rebuild needed)
# After modifying ConfigMap, the config file in the Pod updates automatically
# (if using configmap volume)
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  config.yaml: |
    log_level: debug       # Changed from info to debug, no Pod rebuild needed
    max_connections: 1000    

# Only modify ConfigMap, do not trigger Pod rebuild
kubectl create configmap app-config --from-file=config.yaml -o yaml --dry-run=client | kubectl apply -f -

# If the app supports hot reload (e.g., Go's viper library), config takes effect immediately
# If not, restart Pods but skip image rebuild
kubectl rollout restart deployment/app

# Tuning rolling update parameters for different services
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-gateway
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      # API gateway: fast update (stateless service)
      maxUnavailable: 50%      # Replace half at once, fast but riskier
      maxSurge: 50%
  template:
    spec:
      containers:
      - name: app
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 2     # Don't wait 30s, check immediately if app starts fast
          periodSeconds: 2           # Check every 2s
          successThreshold: 1        # Ready after 1 success
          failureThreshold: 3        # Fail after 3 failures

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-service
spec:
  replicas: 6
  strategy:
    type: RollingUpdate
    rollingUpdate:
      # Payment service: conservative update (stateful, strong consistency)
      maxUnavailable: 0            # Never reduce available Pods
      maxSurge: 1                  # At most 1 extra Pod
  template:
    spec:
      containers:
      - name: app
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          successThreshold: 1
          failureThreshold: 5

# .gitlab-ci.yml: Optimized complete pipeline (target: under 5 minutes)

variables:
  DOCKER_DRIVER: overlay2
  DOCKER_BUILDKIT: "1"
  GOPROXY: "https://goproxy.cn,direct"

# Global cache
cache:
  key:
    files: [go.sum, package-lock.json]
    prefix: ${CI_COMMIT_REF_SLUG}
  paths:
    - .cache/go-build/
    - .cache/go-pkg/
    - node_modules/
  policy: pull-push           # Pull then push

stages:
  - check
  - build
  - test
  - package
  - deploy

# 1. Code check (lint + security scan, parallel)
lint:
  stage: check
  image: golangci/golangci-lint:v1.57
  script: golangci-lint run --timeout 2m ./...
  needs: []                   # No dependencies, start immediately

security_scan:
  stage: check
  image: aquasec/trivy:latest
  script: trivy fs --severity HIGH,CRITICAL --exit-code 1 .
  needs: []                   # Parallel with lint

# 2. Compile (with cache, completes in seconds)
build:
  stage: build
  image: golang:1.22-alpine
  variables:
    GOCACHE: ${CI_PROJECT_DIR}/.cache/go-build
    GOPATH: ${CI_PROJECT_DIR}/.cache/go-pkg
  script:
    - go mod download
    - go build -ldflags="-s -w" -o bin/server ./cmd/server
  artifacts:
    paths: [bin/]
    expire_in: 1 hour

# 3. Tests (split parallel, 4-way)
test_unit:
  stage: test
  image: golang:1.22-alpine
  parallel: 4
  needs: ["build"]
  script:
    - go test -parallel 4 -count=1 -coverprofile=cov.out ./...
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: cov.out

# 4. Image build (multi-stage + BuildKit + remote cache)
docker_build:
  stage: package
  image: docker:24.0
  services: [docker:24.0-dind]
  needs: ["build"]            # Does not wait for tests, runs in parallel
  script:
    - export DOCKER_BUILDKIT=1
    - docker build
        --cache-from type=registry,ref=$CI_REGISTRY_IMAGE:cache
        --cache-to type=registry,ref=$CI_REGISTRY_IMAGE:cache,mode=max
        -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
        .
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA

# 5. Deploy (progressive by environment)
deploy_staging:
  stage: deploy
  image: bitnami/kubectl:1.29
  needs: ["test_unit", "docker_build"]
  script:
    - kubectl set image deployment/app-staging
        app=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
    - kubectl rollout status deployment/app-staging --timeout=120s
  environment:
    name: staging

deploy_production:
  stage: deploy
  image: bitnami/kubectl:1.29
  needs: ["deploy_staging"]
  script:
    - kubectl set image deployment/app-prod
        app=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
    - kubectl rollout status deployment/app-prod --timeout=180s
  environment:
    name: production
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
      when: manual             # Production requires manual confirmation

// DAG scheduling engine core structure (simplified)
type TaskNode struct {
    ID           string
    ServiceName  string
    Priority     int          // 1=high (core business), 5=low
    DependsOn    []string     // Other Task IDs this depends on
    Status       TaskStatus   // pending / running / success / failed
    ResourceReq  ResourceSpec // CPU/memory requirements
}

type Scheduler struct {
    nodes      map[string]*TaskNode
    runnerPool chan *Runner         // Runner resource pool
    maxParallel int                  // Max parallelism
}

// Scheduling core logic: priority + dependency + resource, three-dimensional decision
func (s *Scheduler) Schedule() {
    for {
        // Find all pending tasks whose dependencies are satisfied
        ready := s.findReadyTasks()
        
        // Sort by priority
        sort.Slice(ready, func(i, j int) bool {
            return ready[i].Priority < ready[j].Priority
        })
        
        // Try to assign a runner to each ready task
        for _, task := range ready {
            runner := s.tryAcquireRunner(task.ResourceReq)
            if runner != nil {
                go s.execute(task, runner)
            }
        }
    }
}

# Alert if build duration exceeds 10 minutes
ci_build_duration_seconds{stage="total"} > 600

# Alert if cache hit rate drops below 50%
1 - (
  ci_build_cache_miss_total / ci_build_total
) < 0.5

# Wrong: Using commit SHA as cache key—different every commit, cache never hits
cache:
  key: ${CI_COMMIT_SHA}
  paths: [node_modules/]

# Correct: Use lockfile hash as cache key
cache:
  key:
    files: [package-lock.json]
  paths: [node_modules/]

# Wrong: Multiple parallel test jobs connect to the same database, data conflicts
# test_unit_job_1 INSERT INTO users (id=1)
# test_unit_job_2 INSERT INTO users (id=1)  → conflict

# Correct: Each parallel job gets its own database instance
test_unit:
  parallel: 4
  services:
    - name: postgres:15
      alias: db
      # Each parallel job automatically gets an independent db instance
  variables:
    DATABASE_URL: "postgres://test:test@db:5432/test_$CI_NODE_INDEX"

# Wrong: COPY source first, every code change triggers go mod download re-execution
COPY . .
RUN go mod download

# Correct: COPY dependency files first, then go mod download, then COPY source
COPY go.mod go.sum ./
RUN go mod download
COPY . .

# Dangerous: maxUnavailable: 100% means delete all old Pods before creating new ones
# If new Pods fail to start, service is completely down
strategy:
  rollingUpdate:
    maxUnavailable: 100%    # Do not do this
    maxSurge: 0

# Every build generates artifacts, never cleaned, disk fills up in a month
artifacts:
  paths: [bin/]
  expire_in: 1 hour        # Must set expiration