IaC 不测试就上生产？Terraform 四层验证体系让凌晨告警少 80%

resource "aws_instance" "web" {
  ami           = "ami-12345678"        # 这个 AMI 在 ap-northeast-1 存在
  instance_type = "t2.nano"              # 但 t2.nano 在 us-east-2 不支持
  region        = "us-east-2"            # terraform validate 完全不报错
}

# macOS
brew install tflint

# Linux (amd64)
curl -s https://raw.githubusercontent.com/terraform-linters/tflint/master/install_linux.sh | bash

# 初始化规则集（必须执行，否则不加载任何插件）
tflint --init

plugin "aws" {
  enabled = true
  version = "0.27.0"
  source  = "github.com/terraform-linters/tflint-ruleset-aws"
}

# 关闭不需要的规则（减少噪音）
rule "aws_instance_invalid_type" {
  enabled = true
}

# 关闭命名规范检查（如果你有自己的规范）
rule "terraform_naming_convention" {
  enabled = false
}

# 强制使用最新语法
rule "terraform_deprecated_interpolation" {
  enabled = true
}

# 检查未使用的变量
rule "terraform_unused_declarations" {
  enabled = true
}

- name: Setup tflint
  uses: tflint/setup-tflint@v3

- name: Run tflint
  run: tflint --config=.tflint.hcl --format=compact

# 安装
pip install checkov

# 基础扫描
checkov -d .

# 只扫描特定框架
checkov -d . --framework terraform

# 输出 SARIF 格式（用于 GitHub Code Scanning）
checkov -d . --output sarif > results.sarif

# 只跑特定检查（如只检查 S3 相关）
checkov -d . --check CKV_AWS_S3_*

# 跳过特定检查（误报太多时）
checkov -d . --skip-check CKV_AWS_18

# .checkov.yaml（项目级配置）
framework:
  - terraform

# 启用的检查类别
checks:
  - CKV_AWS_18   # S3 bucket logging
  - CKV_AWS_19   # S3 bucket encryption
  - CKV_AWS_338  # ECR image scanning
  - CKV_AWS_40   # IAM policy attached to role not user
  - CKV_AWS_52   # S3 bucket MFA delete
  - CKV_AWS_145  # S3 bucket kms encryption
  - CKV_AWS_28   # DynamoDB KMS encryption
  - CKV_AWS_21   # S3 versioning enabled

# 跳过的检查（业务上不需要或已知误报）
skip-checks:
  - CKV_AWS_117   # Aurora cluster encryption（业务不需要）
  - CKV_AWS_136   # ECR tag policy（团队不用 tag 策略）

# 跳过特定目录（第三方模块不扫）
skip-path:
  - .terraform/
  - modules/third-party/

# GitHub Actions 示例
- name: Run checkov
  env:
    CHECKOV_WORKERS_NUMBER: 4
  run: |
    checkov -d . --framework terraform \
      --skip-path .terraform/ \
      --skip-path modules/third-party/ \
      --output cli --soft-fail

# tests/main.tftest.hcl

variables {
  region         = "us-east-1"
  instance_type  = "t3.micro"
  environment    = "test"
  vpc_cidr       = "10.0.0.0/16"
}

# 测试1：验证 VPC CIDR 配置正确
run "validate_vpc_cidr" {
  command = plan

  assert {
    condition     = aws_vpc.main.cidr_block == var.vpc_cidr
    error_message = "VPC CIDR should match the variable"
  }
}

# 测试2：验证子网数量正确
run "validate_subnet_count" {
  command = plan

  assert {
    condition     = length(aws_subnet.public) == 3
    error_message = "Should create 3 public subnets for HA"
  }
}

# 测试3：验证安全组不暴露 22 端口
run "no_ssh_to_world" {
  command = plan

  assert {
    condition     = !anytrue([for rule in aws_security_group.web.ingress : contains(rule.cidr_blocks, "0.0.0.0/0") if rule.from_port == 22])
    error_message = "Security group must not allow SSH from 0.0.0.0/0"
  }
}

# 测试4：实际 apply 后验证（需要真实云凭证）
run "deploy_and_verify" {
  command = apply

  assert {
    condition     = aws_instance.web.id != ""
    error_message = "Instance should have an ID after creation"
  }
}

# 运行所有测试
terraform test

# 指定测试目录
terraform test -test-directory=tests/

# JSON 格式输出（用于 CI 解析）
terraform test -json

// test/infrastructure_test.go
package test

import (
	"crypto/tls"
	"fmt"
	"testing"
	"time"

	"github.com/gruntwork-io/terratest/modules/aws"
	http_helper "github.com/gruntwork-io/terratest/modules/http-helper"
	"github.com/gruntwork-io/terratest/modules/terraform"
)

func TestWebAppInfrastructure(t *testing.T) {
	t.Parallel()

	// 1. 构造 Terraform 选项
	terraformOptions := &terraform.Options{
		// Terraform 代码路径
		TerraformDir: "../modules/web-app",

		// 传入测试变量
		Vars: map[string]interface{}{
			"region":         "us-east-1",
			"environment":    "test",
			"instance_type":  "t3.micro",
			"min_size":       2,
			"max_size":       4,
			"health_check_path": "/health",
		},

		// 测试结束后自动销毁（即使测试失败也会执行）
		// 这是 Terratest 最关键的设计——不会留下垃圾资源
	},

	// 2. 部署基础设施，测试结束后销毁
	defer terraform.Destroy(t, terraformOptions)
	terraform.InitAndApply(t, terraformOptions)

	// 3. 获取部署后的输出
	albDnsName := terraform.Output(t, terraformOptions, "alb_dns_name")
	url := fmt.Sprintf("http://%s/health", albDnsName)

	// 4. 验证：ALB 健康检查通过，HTTP 200
	// 重试 30 次，每次间隔 10 秒（给 ASG 扩容时间）
	maxRetries := 30
	timeBetweenRetries := 10 * time.Second

	http_helper.HttpGetWithRetryWithCustomValidation(
		t,
		url,
		&tls.Config{InsecureSkipVerify: true},
		maxRetries,
		timeBetweenRetries,
		func(statusCode int, body string) bool {
			return statusCode == 200 && body == "OK"
		},
	)
}

// 强制使用最便宜的实例类型
Vars: map[string]interface{}{
    "instance_type":    "t3.micro",   // 最小规格
    "disk_size":         8,            // 最小磁盘
    "retention_days":    1,            // 日志保留1天
},

// 设置超时，防止卡住导致资源不销毁
terraformOptions.MaxRetries = 3,
terraformOptions.RetrySleepSeconds = 10,

// 每个测试用例加唯一后缀，避免资源名冲突
uniqueId := random.UniqueId()
terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
    TerraformDir: "../modules/web-app",
    Vars: map[string]interface{}{
        "name_prefix": fmt.Sprintf("terratest-%s-", uniqueId),
    },
})

# 第一层：CI 级别，定时清理残留资源
# crontab: 每天凌晨 3 点执行
0 3 * * * aws ec2 describe-instances --filters "Name=tag:CreatedBy,Values=terratest" \
    --query 'Reservations[].Instances[?State.Name==`running`].InstanceId' \
    --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {}

# 第二层：Terratest 代码级别，强制超时
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
go func() {
    terraform.InitAndApply(t, terraformOptions)
    defer terraform.Destroy(t, terraformOptions)
    cancel()
}()
<-ctx.Done()

# .github/workflows/terraform-ci.yml
name: Terraform CI

on:
  pull_request:
    paths:
      - '**.tf'
      - '**.tfvars'

env:
  TF_VERSION: '1.9.5'
  AWS_REGION: 'us-east-1'

jobs:
  # L1: 静态检查（最快，先跑）
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: ${{ env.TF_VERSION }}

      - name: terraform fmt
        run: terraform fmt -check -recursive

      - name: Setup tflint
        uses: tflint/setup-tflint@v3

      - name: tflint init
        run: tflint --init --config=.tflint.hcl

      - name: Run tflint
        run: tflint --config=.tflint.hcl --format=compact

  # L2: 安全扫描
  security:
    runs-on: ubuntu-latest
    env:
      CHECKOV_WORKERS_NUMBER: 4
    steps:
      - uses: actions/checkout@v4

      - name: Run checkov
        uses: bridgecrewio/checkov-action@v12
        with:
          directory: .
          framework: terraform
          skip_path: .terraform/,modules/third-party/
          output_format: cli
          soft_fail: true  # 先软告警，团队习惯后改 false

  # L3: 计划验证（需要云凭证）
  plan:
    needs: [lint, security]
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v4

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: ${{ env.TF_VERSION }}

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::123456789012:role/github-actions-tf
          aws-region: ${{ env.AWS_REGION }}

      - name: terraform init
        run: terraform init -backend=false

      - name: terraform validate
        run: terraform validate

      - name: terraform test
        run: terraform test -test-directory=tests/

      - name: terraform plan
        run: terraform plan -no-color -out=tfplan
        continue-on-error: true

      - name: Post plan to PR
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const plan = fs.readFileSync('tfplan', 'utf8');
            github.rest.issues.createComment({
              ...context.repo,
              issue_number: context.issue.number,
              body: `## Terraform Plan\n\`\`\`\n${plan}\n\`\`\``
            });            

  # L4: 集成测试（仅主分支 PR 触发，成本最高）
  integration:
    needs: plan
    if: github.base_ref == 'main'
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-go@v5
        with:
          go-version: '1.22'

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: ${{ env.TF_VERSION }}

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::123456789012:role/github-actions-tf-test
          aws-region: ${{ env.AWS_REGION }}

      - name: Run Terratest
        run: |
          cd test
          go test -v -timeout 900s -parallel 2          

stages:
  - lint
  - security
  - plan
  - integration

variables:
  TF_VERSION: "1.9.5"

tflint:
  stage: lint
  image: hashicorp/terraform:$TF_VERSION
  script:
    - apk add --no-cache curl
    - curl -sL https://github.com/terraform-linters/tflint/releases/latest/download/tflint_linux_amd64.zip -o /tmp/tflint.zip
    - unzip /tmp/tflint.zip -d /usr/local/bin
    - tflint --init --config=.tflint.hcl
    - tflint --config=.tflint.hcl --format=compact
  rules:
    - changes: ["**/*.tf"]

checkov:
  stage: security
  image: bridgecrew/checkov:latest
  script:
    - checkov -d . --framework terraform --soft-fail
  rules:
    - changes: ["**/*.tf"]

terraform-plan:
  stage: plan
  image: hashicorp/terraform:$TF_VERSION
  script:
    - terraform init -backend=false
    - terraform validate
    - terraform plan -no-color
  rules:
    - changes: ["**/*.tf"]

# policy/s3_policy.rego
package main

# 规则1：所有 S3 bucket 必须开启版本控制
deny[msg] {
  resource := input.resource.aws_s3_bucket[_]
  not resource.versioning.enabled
  msg := sprintf("S3 bucket '%s' must have versioning enabled", [resource.name])
}

# 规则2：所有 S3 bucket 必须开启 KMS 加密
deny[msg] {
  resource := input.resource.aws_s3_bucket[_]
  not resource.server_side_encryption_configuration
  msg := sprintf("S3 bucket '%s' must have KMS encryption enabled", [resource.name])
}

# 规则3：禁止创建 0.0.0.0/0 的安全组入站规则
deny[msg] {
  resource := input.resource.aws_security_group_rule[_]
  resource.type == "ingress"
  contains(resource.cidr_blocks, "0.0.0.0/0")
  msg := sprintf("Security group rule '%s' allows 0.0.0.0/0 ingress", [resource.name])
}

# 规则4：强制所有资源必须有 Environment 标签
deny[msg] {
  resource := input.resource[_][_]
  not resource.tags.Environment
  msg := sprintf("Resource '%s' must have Environment tag", [resource.name])
}

# 先把 terraform plan 输出为 JSON
terraform plan -out=tfplan
terraform show -json tfplan > tfplan.json

# 用 conftest 验证
conftest test tfplan.json --policy policy/

# policy/environment_specific.rego
package main

# 生产环境的安全组不允许任何入站 22 端口
deny[msg] {
  input.resource.aws_security_group_rule[_].from_port == 22
  input.resource.aws_security_group_rule[_].type == "ingress"
  input.variables.environment == "production"
  msg := "SSH (port 22) ingress is forbidden in production"
}

# 非生产环境允许 SSH 但限制来源
warn[msg] {
  rule := input.resource.aws_security_group_rule[_]
  rule.from_port == 22
  rule.type == "ingress"
  input.variables.environment != "production"
  not contains(rule.cidr_blocks[0], "10.")
  msg := "SSH access should be restricted to internal IPs"
}

$ terraform test
Success! 0 passed, 0 failed.

# 查看锁信息
terraform force-unlock <LOCK_ID>

# 根因是上一次测试没跑完。检查 CI 日志，确保 terraform destroy 总是执行
# Terratest 的 defer terraform.Destroy 会保证销毁，但如果进程被 kill -9 就不行了

skip-path:
  - .terraform/
  - modules/third-party/
  - examples/

# checkov:skip=CKV_AWS_18:This is a test bucket, logging not needed
resource "aws_s3_bucket" "test" {
  bucket = "my-test-bucket-12345"
}

# 确认插件配置
cat .tflint.hcl | grep source

# 重新初始化
tflint --init --config=.tflint.hcl

# 如果仍然失败，检查网络是否能访问 GitHub
curl -sI https://github.com/terraform-linters/tflint-ruleset-aws