VictoriaMetrics 部署与实践：Prometheus 长期存储的高性能替代方案

# 数据压缩效果对比（基于 100 万活跃时间序列，30 天数据）

存储方案              磁盘占用      压缩比     内存占用
─────────────────────────────────────────────────────────
Prometheus (本地)     350 GB       1x        8 GB
Thanos (S3)           120 GB       2.9x      6 GB (含 Sidecar)
VictoriaMetrics        55 GB       6.4x      3 GB
VictoriaMetrics (集群)  58 GB       6.0x      3.2 GB (总计)

# VictoriaMetrics 支持的数据接入协议

Pull 模式（兼容 Prometheus）:
  └── vmagent → 抓取 Prometheus exporters → 写入 VM

Push 模式:
  ├── Prometheus remote_write → VM（最常用）
  ├── Graphite plaintext protocol
  ├── OpenTSDB telnet/HTTP
  ├── InfluxDB line protocol
  ├── OpenTelemetry
  └── CSV import

查询兼容:
  ├── PromQL（完全兼容）
  ├── MetricsQL（PromQL 超集，扩展功能）
  └── Grafana 原生集成

数据摄入 → Protobuf 编码序列化
         → 内存缓冲区（batch flush）
         → 标签索引构建（倒排索引 + TSID）
         → 列式压缩写入磁盘
         → 后台合并与降采样

/var/lib/victoria-metrics-data/
├── data/
│   ├── small/               # 小表分区（近期数据）
│   │   ├── 2024_01/        # 按月分区的数据块
│   │   │   ├── index.bin   # 倒排索引
│   │   │   ├── timestamps.bin
│   │   │   └── values.bin
│   │   └── ...
│   ├── big/                 # 大表分区（历史数据）
│   │   └── ...
│   └── indexdb/            # 索引数据库
│       ├── index.bin
│       └── metadata.json
├── metadata/
│   └── ...
└── snapshots/               # 快照（备份用）
    └── ...

                    ┌──────────────┐
                    │   vmagent    │  (可选：数据采集/分片/复制)
                    │   (N 个实例)  │
                    └──────┬───────┘
                           │
          ┌────────────────┼────────────────┐
          │                │                │
          ▼                ▼                ▼
  ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
  │  vminsert    │ │  vminsert    │ │  vminsert    │
  │  (实例 1)    │ │  (实例 2)    │ │  (实例 N)    │
  └──────┬───────┘ └──────┬───────┘ └──────┬───────┘
         │                │                │
         └────────────────┼────────────────┘
                          │
          ┌───────────────┼───────────────┐
          │               │               │
          ▼               ▼               ▼
  ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
  │ vmstorage    │ │ vmstorage    │ │ vmstorage    │
  │  (节点 1)    │ │  (节点 2)    │ │  (节点 N)    │
  │  数据存储     │ │  数据存储     │ │  数据存储     │
  └──────┬───────┘ └──────┬───────┘ └──────┬───────┘
         │                │                │
         └────────────────┼────────────────┘
                          │
          ┌───────────────┼───────────────┐
          │               │               │
          ▼               ▼               ▼
  ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
  │  vmselect    │ │  vmselect    │ │  vmselect    │
  │  (实例 1)    │ │  (实例 2)    │ │  (实例 N)    │
  └──────┬───────┘ └──────┬───────┘ └──────┬───────┘
         │                │                │
         └────────────────┼────────────────┘
                          │
                    ┌─────┴──────┐
                    │  Grafana    │
                    │  / Client   │
                    └────────────┘

#!/bin/bash
# VictoriaMetrics 单节点部署脚本

# 下载二进制
VM_VERSION="v1.115.0"
wget "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/victoria-metrics-linux-amd64-v${VM_VERSION#v}.tar.gz"
tar -xzf "victoria-metrics-linux-amd64-v${VM_VERSION#v}.tar.gz"

# 创建数据目录
mkdir -p /var/lib/victoria-metrics-data

# 启动单节点
./victoria-metrics-prod \
  -storageDataPath=/var/lib/victoria-metrics-data \
  -retentionPeriod=90d \
  -httpListenAddr=:8428 \
  -memory.allowedBytes=4GB \
  -search.maxConcurrentQueries=8 \
  > /var/log/victoria-metrics.log 2>&1 &

echo "VictoriaMetrics 单节点已启动，监听 8428 端口"

# /etc/systemd/system/victoria-metrics.service
[Unit]
Description=VictoriaMetrics Single Node
After=network.target

[Service]
Type=simple
User=victoria-metrics
Group=victoria-metrics
ExecStart=/usr/local/bin/victoria-metrics-prod \
  -storageDataPath=/var/lib/victoria-metrics-data \
  -retentionPeriod=90d \
  -httpListenAddr=:8428 \
  -memory.allowedBytes=4GB
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

# prometheus.yml — 添加 remote_write 配置
global:
  scrape_interval: 15s
  evaluation_interval: 15s

remote_write:
  - url: "http://victoria-metrics:8428/api/v1/write"
    queue_config:
      capacity: 10000
      max_samples_per_send: 2000
      batch_send_deadline: 5s
      min_backoff: 1s
      max_backoff: 30s
    # 高可用场景可配置多个远程写入端点
    # remote_timeout: 30s

# 保留原有抓取配置不变
scrape_configs:
  - job_name: "node-exporter"
    static_configs:
      - targets: ["node-exporter:9100"]

  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod

# docker-compose.yml
version: '3.8'

services:
  # --- vmstorage 节点（有状态，需持久化） ---
  vmstorage-1:
    image: victoriametrics/vmstorage:v1.115.0-cluster
    command:
      - '--storageDataPath=/storage'
      - '--retentionPeriod=180d'
      - '--httpListenAddr=:8482'
    volumes:
      - vmstorage-1-data:/storage
    ports:
      - "8482"
    restart: unless-stopped

  vmstorage-2:
    image: victoriametrics/vmstorage:v1.115.0-cluster
    command:
      - '--storageDataPath=/storage'
      - '--retentionPeriod=180d'
      - '--httpListenAddr=:8482'
    volumes:
      - vmstorage-2-data:/storage
    ports:
      - "8482"
    restart: unless-stopped

  # --- vminsert 节点（无状态） ---
  vminsert-1:
    image: victoriametrics/vminsert:v1.115.0-cluster
    command:
      - '--httpListenAddr=:8480'
      - '--storageNode=vmstorage-1:8400'
      - '--storageNode=vmstorage-2:8400'
    ports:
      - "8480"
    depends_on:
      - vmstorage-1
      - vmstorage-2
    restart: unless-stopped

  vminsert-2:
    image: victoriametrics/vminsert:v1.115.0-cluster
    command:
      - '--httpListenAddr=:8480'
      - '--storageNode=vmstorage-1:8400'
      - '--storageNode=vmstorage-2:8400'
    ports:
      - "8480"
    depends_on:
      - vmstorage-1
      - vmstorage-2
    restart: unless-stopped

  # --- vmselect 节点（无状态） ---
  vmselect-1:
    image: victoriametrics/vmselect:v1.115.0-cluster
    command:
      - '--httpListenAddr=:8481'
      - '--storageNode=vmstorage-1:8401'
      - '--storageNode=vmstorage-2:8401'
    ports:
      - "8481"
    depends_on:
      - vmstorage-1
      - vmstorage-2
    restart: unless-stopped

  vmselect-2:
    image: victoriametrics/vmselect:v1.115.0-cluster
    command:
      - '--httpListenAddr=:8481'
      - '--storageNode=vmstorage-1:8401'
      - '--storageNode=vmstorage-2:8401'
    ports:
      - "8481"
    depends_on:
      - vmstorage-1
      - vmstorage-2
    restart: unless-stopped

  # --- 负载均衡 ---
  lb-insert:
    image: nginx:alpine
    volumes:
      - ./nginx-insert.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "8480:8480"
    depends_on:
      - vminsert-1
      - vminsert-2
    restart: unless-stopped

  lb-select:
    image: nginx:alpine
    volumes:
      - ./nginx-select.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "8481:8481"
    depends_on:
      - vmselect-1
      - vmselect-2
    restart: unless-stopped

  # --- Grafana ---
  grafana:
    image: grafana/grafana:latest
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    ports:
      - "3000:3000"
    restart: unless-stopped

volumes:
  vmstorage-1-data:
  vmstorage-2-data:

# nginx-insert.conf
events {}

http {
    upstream vminsert {
        least_conn;
        server vminsert-1:8480;
        server vminsert-2:8480;
    }

    server {
        listen 8480;

        location / {
            proxy_pass http://vminsert;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_connect_timeout 10s;
            proxy_send_timeout 30s;
            proxy_read_timeout 30s;
        }
    }
}

# nginx-select.conf
events {}

http {
    upstream vmselect {
        least_conn;
        server vmselect-1:8481;
        server vmselect-2:8481;
    }

    server {
        listen 8481;

        location / {
            proxy_pass http://vmselect;
            proxy_set_header Host $host;
            proxy_connect_timeout 10s;
            proxy_send_timeout 60s;
            proxy_read_timeout 60s;
        }
    }
}

# vmcluster.yaml — 使用 VictoriaMetrics Operator
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMCluster
metadata:
  name: vm-cluster
  namespace: monitoring
spec:
  retentionPeriod: "180d"
  replicationFactor: 2

  # vmstorage 配置
  vmstorage:
    replicaCount: 3
    storageDataPath: "/vm-data"
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: fast-ssd
          resources:
            requests:
              storage: 500Gi
    resources:
      limits:
        cpu: 4
        memory: 16Gi
      requests:
        cpu: 2
        memory: 8Gi

  # vminsert 配置
  vminsert:
    replicaCount: 2
    resources:
      limits:
        cpu: 2
        memory: 4Gi
      requests:
        cpu: 1
        memory: 2Gi

  # vmselect 配置
  vmselect:
    replicaCount: 2
    resources:
      limits:
        cpu: 2
        memory: 4Gi
      requests:
        cpu: 1
        memory: 2Gi
    cacheMountPath: "/cache"
    storage:
      volumeClaimTemplate:
        spec:
          resources:
            requests:
              storage: 10Gi
---
# vmagent 配置 — 替代 Prometheus 抓取
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAgent
metadata:
  name: vm-agent
  namespace: monitoring
spec:
  replicaCount: 2
  serviceScrapeNamespaceSelector: {}
  podScrapeNamespaceSelector: {}
  nodeScrapeNamespaceSelector: {}
  staticScrapeNamespaceSelector: {}
  remoteWrite:
    - url: "http://vm-cluster-vminsert.monitoring.svc:8480/insert/0/prometheus/api/v1/write"
  resources:
    limits:
      cpu: 1
      memory: 1Gi
    requests:
      cpu: 500m
      memory: 512Mi
---
# vmalert 配置 — 告警规则评估
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: vm-alert
  namespace: monitoring
spec:
  replicaCount: 2
  datasource:
    url: "http://vm-cluster-vmselect.monitoring.svc:8481/select/0/prometheus"
  notifier:
    url: "http://alertmanager.monitoring.svc:9093"
  evaluationInterval: "30s"
  ruleNamespaceSelector: {}
  resources:
    limits:
      cpu: 1
      memory: 1Gi
    requests:
      cpu: 500m
      memory: 512Mi

# vmagent 配置示例
global:
  scrape_interval: 15s
  external_labels:
    cluster: "production"
    region: "us-east-1"

# 抓取配置（与 Prometheus 完全兼容）
scrape_configs:
  - job_name: "kubernetes-nodes"
    kubernetes_sd_configs:
      - role: node
    relabel_configs:
      - source_labels: [__address__]
        regex: "(.*):.*"
        target_label: __address__
        replacement: "${1}:9100"

  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"

# 远程写入 VictoriaMetrics
remote_write:
  - url: "http://vminsert:8480/insert/0/prometheus/api/v1/write"
    # 高可用：写入多个 vminsert 实例
    # vmagent 会自动处理重复数据

# 多实例 vmagent 分片抓取（每个实例只抓取一部分目标）
# 实例 1
./vmagent-prod \
  -promscrape.config=/etc/vmagent/scrape.yml \
  -remoteWrite.url=http://vminsert:8480/insert/0/prometheus/api/v1/write \
  -promscrape.cluster.membersCount=2 \
  -promscrape.cluster.memberNum=0

# 实例 2
./vmagent-prod \
  -promscrape.config=/etc/vmagent/scrape.yml \
  -remoteWrite.url=http://vminsert:8480/insert/0/prometheus/api/v1/write \
  -promscrape.cluster.membersCount=2 \
  -promscrape.cluster.memberNum=1

# 1. 在 Prometheus 端创建 TSDB 快照
curl -XPOST http://prometheus:9090/api/v1/admin/tsdb/snapshot

# 2. 使用 vmrestore 将快照导入 VM
./vmrestore-prod \
  -src=s3://my-bucket/prom-snapshots/ \
  -dst=/var/lib/victoria-metrics-data/

# vmctl 是官方数据迁移工具，支持多种数据源

# 从 Prometheus TSDB 迁移
./vmctl-prod prometheus \
  -src.address=http://prometheus:9090 \
  -dst.url=http://victoria-metrics:8428 \
  -dst.addr=http://victoria-metrics:8428

# 从 InfluxDB 迁移
./vmctl-prod influxdb \
  -src.addr=http://influxdb:8086 \
  -src.database=monitoring \
  -dst.url=http://victoria-metrics:8428

# 从 OpenTSDB 迁移
./vmctl-prod opentsdb \
  -src.addr=http://opentsdb:4242 \
  -dst.url=http://victoria-metrics:8428

# 从远程 Prometheus 兼容存储迁移
./vmctl-prod remote \
  -src.addr=http://remote-storage:9090 \
  -dst.url=http://victoria-metrics:8428 \
  -time-filter='{"start":"2025-01-01T00:00:00Z","end":"2026-07-01T00:00:00Z"}'

# 过渡期：同时写入 Prometheus 本地和 VictoriaMetrics
# prometheus.yml
remote_write:
  - url: "http://victoria-metrics:8428/api/v1/write"
    queue_config:
      capacity: 10000

# 步骤：
# 1. 配置 Prometheus remote_write 到 VM（开始双写）
# 2. 观察数据一致性，确认无丢失
# 3. 迁移历史数据（vmctl）
# 4. 验证历史数据完整
# 5. 切换 Grafana 数据源到 VM
# 6. 确认稳定后停用 Prometheus remote_write
# 7. 最终用 vmagent 完全替代 Prometheus

# Prometheus remote_write 调优
remote_write:
  - url: "http://victoria-metrics:8428/api/v1/write"
    queue_config:
      capacity: 25000          # 队列容量（默认 10000）
      max_samples_per_send: 5000  # 每批发送样本数（默认 100）
      batch_send_deadline: 2s   # 批量发送超时（默认 5s）
      min_backoff: 500ms       # 最小重试间隔（默认 1s）
      max_backoff: 10s         # 最大重试间隔（默认 30s）
    remote_timeout: 30s

# vmstorage 内存分配
./victoria-metrics-prod \
  -memory.allowedBytes=8GB \
  # VM 会使用允许内存的约 60% 作为缓存
  # 剩余留给操作系统和其他进程
  
  # 缓存大小调优
  -cacheExpireDuration=6h \
  -dedup.minScrapeInterval=30s
  
  # 写入优化
  -insert.maxQueueDuration=1m \
  -insert.maxBlockDuration=5m

# 检查高基数指标
# 按时间序列数量排序，找出最占资源的指标
topk(20, count by (__name__)({__name__=~".+"}))

# 检查标签基数
topk(20, count by (__name__, job)({__name__=~".+"}))

# 找出标签组合爆炸的指标
topk(10, count by (__name__)({__name__=~"http_request_.*"}))

# 标准 PromQL 查询
rate(http_requests_total[5m])

# MetricsQL 优化版 — 使用 range_first / range_last
# 只取窗口起始和结束值，减少计算量
# 适合 counter 类型的大范围查询
rate(http_requests_total[5m] @ end())

# MetricsQL 的 keep_metric_names 修饰符
# 保留原始指标名，便于聚合后识别
sum by (job) (rate(http_requests_total[5m])) keep_metric_names

# MetricsQL 的 lag() 函数
# 处理延迟到达的数据，避免计算偏差
rate(http_requests_total[5m] lag(30s))

# 原始查询（30 天数据，每 15s 一个点 = 172800 个点）
rate(cpu_usage[30d])

# 使用 MetricsQL 的 downgrade
# 将 30 天数据降采样到 1 小时粒度 = 720 个点
# 大幅减少查询数据量和响应时间
rate(cpu_usage[30d:1h])

# 手动降采样
# 每小时取一个平均值
avg_over_time(rate(cpu_usage[5m])[30d:1h])

# vmselect 启用查询缓存
./vmselect-prod \
  -cacheDataPath=/cache \
  # 查询结果缓存
  -search.cacheSize=2GB \
  -search.cacheTTL=5m \
  # 索引缓存
  -search.indexCacheSize=1GB \
  # 过滤缓存
  -search.filterCacheSize=1GB

# vmstorage 存储优化参数
./vmstorage-prod \
  -storageDataPath=/vm-data \
  -retentionPeriod=180d \
  
  # 降采样配置
  -downsampling.period=30d:5m,180d:1h \
  # 含义：30天以上数据降采样到5分钟粒度，180天以上降采样到1小时
  
  # 数据去重
  -dedup.minScrapeInterval=30s \
  # 当多个 vmagent 采集相同目标时，自动去重
  
  # 索引优化
  -index.maxSeriesPerIndexBlock=300000 \
  
  # 内存管理
  -memory.allowedBytes=16GB

# vminsert 配置数据复制
# replicationFactor=2 表示每条数据写入 2 个 vmstorage 节点
./vminsert-prod \
  -httpListenAddr=:8480 \
  -storageNode=vmstorage-1:8400 \
  -storageNode=vmstorage-2:8400 \
  -storageNode=vmstorage-3:8400 \
  -replicationFactor=2

# 复制架构（replicationFactor=2, 3 个 storage 节点）

写入请求 → vminsert
              │
              ├──→ vmstorage-1 (主)  ──→ vmstorage-2 (副本)  ✓ 成功
              ├──→ vmstorage-2 (主)  ──→ vmstorage-3 (副本)  ✓ 成功
              └──→ vmstorage-3 (主)  ──→ vmstorage-1 (副本)  ✓ 成功

# vmstorage-1 宕机后
# vminsert 自动将数据写入 vmstorage-2 和 vmstorage-3
# 查询时 vmselect 从存活的节点获取数据，无感知

#!/bin/bash
# VictoriaMetrics 备份脚本

# 1. 创建快照
SNAPSHOT_PATH=$(curl -s -X POST http://localhost:8428/snapshot/create | jq -r '.snapshot')
echo "创建快照: ${SNAPSHOT_PATH}"

# 2. 使用 vmbackup 推送到 S3
./vmbackup-prod \
  -storageDataPath=/var/lib/victoria-metrics-data \
  -snapshotName="${SNAPSHOT_PATH}" \
  -dst=s3://monitoring-backup/vm-snapshots/$(date +%Y%m%d)/

# 3. 验证备份完整性
./vmbackupmanager-prod verify \
  -dst=s3://monitoring-backup/vm-snapshots/$(date +%Y%m%d)/

# 4. 清理旧快照（保留最近 7 天）
curl -X POST "http://localhost:8428/snapshot/delete?keep=7"

echo "备份完成"

# 恢复流程
./vmrestore-prod \
  -src=s3://monitoring-backup/vm-snapshots/20260711/ \
  -dst=/var/lib/victoria-metrics-data/

# Grafana 数据源配置（Provisioning）
apiVersion: 1
datasources:
  # VictoriaMetrics 作为 Prometheus 数据源
  - name: VictoriaMetrics
    type: prometheus
    access: proxy
    url: http://vmselect:8481/select/0/prometheus/
    isDefault: true
    jsonData:
      timeInterval: "15s"
      httpMethod: "POST"
      # 启用 MetricsQL 扩展
      customQueryParameters: "extra_label=cluster=production"

  # 如果使用多租户
  - name: VictoriaMetrics (tenant-a)
    type: prometheus
    access: proxy
    url: http://vmselect:8481/select/tenant-a/prometheus/
    jsonData:
      timeInterval: "15s"

# 导入官方 Dashboard（Grafana → Import → ID）

Dashboard ID    说明
────────────────────────────────────
11176          VictoriaMetrics 单节点概览
14289          VictoriaMetrics 集群概览
14592          vmagent 状态概览
14594          vmalert 规则执行概览
14595          vmrestore 备份概览
14596          vmbackup 备份状态

#!/usr/bin/env python3
"""
VictoriaMetrics 存储容量估算工具
基于活跃时间序列数量、保留期和采样间隔进行估算
"""

def estimate_storage(
    active_series: int,
    retention_days: int,
    scrape_interval_sec: int = 15,
    avg_label_size: int = 100,  # 每条序列的平均标签字节数
):
    """
    估算 VictoriaMetrics 所需的磁盘空间
    
    参数:
        active_series: 活跃时间序列数量
        retention_days: 数据保留天数
        scrape_interval_sec: 采集间隔（秒）
        avg_label_size: 平均标签字节数
    """
    # 每个数据点大小（VM 压缩后约 1-2 字节）
    bytes_per_point = 1.5
    
    # 计算总数据点数
    points_per_series_per_day = 86400 / scrape_interval_sec
    total_points = active_series * points_per_series_per_day * retention_days
    
    # 数据点存储空间
    data_storage = total_points * bytes_per_point
    
    # 索引空间（约为数据空间的 20-30%）
    index_storage = data_storage * 0.25
    
    # 标签存储空间
    label_storage = active_series * avg_label_size * 2  # 压缩后
    
    # 总存储空间
    total_storage = data_storage + index_storage + label_storage
    
    # 额外开销（ WAL、临时文件等，约 10%）
    total_with_overhead = total_storage * 1.1
    
    return {
        "active_series": active_series,
        "retention_days": retention_days,
        "scrape_interval_sec": scrape_interval_sec,
        "total_points": int(total_points),
        "data_storage_gb": data_storage / 1024**3,
        "index_storage_gb": index_storage / 1024**3,
        "label_storage_gb": label_storage / 1024**3,
        "total_storage_gb": total_with_overhead / 1024**3,
    }

# 示例计算
configs = [
    ("小型", 100_000, 90, 15),
    ("中型", 1_000_000, 180, 15),
    ("大型", 5_000_000, 180, 15),
    ("超大", 20_000_000, 365, 30),
]

print(f"{'规模':<8} {'序列数':>12} {'保留天数':>8} {'存储(GB)':>10} {'存储(TB)':>10}")
print("-" * 55)

for name, series, days, interval in configs:
    result = estimate_storage(series, days, interval)
    print(f"{name:<8} {series:>12,} {days:>8} {result['total_storage_gb']:>10.1f} {result['total_storage_gb']/1024:>10.2f}")

规模       序列数    保留天数   存储(GB)   存储(TB)
-------------------------------------------------------
小型        100,000       90        1.2       0.00
中型      1,000,000      180       25.6       0.02
大型      5,000,000      180      128.0       0.13
超大     20,000,000      365      768.0       0.80

# vmagent 抓取 VictoriaMetrics 自身指标的配置
scrape_configs:
  - job_name: "victoria-metrics"
    static_configs:
      - targets: ["victoria-metrics:8428"]
    # VM 在 /metrics 端点暴露自身指标

  - job_name: "vmstorage"
    static_configs:
      - targets: ["vmstorage-1:8482", "vmstorage-2:8482"]

  - job_name: "vminsert"
    static_configs:
      - targets: ["vminsert-1:8480", "vminsert-2:8480"]

  - job_name: "vmselect"
    static_configs:
      - targets: ["vmselect-1:8481", "vmselect-2:8481"]

# vmalert 规则 — VictoriaMetrics 自身健康监控
groups:
  - name: victoriametrics-alerts
    rules:
      # vmstorage 磁盘使用率
      - alert: VMStorageDiskUsageHigh
        expr: |
          100 * (1 - vm_data_disk_free_bytes / vm_data_disk_total_bytes) > 85          
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "VM storage 磁盘使用率超过 85%"
          description: "节点 {{ $labels.instance }} 磁盘使用率: {{ $value }}%"

      # 写入速率异常
      - alert: VMIngestionRateDrop
        expr: |
          rate(vm_rows_ingested_total[5m]) < 100          
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "VM 数据写入速率异常下降"
          description: "当前写入速率: {{ $value }} rows/s"

      # 查询延迟过高
      - alert: VMSlowQueries
        expr: |
          histogram_quantile(0.95, 
            rate(vm_select_query_duration_seconds_bucket[5m])
          ) > 10          
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "VM 查询 P95 延迟超过 10 秒"
          description: "P95 延迟: {{ $value }}s"

      # 内存使用过高
      - alert: VMHighMemoryUsage
        expr: |
          100 * vm_memory_bytes / vm_memory_allowed_bytes > 90          
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "VM 内存使用率超过 90%"
          description: "内存使用: {{ $value }}%"

      # 节点不可达
      - alert: VMNodeDown
        expr: up{job=~"victoria-metrics|vmstorage|vminsert|vmselect"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "VM 节点不可达"
          description: "{{ $labels.instance }} 不可达"