Prometheus Monitoring Stack with Docker Compose

# 一键安装 Docker
curl -fsSL https://get.docker.com | sh

# 查看 Docker 版本
docker --version

# 查看 Docker Compose 版本
docker compose version

mkdir monitoring
cd monitoring

mkdir -p prometheus_data rules grafana_data

global:
  scrape_interval: 15s # 默认抓取间隔
  evaluation_interval: 15s # 默认规则评估间隔

rule_files:
  - "/etc/prometheus/rules/*.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093'] # 指向 Alertmanager 的地址

scrape_configs:
  # Job 1: 监控无需 Basic Auth 的域名证书
  - job_name: 'blackbox_ssl_no_auth'
    metrics_path: /probe
    # 引用无需 Basic Auth 的模块
    params:
      module: [http_2xx_ssl_no_auth]

    # Blackbox Exporter 的地址和端口
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115 # Blackbox Exporter 的实际地址和端口

    # 放置无需 Basic Auth 的域名
    static_configs:
      - targets:
          - https://google.com
          # 添加其他无需 Basic Auth 的域名


  # 抓取 Prometheus 自身指标
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
        labels:
          instance: 'prometheus-server'
          service: 'self-monitoring'

  # 示例：如果要监控其他远程服务器上的 Node Exporter
  - job_name: 'node_exporter'
    static_configs:
      - targets: ['node-exporter:9100'] #  test 宿主机物理内网 IP
        labels:
          instance: 'test'
          location: 'https://text.com/'


    relabel_configs:
      - source_labels: [__address__]
        # 使用命名捕获组 ip_v6 和 ip_v4
        regex: '(?:\[([0-9a-fA-F:]+)\]|([^:]+)):[0-9]+'
        target_label: ip
        replacement: '$1$2'

global:
  resolve_timeout: 5m  # 默认情况下，告警解决的超时时间

route:
  group_by: ['alertname']  # 按照 `alertname` 来分组告警
  group_wait: 30s          # 收到第一个告警后，等待30s看有没有同组告警一起打包
  group_interval: 3m       # 🌟 调整为3m：同组内有新老告警交替时，至少间隔3分钟再发新通知
  repeat_interval: 30m      # 🌟 调整为3h：同一个告警如果没有解决，每隔3小时重复提醒一次，防止轰炸
  receiver: 'telegram'

receivers:
  - name: 'telegram'
    telegram_configs:
      - chat_id: xxx
        bot_token: 'xxx'
        parse_mode: 'HTML'
        send_resolved: true
        message: |
          {{ if eq .Status "firing" }}
          🚨 <b>严重告警</b> 🚨
          ⚠️ <b>告警名称:</b> {{ .CommonLabels.alertname | html }} (共 {{ .Alerts | len }} 个实例)
          {{ range .Alerts }}
          ----------------------------------------
          ⚠️ <b> IP地址:</b> {{ .Labels.ip | html }}
          ⚠️ <b>数据中心:</b> {{ .Labels.location | html }}
          ⚠️ <b>详细描述:</b> {{ .Annotations.description | html }}
          ⚠️ <b>通知时间:</b> {{ .StartsAt.Format "2006-01-02 15:04:05" }}
          ----------------------------------------
          {{ end }}
          {{ else }}
          ✅ <b>严重告警恢复</b> ✅
          <b>告警名称:</b> {{ .CommonLabels.alertname | html }} (共 {{ .Alerts | len }} 个实例)
          {{ range .Alerts }}
          ----------------------------------------
          <b> IP地址:</b> {{ .Labels.ip | html }}
          <b>数据中心:</b> {{ .Labels.location | html }}
          <b>详细描述:</b> {{ .Annotations.description | html }}
          <b>恢复时间:</b> {{ .EndsAt.Format "2006-01-02 15:04:05" }}
          ----------------------------------------
          {{ end }}
          {{ end }}

modules:
  # 🌟 模块1: 用于无需 Basic Auth 的 HTTPS 证书与存活监控
  http_2xx_ssl_no_auth:
    prober: http
    timeout: 5s
    http:
      method: GET
      # 🌟 核心优化：禁止跟随重定向。精准锁定并验证当前域名的证书，防止跳转到 CDN 或外部域名
      no_follow_redirects: true
      # 🌟 核心优化：将 2xx 和 3xx (重定向) 都视为成功，防止因反代返回 301/302 导致 Prometheus 误报网站离线
      valid_status_codes: [200, 201, 202, 204, 301, 302, 303, 307, 308]
      preferred_ip_protocol: "ip4"
      tls_config:
        insecure_skip_verify: false # 严格验证证书有效性，这是获取证书过期时间的前提

  # 🌟 模块2: 用于需要经过 NPM Basic Auth 认证的内部域名证书与存活监控
  http_2xx_ssl_with_auth:
    prober: http
    timeout: 5s
    http:
      method: GET
      # 🌟 核心优化：禁止跟随重定向
      no_follow_redirects: true
      # 🌟 核心优化：接受 2xx 和 3xx 状态码
      valid_status_codes: [200, 201, 202, 204, 301, 302, 303, 307, 308]
      preferred_ip_protocol: "ip4"
      tls_config:
        insecure_skip_verify: false
      # 密码已暴露，记得去 NPM 后台修改一把新密码再填进来
      basic_auth:
        username: "admin-s"
        password: "ojbKRob3GoY23xBi5pkTtT"

route:
  # Default receiver
  receiver: 'operations-team'
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

  # Specific routing rules
  routes:
  - match:
      severity: critical
    receiver: 'pager-duty'
    repeat_interval: 1h
    continue: true

  - match_re:
      service: database|redis|elasticsearch
    receiver: 'database-team'

  - match_re:
      service: frontend|api
    receiver: 'application-team'

receivers:
- name: 'operations-team'
  email_configs:
  - to: 'ops@example.com'

- name: 'pager-duty'
  pagerduty_configs:
  - service_key: 'your-pagerduty-key'

- name: 'database-team'
  slack_configs:
  - api_url: 'https://hooks.slack.com/services/YOUR_KEY'
    channel: '#db-alerts'

- name: 'application-team'
  slack_configs:
  - api_url: 'https://hooks.slack.com/services/YOUR_KEY'
    channel: '#app-alerts'

groups:
  - name: node-exporter-alerts
    rules:
      - alert: 实例离线
        expr: up == 0
        for: 1m
        labels:
          severity: critical
          instance: "{{ $labels.instance }}"
          ip: "{{ $labels.ip }}"
          location: "{{ $labels.location }}"
        annotations:
          summary: "实例 {{ $labels.instance }} 离线"
          description: "{{ $labels.instance }} 已停机超过 1 分钟。"

      - alert: 主机CPU使用率过高
        expr: 100 - (avg by(instance, ip, location) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 70
        for: 1m
        labels:
          severity: warning
          instance: "{{ $labels.instance }}"
          ip: "{{ $labels.ip }}"
          location: "{{ $labels.location }}"
        annotations:
          summary: "主机 CPU 使用率过高 (实例 {{ $labels.instance }})"
          description: "实例 {{ $labels.instance }} 的 CPU 使用率在过去 1 分钟内超过 70%。当前值: {{ $value | humanize }}%"

      - alert: 主机内存不足
        expr: node_memory_MemAvailable_bytes{job="node_exporter"} / node_memory_MemTotal_bytes{job="node_exporter"} * 100 < 10
        for: 5m
        labels:
          severity: critical
          instance: "{{ $labels.instance }}"
          ip: "{{ $labels.ip }}"
          location: "{{ $labels.location }}"
        annotations:
          summary: "主机内存不足 (实例 {{ $labels.instance }})"
          description: "实例 {{ $labels.instance }} 的可用内存低于总内存的 10% (过去 5 分钟)。当前值: {{ $value | humanize }}%"

      - alert: 主机磁盘空间不足
        expr: node_filesystem_avail_bytes{job="node_exporter",fstype="ext4",mountpoint="/"} / node_filesystem_size_bytes{job="node_exporter",fstype="ext4",mountpoint="/"} * 100 < 10
        for: 5m
        labels:
          severity: warning
          instance: "{{ $labels.instance }}"
          ip: "{{ $labels.ip }}"
          location: "{{ $labels.location }}"
        annotations:
          summary: "主机磁盘空间不足 (实例 {{ $labels.instance }}, 挂载点 {{ $labels.mountpoint }})"
          description: "实例 {{ $labels.instance }} (挂载点 {{ $labels.mountpoint }}) 的可用磁盘空间低于 10% (过去 5 分钟)。当前值: {{ $value | humanize }}%"

      - alert: HighNetworkTraffic
        expr: rate(node_network_transmit_bytes_total[1m]) > 1 * 1024 * 1024 / 8
        for: 5m
        labels:
          severity: critical
          instance: "{{ $labels.instance }}"
          ip: "{{ $labels.ip }}"
          location: "{{ $labels.location }}"
        annotations:
          summary: "实例 {{ $labels.instance }} 流量异常"
          description: "实例 {{ $labels.instance }} 的出站流量在过去5分钟内持续超过10Mb/s。当前速率: {{ printf \"%.2f Mb/s\" $value }}"

groups:
  - name: certificate_expiration_alerts
    rules:
      # 1. 证书即将过期 (保持现状，但优化描述)
      - alert: TLSCertificateExpiringSoon
        expr: |
          (probe_ssl_earliest_cert_expiry{job=~"blackbox_ssl_.*"} - time()) < 86400 * 30
        for: 5m
        labels:
          severity: warning
          category: certificate
        annotations:
          summary: "SSL 证书即将过期"
          description: "域名: {{ $labels.instance }}\n剩余时间: {{ $value | humanizeDuration }}\n过期时间: {{ with printf \"probe_ssl_earliest_cert_expiry{instance='%s'}\" $labels.instance | query }}{{ . | first | value | humanizeTimestamp }}{{ end }}"

      # 2. 证书已过期
      - alert: TLSCertificateExpired
        expr: |
          (probe_ssl_earliest_cert_expiry{job=~"blackbox_ssl_.*"} - time()) < 0
        for: 1m
        labels:
          severity: critical
          category: certificate
        annotations:
          summary: "SSL 证书已过期"
          description: "域名: {{ $labels.instance }}\n状态: 该域名证书已失效，请立即更换！"

      # 3. 探测失败 (增加失败原因判断)
      - alert: TLSCertificateProbeFailed
        expr: |
          probe_success{job=~"blackbox_ssl_.*"} == 0
        for: 3m
        labels:
          severity: critical
          category: network
        annotations:
          summary: "服务探测失败 ({{ $labels.instance }})"
          # 这里通过判断 probe_http_status_code 是否存在来辅助排查
          description: |
            域名: {{ $labels.instance }}
            详细描述: Blackbox 探测失败。
            可能原因:
            1. 域名无法解析或网络不通；
            2. 证书链不完整；
            3. 服务返回了非 2xx/3xx 状态码。
            当前状态码: {{ with printf "probe_http_status_code{instance='%s'}" $labels.instance | query }}{{ . | first | value }}{{ else }}无法连接/超时{{ end }}

networks:
  npm_public:
    external: true

services:
  # 1. Prometheus：监控时序数据库
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: always
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./prometheus_data:/prometheus
      - ./rules:/etc/prometheus/rules:ro        # 挂载报警规则目录
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d' # 默认保留30天数据，防止撑爆小鸡硬盘
    networks:
      - npm_public

  # 2. Alertmanager：报警管理器
  alertmanager:
    image: prom/alertmanager:latest
    container_name: alertmanager
    restart: always
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
    networks:
      - npm_public

  # 3. Grafana：可视化看板
  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: always
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=Admin1008611 # 🌟 请改成你自己的 Grafana 强密码
    volumes:
      - ./grafana_data:/var/lib/grafana
    networks:
      - npm_public

  # 4. Loki：轻量级日志聚合中心

  # 5. Blackbox Exporter：网络与证书探测器
  blackbox-exporter:
    image: prom/blackbox-exporter:latest
    container_name: blackbox-exporter
    restart: always
    volumes:
      - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
    command:
      - '--config.file=/etc/blackbox_exporter/config.yml'
    networks:
      - npm_public


  # 6. Node Exporter：容器化宿主机硬件监控
  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    restart: always
    command:
      - --path.procfs=/host/proc
      - --path.sysfs=/host/sys
      - --path.rootfs=/rootfs # 🌟 必须加上这行，才能正确监控宿主机磁盘
      - --collector.filesystem.mount-points-exclude="^/(sys|proc|dev|host|etc|mnt/host|var/lib/docker/containers)($|/)"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro,rslave # 🌟 使用 rslave 挂载，防止动态挂载点导致容器卡死
    networks:
      - npm_public # 🌟 牢牢对齐你的双栈公共网络

docker compose up -d
docker ps