Prometheus监控实战（四）

global:

  # 已经触发了告警的消息，在多⻓时间内没有收到该告警的消息，则会⾃动将该告警标记为已解决。默认值是 5 分钟。

  [ resolve_timeout: <duration> | default = 5m ]

  # 邮件配置

  [ smtp_from: <tmpl_string> ]

  [ smtp_smarthost: <string> ]

  [ smtp_hello: <string> | default = "localhost" ]

  [ smtp_auth_username: <string> ]

  [ smtp_auth_password: <secret> ]

  [ smtp_auth_identity: <string> ]

  [ smtp_auth_secret: <secret> ]

  [ smtp_require_tls: <bool> | default = true ]

  # 其他全局配置...

  [ wechat_api_url: <string> | default = "https://qyapi.weixin.qq.com/cgi␂bin/" ]

  [ wechat_api_secret: <secret> ]

  [ wechat_api_corp_id: <string> ]

# 模板配置

templates:

  # 路径到告警模板⽂件

  [ - <filepath> ... ]

# 告警路由

route:

  # 告警的⼦路由配置

  <route>

# 定义接收告警的接收⼈或服务

receivers:

  - <receiver> ...

inhibit_rules:

  # 抑制规则，⽤于减少告警噪声

  [ - <inhibit_rule> ... ]

#1. 下载 AlertManager

[root@prom-node01 ~]# wget https://github.com/prometheus/alertmanager/releases/download/v0.26.0/alertmanager-0.26.0.linux-amd64.tar.gz

# 加速地址

[root@prom-node01 ~]# wget https://mirror.ghproxy.com/https://github.com/prometheus/alertmanager/releases/download/v0.26.0/alertmanager-0.26.0.linux-amd64.tar.gz

#2. 解压 AlertManager

[root@prom-node01 ~]# tar xf alertmanager-0.26.0.linux-amd64.tar.gz -C /etc/

[root@prom-node01 ~]# ln -s /etc/alertmanager-0.26.0.linux-amd64/ /etc/alertmanager

#3. 查看 AlertManager ⽬录结构

[root@prom-node01 ~]# ll /etc/alertmanager/

total 62504

-rwxr-xr-x 1 3434 3434 35410965 Aug 24  2023 alertmanager

-rw-r--r-- 1 3434 3434      356 Aug 24  2023 alertmanager.yml

-rwxr-xr-x 1 3434 3434 28566971 Aug 24  2023 amtool

-rw-r--r-- 1 3434 3434    11357 Aug 24  2023 LICENSE

-rw-r--r-- 1 3434 3434      457 Aug 24  2023 NOTICE

[root@prom-node01 ~]# cp /etc/alertmanager/alertmanager.yml /etc/alertmanager/alertmanager.yml.bak

[root@prom-node01 ~]# cat /etc/alertmanager/alertmanager.yml

global:

  resolve_timeout: 2h    # 已经触发了告警的消息，在多长时间内没有收到该告警的消息，则会自动将该告警标记为已解决。

  smtp_smarthost: 'smtp.qq.com:587'

  smtp_from: '373370405@qq.com'  # 发件人邮件

  smtp_auth_username: '373370405@qq.com'  # 发件人用户名

  smtp_auth_password: '******' # 发件人密码 [授权码]

  smtp_hello: 'qq.com'

  smtp_require_tls: true

# 所有报警信息进入后的根路由，用来设置报警的分发策略

route:

  group_by: ['alertname'] #告警信息会根据告警名称分组，同名的告警消息会被归到一组，然后一起发送。（一个学校的一起搭⻋回家）

  group_wait: 30s         #当一组新告警产生时，系统会等待 30 秒，以便把这段时间内相同组的其他告警一起合并发送。（发车时间，保不齐在等待的 30s 内会有新的同校学生上车）

  group_interval: 1m     #已经发送了一个分组的告警通知之后，即使又有新的相同分组的告警到来，也需要等待至少 1 分钟后才会发送该分组的告警通知。（类似于发车间隔，没赶上就得等）

  repeat_interval: 5m  #如果同一个组中的报警信息已经发送成功了，下一次这个组发送告警的时间间隔（重复发送相同告警的时间间隔）

  receiver: default    #默认的 receiver：如果一个报警没有被一个 route 匹配，则发送给默认的接收器

receivers:

- name: 'default'

  email_configs:

  - to: '373370405@qq.com'

    send_resolved: true # 接受告警恢复的通知

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml

Checking '/etc/alertmanager/alertmanager.yml'  SUCCESS

Found:

 - global config

 - route

 - 0 inhibit rules

 - 1 receivers

 - 0 templates

[root@prom-node01 ~]# cat /usr/lib/systemd/system/alertmanager.service

[Unit]

Description=alertmanager

Documentation=https://prometheus.io/

After=network.target

[Service]

ExecStart=/etc/alertmanager/alertmanager \

  --web.listen-address=:9093 \

  --config.file=/etc/alertmanager/alertmanager.yml \

  --storage.path=/etc/alertmanager/data \

  --data.retention=120h

ExecReload=/bin/kill -HUP

TimeoutStopSec=20s

Restart=always

[Install]

WantedBy=multi-user.target

[root@prom-node01 ~]# systemctl daemon-reload

[root@prom-node01 ~]# systemctl enable alertmanager && systemctl start alertmanager

[root@prom-node01 ~]# netstat -lntp|grep 9093

tcp6       0      0 :::9093                 :::*                    LISTEN      2648/alertmanager

# 下载告警程序

[root@prom-node01 ~]# wget http://file.oldxu.net/prometheus/Alert/alert_test_oldxu

[root@prom-node01 ~]# chmod +x alert_test_oldxu

[root@prom-node03 ~]# docker run -it --rm -v `pwd`:/app/program ubuntu

root@4c83c1816d0b:/# cd /app/program/

# alertname=nodedown（--alertURL 指定 alertmanager 服务器所在的地址、端⼝以及接⼝）

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=nodedown,instance=prom-node01.oldxu.net,severity=critical,job=node_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=nodedown,instance=prom-node02.oldxu.net,severity=critical,job=node_exporter"

# alertname=cpuhigh

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=cpuhigh,instance=prom-node01.oldxu.net,severity=critical,job=node_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=cpuhigh,instance=prom-node02.oldxu.net,severity=critical,job=node_exporter"

[root@prom-node01 ~]# cat /etc/prometheus/prometheus.yml

# 全局段定义

global:

  scrape_interval: 15s # 设置 Prometheus 抓取指标数据的间隔，默认为 15 秒。

# Alertmanager 配置

alerting:

  alertmanagers:

  - static_configs:

    - targets: ["prom-node01.oldxu.net:9093"]

[root@prom-node01 ~]# cat /etc/prometheus/prometheus.yml

# 全局段定义

global:

  scrape_interval: 15s # 设置 Prometheus 抓取指标数据的间隔，默认为 15 秒。

# Alertmanager 配置

alerting:

  alertmanagers:

  - static_configs:

    - targets: ["prom-node01.oldxu.net:9093"]

rule_files:

  - "/etc/prometheus/rules/*.yml" 

# 抓取指定的目标

scrape_configs:

  - job_name: "prometheus" # 定义一个抓取任务，名为 'prometheus'。

    metrics_path: "/metrics" # 指定 Prometheus 从监控目标暴露的 HTTP 端点路径抓取指标，默认为 '/metrics'。

    static_configs: # 配置静态目标地址，Prometheus 将定期从如下这些地址抓取指标。

    - targets: ["prom-node01.oldxu.net:9090"]

  - job_name: "grafana"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node01.oldxu.net:3000"]

  - job_name: "node_exporter"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node01.oldxu.net:9100","prom-node02.oldxu.net:9100","prom-node03.oldxu.net:9100"]

  - job_name: "weather_exporter"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node02.oldxu.net:7001"]

  - job_name: "webserver"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node02.oldxu.net:7002"]

  - job_name: "rabbitmq"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node02.oldxu.net:15692"]

  - job_name: "nginx"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node03.oldxu.net:9113"]

  - job_name: "tomcat"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node03.oldxu.net:8080"]

  - job_name: "jmx_exporter"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node03.oldxu.net:12345"]

  - job_name: "mysqld_exporter"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node03.oldxu.net:9104"]

      labels:

        service: database

        role: master

    - targets: ["prom-node04.oldxu.net:9104"]

      labels:

        service: database

        role: slave

  - job_name: "redis_exporter"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node03.oldxu.net:9121"]

  - job_name: 'blackbox_http'

    metrics_path: /probe # metrics 的 path 这次不是 /metrics，是 /probe

    params: # 传递参数

      module: [http_2xx] # 调哪个模块进探测

    static_configs:

    - targets: ["https://www.xuliangwei.com","http://www.oldxu.net","https://www.baidu.com","http://httpbin.org/status/400","https://httpstat.us/500","https://httpstat.us/502"]

    relabel_configs:

    - source_labels: [__address__]

      target_label: __param_target

    - source_labels: [__param_target]

      target_label: instance

    - target_label: __address__

      replacement: prom-node04.oldxu.net:9115

  - job_name: 'blackbox_tcp'

    metrics_path: /probe

    params:

      module: [tcp_connect] # 使 tcp_connect 模块

    static_configs:

    - targets: ["prom-node03.oldxu.net:3306","prom-node03.oldxu.net:6379"]

    relabel_configs:

    - source_labels: [__address__]

      target_label: __param_target

    - source_labels: [__param_target]

      target_label: instance

    - target_label: __address__

      replacement: prom-node04.oldxu.net:9115

  - job_name: 'blackbox_icmp'

    metrics_path: /probe

    params:

      module: [icmp] # 使 icmp 模块

    static_configs:

    - targets: ["prom-node01.oldxu.net","prom-node02.oldxu.net","prom-node03.oldxu.net"]

    relabel_configs:

    - source_labels: [__address__]

      target_label: __param_target

    - source_labels: [__param_target]

      target_label: instance

    - target_label: __address__

      replacement: prom-node04.oldxu.net:9115

  - job_name: 'blackbox_ssh'

    metrics_path: /probe

    params:

      module: [ssh_banner] # 使⽤ ssh_banner 模块

    static_configs:

    - targets: ["prom-node01.oldxu.net:22","prom-node02.oldxu.net:22","prom-node03.oldxu.net:22"]

    relabel_configs:

    - source_labels: [__address__]

      target_label: __param_target

    - source_labels: [__param_target]

      target_label: instance

    - target_label: __address__

      replacement: prom-node04.oldxu.net:9115

  - job_name: 'domain_exporter'

    metrics_path: /probe # metrics 的 path 不是 /metrics，⽽是 /probe

    static_configs:

    - targets: ["nf-leasing.com","hmallleasing.com","jd.com"]

    relabel_configs:

    - source_labels: [__address__]

      target_label: __param_target

    - source_labels: [__param_target]

      target_label: instance

    - target_label: __address__

      replacement: prom-node04.oldxu.net:9222

  - job_name: "pushgateway"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node04.oldxu.net:9091"]

  - job_name: "alertmanager"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node01.oldxu.net:9093"]

[root@prom-node01 ~]# curl -X POST http://localhost:9090/-/reload

[root@prom-node01 ~]# cat /etc/prometheus/rules/node_rules.yml 

groups:

- name: 节点告警规则

  rules:

  - alert: 节点处于Down状态

    expr: up == 0

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "节点处于Down状态,实例:"

      description: " 节点已关闭"

  - alert: 节点CPU使率超过80%

    expr: ( 1 - avg(irate(node_cpu_seconds_total{mode="idle"}[1m])) by (instance,job) ) * 100 > 80

    for: 1m

    labels:

      severity: warning

    annotations:

      summary: "主机CPU利率过高，实例：, 任务:"

      description: "该实例的CPU利用率低于20%，当前利用率：%。可能存在CPU资源浪费情况。"

  - alert: CPU饱和度过高

    expr: sum(node_load1) by (instance,job) / (count(node_cpu_seconds_total{mode="idle"}) by (instance,job) * 2) * 100 > 80

    for: 2m

    labels:

      severity: critical

    annotations:

      summary: "CPU饱和度过高，实例：, 任务:"

      description: "该实例的1分钟平均CPU负载超过了核心数的两倍，已经持续2分钟，当前CPU饱和度：%。需要立即检查系统负载情况。"

  - alert: 主机内存不足

    expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 80

    for: 2m

    labels:

      severity: warning

    annotations:

      summary: "主机内存使用率较高, 实例:, 任务:"

      description: "该实例的内存使用率持续2分钟高于80%，当前利用率：%"

  - alert: 内存饱和度高

    expr: ( 1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 10

    for: 2m

    labels:

      severity: warning

    annotations:

      summary: "主机内存内存饱和度高, 实例:, 任务:"

      description: "SWAP内存使用率已连续2分钟超过10%，表明内存饱和度过高，当前SWAP使用率为：%。"

  - alert: 磁盘空间告急

    expr: ( node_filesystem_size_bytes{device!="tmpfs"} - node_filesystem_avail_bytes{device!="tmpfs"} ) / node_filesystem_size_bytes{device!="tmpfs"} * 100 > 70

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  磁盘 分区空间不足"

      description: "实例  磁盘  分区空间使用率已超过 70%，当前使用率为 %，请及时处理。"

  - alert: 磁盘Inode空间告急

    expr: (node_filesystem_files{device!="tmpfs"} - node_filesystem_files_free{device!="tmpfs"} ) / node_filesystem_files{device!="tmpfs"} * 100 > 70

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  磁盘 分区Inode空间不足"

      description: "实例  磁盘  分区的Inode空间使用率已超过 70%，，当前使用率为 %，请及时处理。"

  - alert: 磁盘IOPS写入较高

    #expr: sum(rate(node_disk_writes_completed_total[1m])) by (instance,job) / 120 * 100 >60

    #round 函数可以对值进行四舍五入，磁盘最大 IOPS 为 120 次 /s

    expr: round(max(irate(node_disk_writes_completed_total[1m])) by (instance,job) / 120 * 100) > 60

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  IOPS每秒写入次数超过120次/s"

      description: 

        当前磁盘IOPS写入饱和度是 <!--swig25-->%

        当前磁盘IOPS每秒写入最大 <!--swig26--> 次/s

  - alert: 磁盘IOPS读取较高

    expr: round(max(irate(node_disk_reads_completed_total[1m])) by (instance,job) / 120 * 100) > 60

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  IOPS每秒读取次数超过120次/s"

      description:

        当前磁盘IOPS读取饱和度是 <!--swig28-->%

        当前磁盘IOPS每秒读取最⼤ <!--swig29--> 次/s

  - alert: 磁盘IO写入吞吐较高

    expr: round(max(rate(node_disk_written_bytes_total[1m])) by (instance,job) / 1024 /1024 / 30 * 100) > 60

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  磁盘IO写入每秒超过最高30MB/s"

      description:

        当前磁盘IO写入吞吐量的饱和度是 <!--swig31-->%。

        当前磁盘IO写入吞吐量每秒最大是 <!--swig32-->MB/s

  - alert: 磁盘IO读取吞吐较高

    expr: round(max(rate(node_disk_read_bytes_total[1m])) by (instance,job) / 1024 /1024 /30 * 100 ) > 60

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  磁盘IO读取每秒超过最大30MB/s"

      description:

        当前磁盘IO读取吞吐量的饱和度是 <!--swig34-->%。

        当前磁盘IO读取吞吐量每秒最大是 <!--swig35-->MB/s

  - alert: 网络下载带宽异常

    expr: max(irate(node_network_receive_bytes_total[1m]) * 8 / 1024 / 1024) by (instance,job,device) / 50 * 100 >= 80

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  的  接口下载流量已经超过公司实际50Mbps"

      description:

        当前下载带宽已经达到 <!--swig38--> Mbps/s

        当前下载带宽使用率在 <!--swig39-->%

  - alert: 网络上传带宽异常

    expr: max(irate(node_network_transmit_bytes_total[1m]) * 8 / 1024 / 1024) by (instance,job,device) / 50 * 100 >= 80

    for: 1m

    labels:

      severity: critical

    annotations:

      summary: "实例  的  接口上传流量已经超过公司实际50Mbps"

      description:

        当前上传带宽已经达到 <!--swig42--> Mbps/s

        当前上传带宽使用率在 <!--swig43-->%

[root@prom-node01 ~]# cat /etc/prometheus/prometheus.yml

# 全局段定义

global:

  scrape_interval: 15s # 设置 Prometheus 抓取指标数据的间隔，默认为 15 秒。

# Alertmanager 配置

alerting:

  alertmanagers:

  - static_configs:

    - targets: ["prom-node01.oldxu.net:9093"]

rule_files:

  - "/etc/prometheus/rules/*.yml" 

# 抓取指定的目标

scrape_configs:

  - job_name: "prometheus" # 定义一个抓取任务，名为 'prometheus'。

    metrics_path: "/metrics" # 指定 Prometheus 从监控目标暴露的 HTTP 端点路径抓取指标，默认为 '/metrics'。

    static_configs: # 配置静态目标地址，Prometheus 将定期从如下这些地址抓取指标。

    - targets: ["prom-node01.oldxu.net:9090"]

  - job_name: "grafana"

    metrics_path: "/metrics"

    static_configs:

    - targets: ["prom-node01.oldxu.net:3000"]

[root@prom-node01 ~]# /etc/prometheus/promtool check rules /etc/prometheus/rules/node_rules.yml 

Checking /etc/prometheus/rules/node_rules.yml

  SUCCESS: 13 rules found

curl -X POST http://localhost:9090/-/reload

[root@prom-node02 ~]# systemctl stop node_exporter.service

[root@prom-node03 ~]# systemctl stop node_exporter.service

[root@prom-node01 ~]# mkdir /etc/alertmanager/template

[root@prom-node01 ~]# cat /etc/alertmanager/template/email.tmpl

<!--swig44-->

<!--swig45-->

<!--swig46-->

<h2 style="color: red;">@告警通知</h2>

告警程序: AlertManager <br>

告警级别: <!--swig47--> <br>

告警类型: <!--swig48--> <br>

故障主机: <!--swig49--> <br>

告警主题: <!--swig50--> <br>

告警详情: <!--swig51--> <br>

触发时间: <!--swig52--> <br>

<!--swig53--><!--swig54-->

<!--swig55-->

<!--swig56-->

<h2 style="color: green;">@告警恢复</h2>

告警程序: AlertManager <br>

告警级别: <!--swig57--> <br>

告警类型: <!--swig58--> <br>

告警主机: <!--swig59--> <br>

告警主题: <!--swig60--> <br>

告警详情: <!--swig61--> <br>

触发时间: <!--swig62--> <br>

恢复时间: <!--swig63--> <br>

<!--swig64--><!--swig65-->

<!--swig66-->

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml

Checking '/etc/alertmanager/alertmanager.yml'  SUCCESS

Found:

 - global config

 - route

 - 0 inhibit rules

 - 1 receivers

 - 1 templates

  SUCCESS

[root@prom-node01 ~]# curl -X POST http://localhost:9093/-/reload

https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=47ffbdd9-4308-4db0-aa08-f309546cc92d

[root@prom-node01 ~]# wget http://file.oldxu.net/prometheus/Alert/webhook_wechat_oldxu

[root@prom-node01 ~]# mv webhook_wechat_oldxu /usr/local/bin/

[root@prom-node01 ~]# chmod +x /usr/local/bin/webhook_wechat_oldxu

[root@prom-node01 ~]# cat /usr/lib/systemd/system/webhook_wechat.service

[Unit]

Description=webhook_wechat

Documentation=https://prometheus.io/

After=network.target

[Service]

ExecStart=/usr/local/bin/webhook_wechat_oldxu --port 5001

ExecReload=/bin/kill -HUP

TimeoutStopSec=20s

Restart=always

[Install]

WantedBy=multi-user.target

[root@prom-node01 ~]# systemctl daemon-reload

[root@prom-node01 ~]# systemctl start webhook_wechat && systemctl enable webhook_wechat

[root@prom-node01 ~]# netstat -lntp|grep 5001

tcp6       0      0 :::5001                 :::*                    LISTEN      2576/webhook_wechat

[root@prom-node01 ~]# curl -X POST http://localhost:5001/alert?token=< 企业微信机器人 token> \

-H "Content-Type: application/json" \

-d '{

  "alerts": [

    {

      "status": "firing",

      "labels": {

        "severity": "critical",

        "alertname": "InstanceDown",

        "instance": "example1"

      },

      "annotations": {

        "summary": "Instance example1 down",

        "description": "The instance example1 is down."

      },

      "startsAt": "2024-12-20T15:04:05Z",

      "endsAt": "0001-01-01T00:00:00Z"

    },

   {

      "status": "resolved",

      "labels": {

        "severity": "critical",

        "alertname": "InstanceDown",

        "instance": "example1"

      },

      "annotations": {

        "summary": "Instance example1 is back up",

        "description": "The instance example1 has recovered."

      },

      "startsAt": "2024-12-20T15:04:05Z",

      "endsAt": "2024-12-20T16:04:05Z"

    }

  ]

}'

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml

Checking '/etc/alertmanager/alertmanager.yml'  SUCCESS

Found:

 - global config

 - route

 - 0 inhibit rules

 - 1 receivers

 - 1 templates

  SUCCESS

[root@prom-node01 ~]#  curl -X POST http://localhost:9093/-/reload

https://oapi.dingtalk.com/robot/send?access_token=906498a49ca8fdd29ae71258393f4ae25aff9ab3d47f007716ebb283f1c8e8db

[root@prom-node01 ~]# wget http://file.oldxu.net/prometheus/Alert/webhook_dingding_oldxu

[root@prom-node01 ~]# mv webhook_dingding_oldxu /usr/local/bin/

[root@prom-node01 ~]# chmod +x /usr/local/bin/webhook_dingding_oldxu

[root@prom-node01 ~]# cat /usr/lib/systemd/system/webhook_dingding.service

[Unit]

Description=webhook_dingding

Documentation=https://prometheus.io/

After=network.target

[Service]

ExecStart=/usr/local/bin/webhook_dingding_oldxu --port 5002

ExecReload=/bin/kill -HUP

TimeoutStopSec=20s

Restart=always

[Install]

WantedBy=multi-user.target

[root@prom-node01 ~]# systemctl daemon-reload

[root@prom-node01 ~]# systemctl start webhook_dingding && systemctl enable webhook_dingding 

[root@prom-node01 ~]# netstat -lntp|grep 5002

tcp6       0      0 :::5002                 :::*                    LISTEN      3141/webhook_dingdi

[root@prom-node01 ~]# curl -X POST http://localhost:5002/alert?token=906498a49ca8fdd29ae71258393f4ae25aff9ab3d47f007716ebb283f1c8e8db \

-H "Content-Type: application/json" \

-d '{

  "alerts": [

    {

      "status": "firing",

      "labels": {

        "severity": "critical",

        "alertname": "InstanceDown",

        "instance": "example1"

      },

      "annotations": {

        "summary": "Instance example1 down",

        "description": "The instance example1 is down."

      },

      "startsAt": "2024-12-20T15:04:05Z",

      "endsAt": "0001-01-01T00:00:00Z"

    },

   {

      "status": "resolved",

      "labels": {

        "severity": "critical",

        "alertname": "InstanceDown",

        "instance": "example1"

      },

      "annotations": {

        "summary": "Instance example1 is back up",

        "description": "The instance example1 has recovered."

      },

      "startsAt": "2024-12-20T15:04:05Z",

      "endsAt": "2024-12-20T16:04:05Z"

    }

  ]

}'

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml

[root@prom-node01 ~]#  curl -X POST http://localhost:9093/-/reload

https://github.com/timonwong/prometheus-webhook-dingtalk

https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v2.1.0/prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz

[root@prom-node01 ~]# tar -xf prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz -C /usr/local/bin/

[root@prom-node01 ~]# ln -s /usr/local/bin/prometheus-webhook-dingtalk-2.1.0.linux-amd64/ /usr/local/bin/webhook-dingtalk

[root@prom-node01 ~]#  cat /usr/local/bin/webhook-dingtalk//template.tmpl

<!--swig70-->

[<!--swig71--><!--swig72-->:<!--swig73--><!--swig74-->]

<!--swig75-->

<!--swig76--><!--swig77-->

---

<!--swig78-->@<!--swig79--><!--swig80-->

**告警程序**: AlertManager <br>

**告警类型**: <!--swig81--> <br>

**告警级别**: <!--swig82--> <br>

**告警主机**: <!--swig83--> <br>

**告警主题**: <!--swig84--> <br>

**告警信息**: <!--swig85--> <br>

**告警时间**: <font color='#FF0000'><!--swig86--> </font><br>

<!--swig87--><!--swig88-->

<!--swig89--><!--swig90-->

---

<!--swig91-->@<!--swig92--><!--swig93-->

**告警程序**: AlertManager <br>

**告警类型**: <!--swig94--> <br>

**告警级别**: <!--swig95--> <br>

**告警主机**: <!--swig96--> <br>

**告警主题**: <!--swig97--> <br>

**告警信息**: <!--swig98--> <br>

**告警时间**: <font color='#FF0000'><!--swig99--> </font><br>

**恢复时间**: <font color='#FF0000'><!--swig100--> </font><br>

<!--swig101--><!--swig102-->

<!--swig103-->

<!--swig104-->

<!--swig105-->

<!--swig106-->

<!--swig107-->

**<h2><font color='#FF7F00'>侦测到<!--swig108-->个故障</font></h2>**

<!--swig109-->

---

<!--swig110-->

<!--swig111-->

**<h2><font color='#00FF00'>恢复<!--swig112-->个故障</font></h2>**

<!--swig113-->

<!--swig114-->

<!--swig115-->

<!--swig116--><!--swig117--><!--swig118-->

<!--swig119--><!--swig120--><!--swig121-->

<!--swig122-->

<!--swig123-->

[root@prom-node01 ~]# cat /usr/lib/systemd/system/webhook.service

[Unit]

Description=Prometheus-Server

After=network.target

[Service]

ExecStart=/usr/local/bin/webhook-dingtalk/prometheus-webhook-dingtalk \

  --config.file=/usr/local/bin/webhook-dingtalk/config.yml

ExecReload=/bin/kill -HUP

TimeoutStopSec=20s

Restart=always

[Install]

WantedBy=multi-user.target

[root@prom-node01 ~]# systemctl daemon-reload

[root@prom-node01 ~]# systemctl start webhook.service && systemctl enable webhook.service

[root@prom-node01 ~]# netstat -lntp|grep 8060

tcp6       0      0 :::8060                 :::*                    LISTEN      3008/prometheus-web

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml

Checking '/etc/alertmanager/alertmanager.yml'  SUCCESS

Found:

 - global config

 - route

 - 0 inhibit rules

 - 3 receivers

 - 1 templates

  SUCCESS

curl -X POST http://localhost:9093/-/reload

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml

Checking '/etc/alertmanager/alertmanager.yml'  SUCCESS

Found:

 - global config

 - route

 - 0 inhibit rules

 - 3 receivers

 - 1 templates

  SUCCESS

[root@prom-node01 ~]# curl -X POST http://localhost:9093/-/reload

[root@prom-node03 ~]# systemctl stop mysqld_exporter.service redis_exporter.service

[root@prom-node02 ~]# systemctl stop node_exporter.service

[root@prom-node03 ~]# systemctl stop node_exporter.service

[root@prom-node03 ~]# systemctl stop nginx_exporter.service

# 下载告警程序

[root@prom-node01 ~]# wget http://file.oldxu.net/prometheus/Alert/alert_test_oldxu

[root@prom-node01 ~]# chmod +x alert_test_oldxu

[root@prom-node03 ~]# docker run -it --rm -v `pwd`:/app/program ubuntu

root@4c83c1816d0b:/# cd /app/program/

# alertname=nodedown（--alertURL 指定 alertmanager 服务器所在的地址、端⼝以及接⼝）

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=nodedown,instance=prom-node01.oldxu.net,severity=critical,job=node_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=nodedown,instance=prom-node02.oldxu.net,severity=critical,job=node_exporter"

# alertname=cpuhigh

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=cpuhigh,instance=prom-node01.oldxu.net,severity=critical,job=node_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=cpuhigh,instance=prom-node02.oldxu.net,severity=critical,job=node_exporter"

inhibit_rules:

- source_matchers:

    - alertname = "NodeDown" # 源告警是节点故障告警。

  target_matchers:

    - job =~ ".*"     # ⽬标告警可以是任何 job 产⽣的，因为⼀个节点上的服务会被划分到不同的 job

  equal: ["instance"] # 确保只有当源告警和⽬标告警具有相同的 instance 的值时，⽬标告警才会被抑制。

# 下载告警程序

[root@prom-node01 ~]# wget http://file.oldxu.net/prometheus/Alert/alert_test_oldxu

[root@prom-node01 ~]# chmod +x alert_test_oldxu

[root@prom-node03 ~]# docker run -it --rm -v `pwd`:/app/program ubuntu

root@4c83c1816d0b:/# cd /app/program/

# 主要故障

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=NodeDown,instance=prom-node01.oldxu.net,job=node_exporter"

# 级联故障

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=tomcatDown,instance=prom-node01.oldxu.net,job=tomcat_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=NginxDown,instance=prom-node01.oldxu.net,job=nginx_exporter"

[root@prom-node01 ~]# vim /etc/alertmanager/alertmanager.yml

inhibit_rules:

- source_matchers:

    - alertname = "NodeDown" # 假设当节点故障时，触发的告警名为 "NodeDown"

  target_matchers:

    - job =~ ".*"            # 则抑制该节点上所有应⽤发送的故障，不论他是哪个 job 产⽣的

  equal: ["instance"]        # instance 必须相等，才能表示是同⼀个节点上的告警

[root@prom-node01 ~]# /etc/alertmanager/amtool check-config /etc/alertmanager/alertmanager.yml 

[root@prom-node01 ~]# curl -X POST http://localhost:9093/-/reload

# 主要故障

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=NodeDown,instance=prom-node01.oldxu.net,job=node_exporter"

# 级联故障

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=tomcatDown,instance=prom-node01.oldxu.net,job=tomcat_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=NginxDown,instance=prom-node01.oldxu.net,job=nginx_exporter"

groups:

- name: database-alerts

  rules:

  - alert: 主库down机了

    expr: up{service="database", role="master"} == 0

    for: 5m

    labels:

      severity: 'critical'

      service: 'database'

      role: 'master'

    annotations:

    summary: "master database down"

 - alert: 从库down机了

   expr: up{service="database", role="slave"} == 0

   for: 5m

   labels:

     severity: 'warning'

     service: 'database'

     role: 'slave'

   annotations:

   summary: "slave database down"

# alertname = 节点故障（--alertURL 指定 alertmanager 服务器所在的地址、端⼝以及接⼝）

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" \

 --label="alertname=主库down机了,instance=mysql-master.oldxu.net,severity=critical,role=master,service=database,job=mysql_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" \

 --label="alertname=从库down机了,instance=mysql-slave.oldxu.net,severity=critical,role=slave,service=database,job=mysql_exporter"

[root@prom-node01 ~]# vim /etc/alertmanager/alertmanager.yml

inhibit_rules:

- source_match:

    service: "database"

    role: "master"

  target_match:

    role: "slave"

  equal: ["service"]

 [root@prom-node01 ~]# curl -X POST http://localhost:9093/-/reload

[root@prom-node01 ~]# scp -rp /etc/alertmanager-0.26.0.linux-amd64/ root@192.168.40.222:/etc/

[root@prom-node01 ~]# scp -rp /etc/alertmanager-0.26.0.linux-amd64/ root@192.168.40.223:/etc/

# node02 配置软链接

[root@prom-node02 ~]# ln -s /etc/alertmanager-0.26.0.linux-amd64/ /etc/alertmanager

# node03 配置软链接

[root@prom-node03 ~]# ln -s /etc/alertmanager-0.26.0.linux-amd64/ /etc/alertmanager

[root@prom-node01 ~]# systemctl stop alertmanager

vim /usr/lib/systemd/system/alertmanager_ha.service

[Unit]

Description=alertmanager

Documentation=https://prometheus.io/

After=network.target

[Service]

ExecStart=/etc/alertmanager/alertmanager \

  --web.listen-address=:9093 \

  --cluster.listen-address=0.0.0.0:9094 \

  --cluster.peer=192.168.40.221:9094 \

  --cluster.peer=192.168.40.222:9094 \

  --cluster.peer=192.168.40.223:9094 \

  --cluster.peer-timeout=15s \

  --config.file=/etc/alertmanager/alertmanager.yml \

  --storage.path=/etc/alertmanager/data \

  --data.retention=120h

ExecReload=/bin/kill -HUP

TimeoutStopSec=20s

Restart=always

[Install]

WantedBy=multi-user.target

# systemctl daemon-reload 

# systemctl start alertmanager_ha.service &&  systemctl start alertmanager_ha.service

[root@prom-node01 ~]# vim /etc/prometheus/prometheus.yml

alerting:

  alertmanagers:

  - static_configs:

    - targets:

      - prom-node01.oldxu.net:9093

      - prom-node02.oldxu.net:9093

      - prom-node03.oldxu.net:9093

[root@prom-node01 ~]# curl -X POST http://localhost:9090/-/reload

# 下载告警程序

[root@prom-node01 ~]# wget http://file.oldxu.net/prometheus/Alert/alert_test_oldxu

[root@prom-node01 ~]# chmod +x alert_test_oldxu

[root@prom-node03 ~]# docker run -it --rm -v `pwd`:/app/program ubuntu

root@4c83c1816d0b:/# cd /app/program/

# alertname=nodedown（--alertURL 指定 alertmanager 服务器所在的地址、端⼝以及接⼝）

./alert_test_oldxu --alertURL="http://192.168.40.221:9093/api/v1/alerts" --label="alertname=nodedown,instance=prom-node01.oldxu.net,severity=critical,job=node_exporter"

./alert_test_oldxu --alertURL="http://192.168.40.222:9093/api/v1/alerts" --label="alertname=nodedown,instance=prom-node01.oldxu.net,severity=critical,job=node_exporter"