Prometheus实战-从0构建高可用监控平台（四）

cd /usr/local/src/
https://github.com/prometheus/alertmanager/releases/download/v0.24.0/alertmanager-0.24.0.linux-amd64.tar.gz

tar xf alertmanager-0.24.0.linux-amd64.tar.gz  -C /usr/local/
ln -sv /usr/local/alertmanager-0.24.0.linux-amd64 /usr/local/alertmanager
chown -R prometheus. /usr/local/alertmanager-0.24.0.linux-amd64 

mkdir /data/alertmanager
chown -R prometheus. /data/alertmanager

cat > /etc/systemd/system/alertmanager.service 

[root@Q-gz-common-prod-thanos-002 alertmanager]# cat alertmanager.yml | grep  -v "^#"
route:
  #group_by: ['instance']
  group_by: ['alertname', 'severity']
  group_wait: 10s
  group_interval: 3m
  # 重新发送相同告警的间隔。比如我主机宕机了，一直没处理，那么就一直有告警，发完第一次后以后每次都要等repeat_interval时间才会发送第二次。
  repeat_interval: 5m
  # 比如一个分组同时有A、B两个告警，A先到达进去group_wait里面等着，在group_wait期间，B也进来了，两个合并成一组，并发送一个告警消息出去；发完后A、B的问题还未解决，还持续触发告警，然后分组内又来了个新告警C，这就触发了group_interval，由于同组的状态发生了变化，A、B、C会在group_interval内被快速推送告警，不会等到group_interval后再发；如果发完后，A、B、C三者都持续无变化，那么就会在repeat_interval周期性发送告警信息。
  receiver: 'web.hook.prometheusalert'
  routes:
  - receiver: txsms
    continue: true
    group_wait: 30s
    matchers:
    - severity="critical"
  #- receiver: flashcat 
  #  continue: true
  #  group_wait: 30s
  #  matchers:
  #  - severity="error"
  # 配置其他级别的告警通知
  - receiver: web.hook.prometheusalert
    continue: true
    group_wait: 10s
    matchers:
    - severity=~".*"

receivers:
  - name: 'web.hook.prometheusalert'
    webhook_configs:
    - url: 'http://10.2.0.27:18080/prometheusalert?type=fs&tpl=prometheus-fs&fsurl=https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxx'

  - name: 'txsms'
    webhook_configs:
    - url: 'http://10.2.0.27:18080/prometheusalert?type=txdx&tpl=prometheus-dx&phone=1xxxx1,1xxxx2'
      send_resolved: true

  - name: 'flashcat'
    webhook_configs:
    - url: 'https://api.flashcat.cloud/event/push/alert/prometheus?integration_key=11111'
      send_resolved: true

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'error|warning|info|notice'
    equal: ['instance', 'alertname']
  - source_match:
      severity: 'error'
    target_match:
      severity: 'warning|info|notice'
    equal: ['instance', 'alertname']
  - source_match:
      severity: 'warning'
    target_match:
      severity: 'info|notice'
    equal: ['instance', 'alertname']

致命  critical
    严重  error
    警告  warning
    提醒  notice
    通知  info

frontend alertmanager-in
    bind     *:19093
    maxconn  20000
    default_backend alertmanager-out

backend alertmanager-out
    server alertmanager-1    10.2.0.10:9093    maxconn 20480  weight 10  check inter 10s  rise 1 fall 2
    server alertmanager-2    10.2.0.41:9093    maxconn 20480  weight 10  check inter 10s  rise 1 fall 2 backup

inhibit_rules:
  # 默认的抑制规则，对于任何两个相同的告警，间隔小于 1 分钟的将被抑制
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning|info'
    equal: ['alertname', 'job']
    # 抑制时间为 1 分钟
    duration: 1m
  # 周期性抑制规则，对于同一个 alertname 和 job 的告警，在每个小时的 0 分和 30 分时，间隔小于 5 分钟的将被抑制
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning|info'
    equal: ['alertname', 'job']
    # 在每个小时的 0 分和 30 分触发抑制
    periods:
      - start: 0m
        end: 5m
      - start: 30m
        end: 35m

# cat amtool_mysql_silence_every_day.sh 
#!/bin/bash

# 获取明天的日期并将其与固定时间组合成友好的日期和时间格式
tomorrow_date=$(date -d "+1 day" +%Y-%m-%d)
start_time='01:00:00'
end_time='06:00:00'
datetime_start="$tomorrow_date $start_time"
datetime_end="$tomorrow_date $end_time"
start_point=$(date -d "$datetime_start"  +"%Y-%m-%dT%H:%M:%S%:z")
end_point=$(date -d "$datetime_end"   +"%Y-%m-%dT%H:%M:%S%:z")

# 定义正则表达式模式，抑制创建者和抑制描述或注释
regex_pattern="module=~mysql.*"
author="admin"
comment="for test"

# 创建amtool命令
command="amtool silence add '$regex_pattern' --start='$start_point' --end='$end_point' -a '$author' -c '$comment'"

# 运行amtool命令
echo "Running command: $command"
eval "$command"

#author len
#This script for create alertmanager silence every day api!
# shell 占位符, 每天凌晨1点到5点mysql告警静默

AlertmanagerUrl="http://10.2.0.10:9093/api/v2/silences"
SILEBCE_DATA_TEL='{"matchers":[{"name":"alertname","value":"UnusualDiskIOutil","isRegex":false,"isEqual":true}],"startsAt":"%sT17:00:00.000Z","endsAt":"%sT21:00:00.000Z","createdBy":"admin","comment":"mysql_silence","id":null}'

#SILEBCE_MYSQL_TEL='{"matchers":[{"name":"module","value":"mysql.*","isRegex":true}],"startsAt":"%sT17:00:00.000Z","endsAt":"%sT21:00:00.000Z","createdBy":"admin","comment":"mysql_silence","id":null}'
SILEBCE_MYSQL_TEL='{"matchers":[{"name":"module","value":"mysql.*","isRegex":true}],"startsAt":"%sT08:00:00.000Z","endsAt":"%sT09:00:00.000Z","createdBy":"admin","comment":"mysql_silence","id":null}'

TODAY=$(date +%Y-%m-%d)
SILEBCE_DATA=`printf ${SILEBCE_DATA_TEL} ${TODAY} ${TODAY}`
SILEBCE_MYSQL=`printf ${SILEBCE_MYSQL_TEL} ${TODAY} ${TODAY}`


echo ${SILEBCE_MYSQL}

function createSilence() {
    curl "${AlertmanagerUrl}" -H "Content-Type: application/json"  
  --data "${SILEBCE_DATA}" 
  --compressed 
  --insecure
}

function createMysqlSilence() {
    curl "${AlertmanagerUrl}" -H "Content-Type: application/json"  
  --data "${SILEBCE_MYSQL}" 
  --compressed 
  --insecure
}

createSilence
createMysqlSilence