首页 > Linux

二进制部署Prometheus alertmanager

来源：原创
时间：2020-04-02
作者：脚本小站
分类：Linux

exporter下载地址：

prometheus.io/download/

prometheus.service：

cat > /usr/lib/systemd/system/prometheus.service <<EOF
[Unit]
Description=Prometheus
 
[Service]
ExecStart=/usr/local/prometheus/prometheus \
--config.file=/usr/local/prometheus/prometheus.yml \
--storage.tsdb.path=/usr/local/prometheus/data \
--storage.tsdb.retention.time=30d \
--web.enable-lifecycle

Restart=on-failure
KillMode=process
RestartSec=10
 
[Install]
WantedBy=multi-user.target
EOF

alertmanager.service：

cat > /usr/lib/systemd/system/alertmanager.service <<EOF
[Unit]
Description=alertmanager

[Service]
ExecStart=/usr/local/alertmanager/alertmanager \
--config.file=/usr/local/alertmanager/alertmanager.yml \
--storage.path=/usr/local/alertmanager/data

Restart=on-failure
KillMode=process
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

prometheusalert.service：

cat > /usr/lib/systemd/system/prometheusalert.service <<EOF
[Unit]
Description=PrometheusAlert
  
[Service]
WorkingDirectory=/usr/local/PrometheusAlert
ExecStart=/usr/local/PrometheusAlert/PrometheusAlert
 
Restart=on-failure
KillMode=process
RestartSec=10
  
[Install]
WantedBy=multi-user.target
EOF

默认账号和密码：
prometheusalert
prometheusalert

node_exporter.service：

cat > /usr/lib/systemd/system/node_exporter.service <<EOF
[Unit]
Description=node_exporter
After=network.target

[Service]
ExecStart=/usr/local/node_exporter/node_exporter

Restart=on-failure
KillMode=process
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

mysqld_exporter.service：

cat > /usr/lib/systemd/system/mysqld_exporter.service <<EOF
[Unit]
Description=node_exporter
Documentation=https://prometheus.io/
After=network.target
 
[Service]
Type=simple
ExecStart=/usr/local/mysqld_exporter/mysqld_exporter \\
--config.my-cnf=/usr/local/mysqld_exporter/.my.cnf
Restart=on-failure
 
[Install]
WantedBy=multi-user.target
EOF

.my.cnf：

cat > .my.cnf <<EOF
[client]
host=192.168.0.123
user=root
password=123456
EOF

dingtalk.service：

cat > /usr/lib/systemd/system/dingtalk.service <<EOF
[Unit]
Description=dingtalk
After=network.target

[Service]
ExecStart=/usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk \
--config.file=/usr/local/prometheus-webhook-dingtalk/config.yml \
--web.listen-address=0.0.0.0:8060 \
--web.enable-ui \
--web.enable-lifecycle

Restart=on-failure
KillMode=process
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

blackbox_exporter.service：

cat > /usr/lib/systemd/system/blackbox_exporter.service <<EOF
[Unit]
Description=blackbox_exporter
After=network.target
 
[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/blackbox_exporter/blackbox_exporter \
    --config.file=/usr/local/blackbox_exporter/blackbox.yml \
    --web.listen-address ":9115"
 
Restart=on-failure
 
[Install]
WantedBy=multi-user.target
EOF

blackbox.yml：

modules:
  http_2xx:
    prober: http
    timeout: 8s
    http:
      method: GET
      preferred_ip_protocol: "ip4" # 改成ipv4的默认不写是ipv6
      ip_protocol_fallback: false
  http_post_2xx:
    prober: http
    http:
      method: POST
      preferred_ip_protocol: "ip4"
      ip_protocol_fallback: false
  tcp_connect:
    prober: tcp
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp


监控elasticsearch：

github.com/prometheus-community/elasticsearch_exporter/releases

elasticsearch_exporter.service：

cat > /etc/systemd/system/elasticsearch_exporter.service <<EOF
[Unit]
Description=Elasticsearch Exporter Service
Documentation=https://github.com/prometheus-community/elasticsearch_exporter
After=network.target elasticsearch.service

[Service]
Type=simple
User=root
Group=root
ExecStart=/usr/local/elasticsearch_exporter/elasticsearch_exporter \\
  --es.uri=http://localhost:9200 \\
  --es.all \\
  --es.indices \\
  --es.shards
Restart=on-failure
RestartSec=30s
Environment="ES_EXPORTER_OPTS=--log.level=info"
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

elasticsearch dashboard：

grafana.com/grafana/dashboards/2322


监控mongodb：

https://github.com/percona/mongodb_exporter

创建监控用户：

use admin
db.createUser({ 
    user: "prometheus",
    pwd: "prometheus",
    roles: [
        { role: "read", db: "admin" },
        { role: "readAnyDatabase", db: "admin" },
        { role: "clusterMonitor", db: "admin" }
    ]
});

mongodb_exporter.service：

cat > /etc/systemd/system/mongodb_exporter.service <<EOF
[Unit]
Description=MongoDB Exporter
After=network.target
Documentation=https://github.com/percona/mongodb_exporter

[Service]
User=root
Group=root

#EnvironmentFile=/etc/default/mongodb_exporter

ExecStart=/usr/local/mongodb_exporter/mongodb_exporter \\
  --mongodb.user=prometheus \\
  --mongodb.password=prometheus \\
  --mongodb.uri=mongodb://192.168.0.123:27018 \\
  --web.listen-address=:9216 \\
  --compatible-mode \\
  --collector.diagnosticdata \\
  --collector.replicasetstatus

Restart=on-failure
RestartSec=5s
TimeoutStopSec=10s

StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=mongodb_exporter

[Install]
WantedBy=multi-user.target
EOF


监控postgresql：

下载地址：

github.com/prometheus-community/postgres_exporter

postgres_exporter.service

cat > /etc/systemd/system/postgres_exporter.service <<EOF
[Unit]
Description=Prometheus PostgreSQL Exporter
After=network.target

[Service]
User=root
Group=root
Environment="DATA_SOURCE_NAME=postgresql://postgres:postgres@172.16.43.249:5432/postgres?sslmode=disable"
ExecStart=/usr/local/postgres_exporter/postgres_exporter \\
    --web.listen-address=:9187 \\
    --log.level=info
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target
EOF


prometheus.yml

# 全局配置
global:
  scrape_interval:     15s # 采集数据间隔
  evaluation_interval: 15s # 评估告警周期，报警要持续15秒才发送alertmanager

# 报警配置
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - 192.168.0.19:9093

# 报警规则
rule_files:
   - "first_rules.yml"
   - "second_rules.yml"

# 静态配置监控
scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ['localhost:9090']
      labels:
        region: local # 添加标签

  - job_name: 'node_exporter'
    static_configs:
    - targets: ['192.168.0.19:9100']
    
# 文件发现
  - job_name: 'nodes'
    file_sd_configs:
    - files:
      - targets/nodes.yml
      refresh_interval: 60m

  - job_name: 'elasticsearch'
    scrape_interval: 60s
    scrape_timeout: 25s
    metrics_path: /metrics
    static_configs:
      - targets: 
        - '172.16.37.126:9114'
    relabel_configs:
      - source_labels: [__address__]
        target_label: cluster
        replacement: '172.16.37.126'

  - job_name: 'mysqld'
    file_sd_configs:
      - files: 
        - targets/mysqld.yml
        refresh_interval: 1m

  - job_name: 'mongodb'
    scrape_interval: 60s
    scrape_timeout: 25s
    static_configs:
      - targets: 
        - '172.16.43.249:9216'
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        replacement: 'mongodb-primary'

  - job_name: 'postgres'
    static_configs:
      - targets: 
        - '172.16.43.249:9187'

targets/nodes.yaml

- labels:
   region: local
   type: virtual
  targets:
  - 10.32.0.12:9100
  - 10.32.0.13:9100
- labels:
   region: dev
   type: physical
  targets:
  - 10.32.0.14:9100
  - 10.32.0.15:9100

重载配置：

curl -X POST http://localhost:9090/-/reload


alertmanager.yml

global:
  resolve_timeout: 5m
  smtp_smarthost: "smtp.163.com:465"
  smtp_from: "chuxiangyi_com@163.com"
  smtp_auth_username: "chuxiangyi_com@163.com"
  smtp_auth_password: "123456"
  smtp_require_tls: false

route:
  group_by: ['alertname','instance'] # 消息按照这个标签分组，一起发送一组消息（一条消息中有多个告警信息）,这个分组写不写不影响路由，主要是和group_wait一起配合使用
  group_wait: 10s # 在这个时间内将收到的告警消息合在一起发送出去，group_by组合使用
  group_interval: 10s # 相同组告警时间间隔
  repeat_interval: 1h # 报警未解决重复报警时间间隔
  receiver: 'email' # 选择填写报警名称，默认路由
  routes: # 子路由，将消息再分组发送到不同的消息出口
  - match: # 匹配标签，多个标签为逻辑与
      env: prod # 这个标签在prometheus的配置文件或alertmanager的配置文件中都可以自己添加
    group_by: ['severity'] # 消息分组一起发送
    receiver: 'prd'
  - match_re: # 使用正则匹配标签
      cluster: redis|mysql
    receiver: 'dev'

receivers: # 接收报警的方式
- name: "email" # 报警方式名称
  email_configs: # 邮件报警，需要在全局中配置SMTP
  - to: "250994559@qq.com"
- name: 'dev'
  webhook_configs: # webhook方式发送报警信息
  - url: 'http://127.0.0.1:5001/'
- name: 'prd'
  webhook_configs: # webhook方式发送报警信息
  - url: 'http://127.0.0.1:5001/'

inhibit_rules: # 告警抑制
  - source_match: # 匹配到如下两个标签开始记录
      alertname: node_exporter_26
      severity: critical
    target_match: # 匹配到如下标签和equal中定义的标签，且equal中的标签要和匹配的标签值一样
      severity: critical
    equal:
    - cluster

重载配置：

curl -XPOST localhost:9093/-/reload


开源告警规则汇总：常用告警规则都有

github.com/samber/awesome-prometheus-alerts

image.png


飞书示例：

17452194932406_upload.png

alertmanager.yml

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 1m
  repeat_interval: 1m
  receiver: 'web.hook'
receivers:
  - name: 'web.hook'
    webhook_configs:
      - url: 'http://0.0.0.0:8080/prometheusalert?type=fs&tpl=prometheus-fs&fsurl=https://open.feishu.cn/open-apis/bot/v2/hook/c534758e-e5db-4dca-8d5e-xxxxxxx'
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']


grafana：

国内下载源：

mirrors.huaweicloud.com/grafana/

默认用户名和密码：

admin/admin

常用的dashboard图：

https://github.com/starsliao/Prometheus

配置文件在：

/etc/grafana/grafana.ini

Node Exporter的Grafana模版：

英文版本：

https://grafana.com/grafana/dashboards/11074

中文版本：

https://grafana.com/grafana/dashboards/8919


报警规则：

这个规则保存为一个文件，配置到prometheus的配置文件中的rule_files字段下，以列表的形式。

rule_files:
   - "first_rules.yml"
   - "second_rules.yml"

规则：

groups:
- name: node_exporter
  rules:
  - alert: node_exporter
    expr: up{job="node_exporter"} == 0
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: node_exporter down
      description: node_exporter down

告警抑制：

inhibit_rules:
  - source_match:
      alertname: node_exporter_26 # alertname是规则中 rules.alert 定义的值
      severity: critical
    target_match:
      severity: critical
    equal:
    - cluster

如下两个告警实例：

A告警：

groups:
- name: node_exporter_26 down
  rules:
  - alert: node_exporter_26
    expr: up{job="node_exporter-26"} == 0
    for: 1m
    labels:
      severity: critical
      cluster: gitlab
    annotations:
      summary: node_exporter_26 down
      description: node_exporter_26 down

B告警：

groups:
- name: haproxy down
  rules:
  - alert: haproxy
    expr: up{job="haproxy_exporter"} == 0
    for: 1m
    labels:
      severity: critical
      cluster: gitlab
    annotations:
      summary: haproxy down

        当有告警时且匹配到source_match下定义的标签时开始记录，当再有告警出现时匹配到target_match中的标签，且这个告警中的标签和equal中的定义的标签名的值要相同时就会抑制告警。如上示例这种A告警出现后B告警就会被抑制。


常用报警规则： 对节点、CPU、内存、磁盘进行监控和报警。

groups:
- name: example
  rules:

  - alert: InstanceDown
    expr: up == 0
    for: 5m
    labels:
      severity: page
    annotations:
      summary: "Instance {{ $labels.instance }} down"
      description: "{{ $labels.instance }} of job {{ $labels.job }} {{ $labels.region }} has been down for more than 5 minutes."

  - alert: MemUsageHigh
    expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 80
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: "High memory usage on {{ $labels.instance }}"
      description: "{{ $labels.instance }} {{ $labels.region }}"

  - alert: CPUUsageHigh
    expr: (1-(sum(increase(node_cpu_seconds_total{mode="idle"}[1m]))by(instance))/(sum(increase(node_cpu_seconds_total[1m]))by(instance)))*100 > 60
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: "High CPU usage on {{ $labels.instance }}"
      description: "{{ $labels.instance }} {{ $labels.region }}"

  - alert: DiskUsageHigh
    expr: (1 - (node_filesystem_free_bytes{device=~"/dev.*"} / node_filesystem_size_bytes{device=~"/dev.*"})) * 100 > 80
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: "High Disk usage on {{ $labels.instance }}"
      description: "{{ $labels.instance }} {{ $labels.region }}"


几个时间参数：

prometheus.yml

global:
  scrape_interval: 15s # 数据采集间隔
  evaluation_interval: 15s # 评估告警周期，报警要持续15秒才发送alertmanager
  scrape_timeout: 10s # 数据采集超时时间默认10s

alertmanager.yml

# route标记：告警如何发送分配
route:
  group_by: ['alertname'] # group_by：采用哪个标签作为分组的依据
  group_wait: 10s # group_wait：分组等待的时间
  group_interval: 10s # 上下两组发送告警的间隔时间
  repeat_interval: 1m # 重复发送告警时间。默认1h


prometheus常用配置：

# my global config
global:
  scrape_interval:     30s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 30s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
       - localhost:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - "rules/*.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
    - targets: ['localhost:9090']

  - job_name: 'nodes'
    file_sd_configs:
    - files:
      - target/nodes.yml
      refresh_interval: 10m

#  - job_name: 'openvpn-metrics'
#    scrape_interval: 15s
#    static_configs:
#    - targets: ['192.168.199.9:9176']

#  - job_name: 'kubernetes-pods-eureka'
#    metrics_path: /actuator/prometheus
#    basic_auth:
#      username: 'user'
#      password: '4yDGnT06'
#    eureka_sd_configs:
#    - server: 'http://autotest-eureka.demo.com/eureka'
#    relabel_configs:
#    - source_labels: 
#      - __meta_eureka_app_name
#      separator: ;
#      regex: (.*)
#      target_label: appname
#      replacement: $1
#      action: replace
#    - action: labelmap
#      regex: __meta_eureka_app_instance_(.+)
#    - regex: metadata_user_(.+)
#      action: labeldrop

  - job_name: 'ping_status'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets:
        - 192.168.199.136
        - 192.168.199.148
        - 192.168.199.163
        - 192.168.199.186
        - 192.168.199.222
        - 192.168.199.250
        labels:
          group: 'icmp'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115

  - job_name: 'port_status'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
    - targets:
      - 101.132.237.99:7500
      labels:
        group: 'tcp'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9115

  - job_name: "http_status"
    scrape_interval: 1m
    metrics_path: /probe
    params:
      module: [http_2xx]
    file_sd_configs: 
    - refresh_interval: 1m
      files: 
      - target/http.yml
    relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      replacement: localhost:9115


  - job_name: 'vmware_vcenter'
    metrics_path: '/metrics'
    static_configs:
      - targets:
        - 'vcenter.demo.com'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: localhost:9272

  - job_name: 'mysqld'     
    file_sd_configs:
      - files: ['/usr/local/prometheus/target/mysqld.yml']
        refresh_interval: 1m

  - job_name: 'redis'
    file_sd_configs:
      - files: ['/usr/local/prometheus/target/redis.yml']
        refresh_interval: 1m

  - job_name: 'windows'
    file_sd_configs:
      - files: ['/usr/local/prometheus/target/windows.yml']
        refresh_interval: 1m

  - job_name: 'portmapping'
    metrics_path: '/metrics'
    static_configs:
      - targets:
        - 'portmapping.demo.com'

  - job_name: 'snmp'
    scrape_interval: 60s
    static_configs:
      - targets:
        - 192.168.101.1  # SNMP device.
        #- switch.local # SNMP device.
        #- tcp://192.168.1.3:1161  # SNMP device using TCP transport and custom port.
    metrics_path: /snmp
    params:
      auth: [public_v2]
      module: [if_mib]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9116  # The SNMP exporter's real hostname:port.

  # Global exporter-level metrics
  - job_name: 'snmp_exporter'
    static_configs:
      - targets: ['localhost:9116']

  - job_name: 'elasticsearch_exporter'
    scrape_interval: 60s
    scrape_timeout: 25s
    metrics_path: /metrics
    static_configs:
      - targets: 
        - '172.16.37.126:9114'

    relabel_configs:
      - source_labels: [__address__]
        target_label: cluster
        replacement: '172.16.37.126'
TOP
Quick Search
epel istio argocd kubernetes jenkins harbor docker gitlab prometheus PromQL Ceph php golang kibana elasticsearch filebeat logstash zabbix vCenter exsi mongodb redis mysql nginx lvs keepalived ansible saltstack
苏ICP备15026757号-3
Copyright © 2015-2022 scriptjc.com All Rights Reserved