vim /usr/local/alertmanager/tmpl/email_template.tmpl
{{ define "email.html" }}
{{- if gt (len .Alerts.Firing) 0 -}}
{{ range .Alerts }}
=========start==========<br>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} <br>
告警类型: {{ .Labels.alertname }} <br>
告警主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }} <br>
=========end==========<br>
{{ end }}{{ end -}}
{{- if gt (len .Alerts.Resolved) 0 -}}
{{ range .Alerts }}
=========start==========<br>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} <br>
告警类型: {{ .Labels.alertname }} <br>
告警主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }} <br>
恢复时间: {{ .EndsAt.Format "2006-01-02 15:04:05" }} <br>
=========end==========<br>
{{ end }}{{ end -}}
{{- end }}

应用模版

]# vim /usr/local/alertmanager/conf/alertmanager.yml
global:
...
templates:            #加下面两行加载模板文件
  - '../tmpl/*.tmpl'  #相对路径是相对于altermanager.yml文件的路径
...
# 收信人员
receivers:
- name: 'email'
 email_configs:
  - to: 'root@xiaoming.com'
   send_resolved: true
   headers: { Subject: "[WARN] 报警邮件"}   #添加此行,定制邮件标题
   html: '{{ template "test.html" . }}'    #添加此行,调用模板显示邮件正文
    #html: '{{ template "email.html" . }}'   #添加此行,调用模板显示邮件正文

然后再重启prometheus进行验证

⑥告警路由

#指定路由分组
vim /usr/local/alertmanager/conf/alertmanager.yml
# 全局配置
global:
 resolve_timeout: 5m
 smtp_smarthost: 'smtp.qq.com:25'
 smtp_from: '123456@qq.com'
 smtp_auth_username: '123456@qq.com'
 smtp_auth_password: 'abefxqzcnxhqaebieb'
 smtp_hello: 'qq.com'
 smtp_require_tls: false
# 模板配置
templates:
  - '../tmpl/*.tmpl'
# 路由配置
#新版：使用指令matchers替换match和match_re,如下示例
route:
 group_by: ['instance', 'cluster']
 group_wait: 10s
 group_interval: 10s
 repeat_interval: 10s
 receiver: 'email'
 routes:
  - receiver: 'leader-team'
   matchers:
    - severity = "critical"
  - receiver: 'ops-team'
   matchers:
    - severity =~ "^(warning)$"
 #子路由  
  #- matchers
  # - severity = "critical"
  # - job =~ "mysql|java"
  # receiver: 'leader-team'
  # routes: #支持routes嵌套和分级
  # - matchers:
  #   - job =~ "mysql"
  #   receiver: 'dba-team'
  #- matchers
  # - severity =~ "warning"
  # - job = "node_exporter"
  # receiver: 'ops-team'    
  
# 收信人员
receivers:
- name: 'email'
 email_configs:
  - to: 'root@wangxiaochun.com'
   send_resolved: true
   html: '{{ template "test.html" . }}'
   headers: { Subject: "[WARN] 报警邮件"}
- name: 'leader-team'
 email_configs:
  - to: 'root@wangxiaochun.com'
   html: '{{ template "test.html" . }}'
   headers: { Subject: "[CRITICAL] 应用服务报警邮件"}
   send_resolved: true  #
- name: 'ops-team'
 email_configs:
  - to: 'root@xiaoming.com'
   html: '{{ template "test.html" . }}'
   headers: { Subject: "[WARNNING] QPS负载报警邮件"}
   send_resolved: true

#检查语法
amtool check-config /usr/local/alertmanager/conf/alertmanager.yml

⑦ 告警抑制

在alertmanager里面添加下面的代码
# 抑制措施                          
inhibit_rules: 
- source_match: 
   severity: critical     #被依赖的告警服务
 target_match:             
   severity: warning      #依赖的告警服务
 equal:
    - instance

#重启alertmanager服务
systemctl reload alertmanager.service

#创建Alertmanager配置文件/etc/prometheus/alertmanager-ha.yml, 为了验证Alertmanager的集群
行为，这里在本地启动一个webhook服务用于打印Alertmanager发送的告警通知信息。
route:
 receiver: 'default-receiver'
receivers:
  - name: default-receiver
   webhook_configs:
    - url: 'http://127.0.0.1:5001/'
    
# 获取alertmanager提供的webhook示例，如果该目录下定义了main函数，go get会自动将其编译成可执行文件
go get github.com/prometheus/alertmanager/examples/webhook
export PATH=$GOPATH/bin:$PATH  # 设置环境变量指向GOPATH的bin目录
webhook  # 启动服务

#a1: 
alertmanager  --web.listen-address=":9093" --cluster.listen-address="127.0.0.1:8001"
--config.file=/etc/prometheus/alertmanager-ha.yml  --storage.path=/data/alertmanager/ 
--log.level=debug
#a2: 
alertmanager  --web.listen-address=":9094" --cluster.listen-address="127.0.0.1:8002"
--cluster.peer=127.0.0.1:8001 --config.file=/etc/prometheus/alertmanager-ha.yml  --
storage.path=/data/alertmanager2/ --log.level=debug
#a3: 
alertmanager  --web.listen-address=":9095" --cluster.listen-address="127.0.0.1:8003"
--cluster.peer=127.0.0.1:8001 --config.file=/etc/prometheus/alertmanager-ha.yml  --
storage.path=/data/alertmanager2/ --log.level=debug



#创建Promthues集群配置文件/etc/prometheus/prometheus-ha.yml，完整内容如下：
global:
 scrape_interval: 15s
 scrape_timeout: 10s
 evaluation_interval: 15s
rule_files:
  - /etc/prometheus/rules/*.rules
alerting:
 alertmanagers:
  - static_configs:
    - targets:
      - 127.0.0.1:9093
      - 127.0.0.1:9094
      - 127.0.0.1:9095
scrape_configs:
- job_name: prometheus
 static_configs:
  - targets:
    - localhost:9090
- job_name: 'node'
 static_configs:
  - targets: ['localhost:9100']
  
  
#同时定义告警规则文件/etc/prometheus/rules/hoststats-alert.rules，如下所示：
groups:
- name: hostStatsAlert
 rules:
  - alert: hostCpuUsageAlert
   expr: sum(avg without (cpu)(irate(node_cpu{mode!='idle'}[5m]))) by (instance) * 
100 > 50
    for: 1m
   labels:
     severity: page
   annotations:
     summary: "Instance {{ $labels.instance }} CPU usgae high"
     description: "{{ $labels.instance }} CPU usage above 50% (current value: {{ 
$value }})"
  - alert: hostMemUsageAlert
   expr: (node_memory_MemTotal - node_memory_MemAvailable)/node_memory_MemTotal * 
100 > 85
    for: 1m
   labels:
     severity: page
   annotations:
     summary: "Instance {{ $labels.instance }} MEM usgae high"
     description: "{{ $labels.instance }} MEM usage above 85% (current value: {{ 
$value }})"
      
#创建prometheus.procfile文件，创建两个Promthues节点，分别监听9090和9091端口:
#p1: 
prometheus --config.file=/etc/prometheus/prometheus-ha.yml --
storage.tsdb.path=/data/prometheus/ --web.listen-address="127.0.0.1:9090"
#p2: 
prometheus --config.file=/etc/prometheus/prometheus-ha.yml --
storage.tsdb.path=/data/prometheus2/ --web.listen-address="127.0.0.1:9091"
#node_exporter
node_exporter: node_exporter -web.listen-address="0.0.0.0:9100"

docker run -d \
-p 8080:8080 \
-e PA_LOGIN_USER=prometheusalert \
-e PA_LOGIN_PASSWORD=prometheusalert \
-e PA_TITLE=PrometheusAlert \
-e PA_OPEN_FEISHU=1 \
-e PA_OPEN_DINGDING=1 \
-e PA_OPEN_WEIXIN=1 \
feiyu563/prometheus-alert:latest

#浏览器访问,用户名/密码默认是prometheusalert/prometheusalert
http://prometheusalert.wang.org:8080

8.服务发现

- job_name: 'node_exporter'
   static_configs:
    - targets: ['10.0.0.101:9100']

#创建目标目录
mkdir /usr/local/prometheus/conf/targets
cd /usr/local/prometheus/conf/targets

#添加linux主机目标
ls /usr/local/prometheus/conf/targets/
prometheues-flask.yml prometheues-node.yml prometheues-server.yml

#创建prometheus的服务配置
cat prometheues-server.yml
- targets:
  - 10.0.0.101:9090
 labels:
   app: prometheus-server
   job: prometheus-server
    
cat prometheues-node.yml
- targets:
  - 10.0.0.101:9100
 labels:
   app: prometheus
   job: prometheus
- targets:
  - 10.0.0.104:9100
  - 10.0.0.105:9100
 labels:
   app: node-exporter
   job: node
cat prometheues-flask.yml 
- targets:
  - 10.0.0.101:8000
 labels:
   app: flask-web
   job: prometheus-flask
   
#修改prometheus的配置文件，让它自动加载文件中的节点信息
cd /usr/local/prometheus/conf
cp prometheus.yml{,.bak}

#编辑配置文件
#cat /usr/local/prometheus/conf/prometheus.yml（全局配置）
global:
 scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is 
every 1 minute.
 evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 
minute.
  # scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
 alertmanagers:
    - static_configs:
        - targets:
rule_files:
scrape_configs:
  - job_name: "file_sd_prometheus"
   scrape_interval: 10s  #指定抓取数据的时间间隔
   file_sd_configs:
    - files:
      - targets/prometheues-server.yml
     refresh_interval: 10s  #指定重读文件的时间间隔,默认值5m
  - job_name: 'file_sd_node_exporter'
   file_sd_configs:
    - files:
      - targets/prometheues-node.yml
     refresh_interval: 10s
  - job_name: 'file_sd_flask_web'
   file_sd_configs:
    - files:
      - targets/prometheues-flask.yml
     refresh_interval: 10s

#配置文件语法检查
promtool check config prometheus.yml

#重启服务
systemctl reload prometheus.service

#稍等几秒钟，到浏览器中查看监控目标
#结果显示：所有的节点都添加完毕了，而且每个节点都有自己的标签信息

9.DNS 服务发现

#安装软件
apt update && apt -y install bind9 bind9utils bind9-doc bind9-host 
named -v

#查看部署软件
dpkg -L bind9 | grep named.conf
/etc/bind/named.conf
/etc/bind/named.conf.default-zones
/etc/bind/named.conf.local
/etc/bind/named.conf.options

#定制正向解析zone的配置
cat >> /etc/bind/named.conf.default-zones 
//定制网站主域名的zone配置
zone "wang.org" {
       type master;
       file "/etc/bind/wang.org.zone";
};

#定制主域名的zone文件
vim /etc/bind/wang.org.zone
;
; BIND reverse data file for local loopback interface
;
$TTL    604800
@       IN     SOA     ns.wang.org. admin.wang.org. (
                              1         ; Serial
                         604800         ; Refresh
                          86400         ; Retry
                        2419200         ; Expire
                         604800 )       ; Negative Cache TTL
;
       IN     NS     master
master IN     A       10.0.0.100
node1   IN     A       10.0.0.101
node2   IN     A       10.0.0.102
node3   IN     A       10.0.0.103
flask   IN     A       10.0.0.101

#检查配置文件
named-checkconf

#重启dns服务
rndc reload
systemctl restart named
systemctl status named

#配置prometheus服务器使用DNS域名服务器
vim /etc/netplan/01-netcfg.yaml
network:
 version: 2
 renderer: networkd
 ethernets:
   eth0:
     addresses:
      - 10.0.0.101/24
     gateway4: 10.0.0.2
     nameservers:
       search: [wang.org,wang.com]
       addresses: [10.0.0.100] #只保留当前的DNS服务器地址,别再加其它DNS服务器地址,否则DNS
解析有问题

#应用网络配置
netplan apply

#确认dns解析效果
dig node1.wang.org
host node1.wang.org
nslookup node1.wang.org

#添加SRV记录
# vim /etc/bind/wang.org.zone
... ...
node1   IN     A       10.0.0.101
node2   IN     A       10.0.0.102
node3   IN     A       10.0.0.103
flask   IN     A       10.0.0.101  #只有A记录

#添加下面的SRV记录,对应上面的三条A记录
_prometheus._tcp.wang.org. 1H IN SRV 10 10 9100 node1.wang.org.
_prometheus._tcp.wang.org. 1H IN SRV 10 10 9100 node2.wang.org.
_prometheus._tcp.wang.org. 1H IN SRV 10 10 9100 node3.wang.org.

#检查配置文件
named-checkconf

#生效
rndc reload

#测试解析
dig srv _prometheus._tcp.wang.org
host -t srv _prometheus._tcp.wang.org
nslookup -q=srv _prometheus._tcp.wang.org

# vim /usr/local/prometheus/conf/prometheus.yml
... 
scrape_configs:
  - job_name: "prometheus"
   .......
    
#添加下面所有行
  - job_name: 'dns_sd_flask'             #实现单个主机定制的信息解析，也支持DNS或/etc/hosts
文件实现解析
   dns_sd_configs:
    - names: ['flask.wang.org']
     type: A                            #指定记录类型，默认SRV
     port: 8000                         #不是SRV时，需要指定Port号
     refresh_interval: 10s
      
  - job_name: 'dns_sd_node_exporter'          #实现批量主机解析
   dns_sd_configs:
    - names: ['_prometheus._tcp.wang.org']    #SRV记录必须通过DNS的实现
     refresh_interval: 10s                   #指定DNS资源记录的刷新间隔,默认30s
   relabel_configs:                          #生成新的标签service，值为_prometheus._tcp
    - source_labels: ['__meta_dns_name']
     regex:         '(.+?)\.wang\.org'
     target_label:  'service'
     replacement:   '$1'
      
#重启prometheus
promtool check config /usr/local/prometheus/conf/prometheus.yml
systemctl reload prometheus

10.Consul 服务发现

docker pull consul:1.6.1 # 拉取指定版本

docker run -d -p 8500 :8500 --restart = always --name = consul consul:latest agent -server-bind = 10 .0.0.100 -client = 0 .0.0.0 -bootstrap-expect = 1 -ui

11 各种 Exporter