Backup monitoring

plays/monitoring.yml

@@ -36,6 +36,29 @@
     - ninjabot
     - grafana
 
+# Deploy backup Prometheus on backup server
+- hosts: odlyd.adm.crans.org
+  vars:
+    # only critical infra
+    prometheus:
+      node_targets:
+        - odlyd.adm.crans.org  # me, myself and I
+        - zamok.adm.crans.org  # parce que WeeChat c'est critique
+        - thot.adm.crans.org  # la bdd adh est critique... enfin a skip
+        - zbee.adm.crans.org  # zbeu! la bay!
+        - stitch.adm.crans.org  # last hope virtu
+      ups_snmp_targets:
+        - pulsar.adm.crans.org  # 0B
+        - quasar.adm.crans.org  # 4J
+
+    snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
+  roles:
+    - prometheus
+    - prometheus-alertmanager
+    - prometheus-snmp-exporter
+    - ninjabot
+
+
 # Monitor all hosts
 - hosts: server,test_vm
   vars:

roles/prometheus/tasks/main.yml

@@ -39,18 +39,21 @@
   copy:
     content: "{{ [{'targets': prometheus.unifi_snmp_targets}] | to_nice_json }}"
     dest: /etc/prometheus/targets_unifi_snmp.json
+  when: prometheus.unifi_snmp_targets is defined
 
 # We don't need to restart Prometheus when updating nodes
 - name: Configure Prometheus Apache targets
   copy:
     content: "{{ [{'targets': prometheus.apache_targets}] | to_nice_json }}"
     dest: /etc/prometheus/targets_apache.json
+  when: prometheus.apache_targets is defined
 
 # We don't need to restart Prometheus when updating nodes
 - name: Configure Prometheus Blackbox targets
   copy:
     content: "{{ [{'targets': prometheus.blackbox_targets}] | to_nice_json }}"
     dest: /etc/prometheus/targets_blackbox.json
+  when: prometheus.blackbox_targets is defined
 
 - name: Activate prometheus service
   systemd:

roles/prometheus/templates/prometheus/prometheus.yml.j2

@@ -61,6 +61,7 @@ scrape_configs:
       - target_label: __address__
         replacement: 127.0.0.1:9116
 
+{% if prometheus.unifi_snmp_targets is defined %}
   - job_name: unifi_snmp
     file_sd_configs:
       - files:
@@ -75,7 +76,9 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9116
+{% endif %}
 
+{% if prometheus.apache_targets is defined %}
   - job_name: apache
     file_sd_configs:
       - files:
@@ -87,7 +90,9 @@ scrape_configs:
       - source_labels: [instance]
         target_label: __address__
         replacement: '$1:9117'
+{% endif %}
 
+{% if prometheus.blackbox_targets is defined %}
   - job_name: blackbox
     file_sd_configs:
       - files:
@@ -102,6 +107,7 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9115
+{% endif %}
 
   - job_name: mtail
     static_configs: