From 5f5201ddcfd66ed7a72945516597e4fc4762958f Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Fri, 15 Nov 2019 14:23:47 +0100 Subject: [PATCH] Add prometheus blackbox --- monitoring.yml | 13 ++++++-- .../handlers/main.yml | 0 .../tasks/main.yml | 0 .../handlers/main.yml | 5 ++++ .../tasks/main.yml | 23 ++++++++++++++ .../handlers/main.yml | 0 .../tasks/main.yml | 0 .../handlers/main.yml | 5 ++++ roles/prometheus-snmp-exporter/tasks/main.yml | 30 +++++++++++++++++++ .../templates/prometheus/snmp.yml.j2 | 0 roles/prometheus/handlers/main.yml | 5 ---- roles/prometheus/tasks/main.yml | 26 +++++----------- .../templates/prometheus/prometheus.yml.j2 | 15 ++++++++++ 13 files changed, 96 insertions(+), 26 deletions(-) rename roles/{prometheus-apache => prometheus-apache-exporter}/handlers/main.yml (100%) rename roles/{prometheus-apache => prometheus-apache-exporter}/tasks/main.yml (100%) create mode 100644 roles/prometheus-blackbox-exporter/handlers/main.yml create mode 100644 roles/prometheus-blackbox-exporter/tasks/main.yml rename roles/{prometheus-node => prometheus-node-exporter}/handlers/main.yml (100%) rename roles/{prometheus-node => prometheus-node-exporter}/tasks/main.yml (100%) create mode 100644 roles/prometheus-snmp-exporter/handlers/main.yml create mode 100644 roles/prometheus-snmp-exporter/tasks/main.yml rename roles/{prometheus => prometheus-snmp-exporter}/templates/prometheus/snmp.yml.j2 (100%) diff --git a/monitoring.yml b/monitoring.yml index eead23b1..57a008fa 100644 --- a/monitoring.yml +++ b/monitoring.yml @@ -9,22 +9,31 @@ - targets: [pulsar.adm.crans.org] prometheus_unifi_snmp_targets: - targets: "{{ groups['crans_unifi'] | list | sort }}" + prometheus_blackbox_targets: + - targets: + - https://crans.org + - https://www.crans.org + - https://grafana.crans.org + - https://wiki.crans.org + - https://pad.crans.org prometheus_apache_targets: - targets: [zamok.adm.crans.org] snmp_unifi_password: "{{ vault_snmp_unifi_password }}" roles: - prometheus - prometheus-alertmanager + - prometheus-snmp-exporter + - prometheus-blackbox-exporter # Monitor all hosts - hosts: server roles: - - prometheus-node + - prometheus-node-exporter # Export apache metrics - hosts: zamok.adm.crans.org roles: - - prometheus-apache + - prometheus-apache-exporter # Deploy grafana - hosts: fy.adm.crans.org diff --git a/roles/prometheus-apache/handlers/main.yml b/roles/prometheus-apache-exporter/handlers/main.yml similarity index 100% rename from roles/prometheus-apache/handlers/main.yml rename to roles/prometheus-apache-exporter/handlers/main.yml diff --git a/roles/prometheus-apache/tasks/main.yml b/roles/prometheus-apache-exporter/tasks/main.yml similarity index 100% rename from roles/prometheus-apache/tasks/main.yml rename to roles/prometheus-apache-exporter/tasks/main.yml diff --git a/roles/prometheus-blackbox-exporter/handlers/main.yml b/roles/prometheus-blackbox-exporter/handlers/main.yml new file mode 100644 index 00000000..72a6fc53 --- /dev/null +++ b/roles/prometheus-blackbox-exporter/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus-blackbox-exporter + service: + name: prometheus-blackbox-exporter + state: restarted diff --git a/roles/prometheus-blackbox-exporter/tasks/main.yml b/roles/prometheus-blackbox-exporter/tasks/main.yml new file mode 100644 index 00000000..cd93f508 --- /dev/null +++ b/roles/prometheus-blackbox-exporter/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Install Prometheus Blackbox exporter + apt: + update_cache: true + name: prometheus-blackbox-exporter + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Make Prometheus Blackbox exporter listen on localhost only + lineinfile: + path: /etc/default/prometheus-blackbox-exporter + regexp: '^ARGS=' + line: > + ARGS='--config.file /etc/prometheus/blackbox.yml + --web.listen-address=\"localhost:9115\"'" + notify: Restart prometheus-blackbox-exporter + +- name: Activate prometheus Blackbox exporter service + systemd: + name: prometheus-blackbox-exporter + enabled: true + state: started diff --git a/roles/prometheus-node/handlers/main.yml b/roles/prometheus-node-exporter/handlers/main.yml similarity index 100% rename from roles/prometheus-node/handlers/main.yml rename to roles/prometheus-node-exporter/handlers/main.yml diff --git a/roles/prometheus-node/tasks/main.yml b/roles/prometheus-node-exporter/tasks/main.yml similarity index 100% rename from roles/prometheus-node/tasks/main.yml rename to roles/prometheus-node-exporter/tasks/main.yml diff --git a/roles/prometheus-snmp-exporter/handlers/main.yml b/roles/prometheus-snmp-exporter/handlers/main.yml new file mode 100644 index 00000000..9b65002c --- /dev/null +++ b/roles/prometheus-snmp-exporter/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus-snmp-exporter + service: + name: prometheus-snmp-exporter + state: restarted diff --git a/roles/prometheus-snmp-exporter/tasks/main.yml b/roles/prometheus-snmp-exporter/tasks/main.yml new file mode 100644 index 00000000..8ee759e1 --- /dev/null +++ b/roles/prometheus-snmp-exporter/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Install Prometheus SNMP exporter + apt: + update_cache: true + name: prometheus-snmp-exporter + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Make Prometheus SNMP exporter listen on localhost only + lineinfile: + path: /etc/default/prometheus-snmp-exporter + regexp: '^ARGS=' + line: "ARGS=\"--web.listen-address=127.0.0.1:9116\"" + notify: Restart prometheus-snmp-exporter + +# This file store SNMP OIDs +- name: Configure Prometheus SNMP exporter + template: + src: "prometheus/snmp.yml.j2" + dest: "/etc/prometheus/snmp.yml" + mode: 0600 + owner: prometheus + notify: Restart prometheus-snmp-exporter + +- name: Activate prometheus SNMP exporter service + systemd: + name: prometheus-snmp-exporter + enabled: true + state: started diff --git a/roles/prometheus/templates/prometheus/snmp.yml.j2 b/roles/prometheus-snmp-exporter/templates/prometheus/snmp.yml.j2 similarity index 100% rename from roles/prometheus/templates/prometheus/snmp.yml.j2 rename to roles/prometheus-snmp-exporter/templates/prometheus/snmp.yml.j2 diff --git a/roles/prometheus/handlers/main.yml b/roles/prometheus/handlers/main.yml index 670847ba..4214def5 100644 --- a/roles/prometheus/handlers/main.yml +++ b/roles/prometheus/handlers/main.yml @@ -3,8 +3,3 @@ service: name: prometheus state: restarted - -- name: Restart prometheus-snmp-exporter - service: - name: prometheus-snmp-exporter - state: restarted diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index 54793522..26b74bf7 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -2,9 +2,7 @@ - name: Install Prometheus apt: update_cache: true - name: - - prometheus - - prometheus-snmp-exporter + name: prometheus register: apt_result retries: 3 until: apt_result is succeeded @@ -24,22 +22,6 @@ - alert.rules.yml - django.rules.yml -- name: Make Prometheus snmp-exporter listen on localhost only - lineinfile: - path: /etc/default/prometheus-snmp-exporter - regexp: '^ARGS=' - line: "ARGS=\"--web.listen-address=127.0.0.1:9116\"" - notify: Restart prometheus-snmp-exporter - -# This file store SNMP OIDs -- name: Configure Prometheus snmp-exporter - template: - src: "prometheus/snmp.yml.j2" - dest: "/etc/prometheus/snmp.yml" - mode: 0600 - owner: prometheus - notify: Restart prometheus-snmp-exporter - # We don't need to restart Prometheus when updating nodes - name: Configure Prometheus nodes copy: @@ -64,6 +46,12 @@ content: "{{ prometheus_apache_targets | to_nice_json }}" dest: /etc/prometheus/targets_apache.json +# We don't need to restart Prometheus when updating nodes +- name: Configure Prometheus Blackbox targets + copy: + content: "{{ prometheus_blackbox_targets | to_nice_json }}" + dest: /etc/prometheus/targets_blackbox.json + - name: Activate prometheus service systemd: name: prometheus diff --git a/roles/prometheus/templates/prometheus/prometheus.yml.j2 b/roles/prometheus/templates/prometheus/prometheus.yml.j2 index f7e11367..0f1b6c67 100644 --- a/roles/prometheus/templates/prometheus/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus/prometheus.yml.j2 @@ -88,6 +88,21 @@ scrape_configs: target_label: __address__ replacement: '$1:9117' + - job_name: blackbox + file_sd_configs: + - files: + - '/etc/prometheus/targets_blackbox.json' + metrics_path: /probe + params: + module: [http_2xx] # Look for a HTTP 200 response. + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 + - job_name: django scheme: https static_configs: