From 946250e32205d2606fd891895d87a9e6ca9797ee Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Mon, 28 Dec 2020 12:01:19 +0100 Subject: [PATCH] Monitor NGINX with Prometheus --- plays/monitoring.yml | 11 ++++++- .../handlers/main.yml | 10 +++++++ .../prometheus-nginx-exporter/tasks/main.yml | 29 +++++++++++++++++++ .../templates/nginx/status.j2 | 8 +++++ .../default/prometheus-node-exporter.j2 | 2 +- roles/prometheus/tasks/main.yml | 8 +++++ .../templates/prometheus/prometheus.yml.j2 | 14 +++++++++ 7 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 roles/prometheus-nginx-exporter/handlers/main.yml create mode 100644 roles/prometheus-nginx-exporter/tasks/main.yml create mode 100644 roles/prometheus-nginx-exporter/templates/nginx/status.j2 diff --git a/plays/monitoring.yml b/plays/monitoring.yml index adb21a07..3aa9960c 100755 --- a/plays/monitoring.yml +++ b/plays/monitoring.yml @@ -16,6 +16,9 @@ - https://grafana.crans.org - https://wiki.crans.org - https://pad.crans.org + nginx_targets: + - hodaur.adm.crans.org + - charybde.adm.crans.org apache_targets: [] # [zamok.adm.crans.org] snmp_unifi_password: "{{ vault_snmp_unifi_password }}" @@ -63,9 +66,15 @@ # Monitor all hosts - hosts: server,test_vm vars: - adm_ipv4: "{{ ansible_all_ipv4_addresses | ipaddr(adm_subnet) | first }}" + adm_ipv4: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ipv4 | first }}" roles: ["prometheus-node-exporter"] +# Export nginx metrics +- hosts: charybde.adm.crans.org,hodaur.adm.crans.org + vars: + adm_ipv4: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ipv4 | first }}" + roles: ["prometheus-nginx-exporter"] + # Export apache metrics #- hosts: zamok.adm.crans.org # vars: diff --git a/roles/prometheus-nginx-exporter/handlers/main.yml b/roles/prometheus-nginx-exporter/handlers/main.yml new file mode 100644 index 00000000..25a545df --- /dev/null +++ b/roles/prometheus-nginx-exporter/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: Restart prometheus-nginx-exporter + service: + name: prometheus-nginx-exporter + state: restarted + +- name: Restart nginx + service: + name: nginx + state: restarted diff --git a/roles/prometheus-nginx-exporter/tasks/main.yml b/roles/prometheus-nginx-exporter/tasks/main.yml new file mode 100644 index 00000000..6a398b79 --- /dev/null +++ b/roles/prometheus-nginx-exporter/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: Install prometheus-nginx-exporter + apt: + update_cache: true + name: prometheus-nginx-exporter + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Make prometheus-nginx-exporter listen on adm only + lineinfile: + path: /etc/default/prometheus-nginx-exporter + regexp: '^ARGS=' + line: | + ARGS="-web.listen-address={{ adm_ipv4 }}:9117" + notify: Restart prometheus-nginx-exporter + +- name: Configure nginx + template: + src: nginx/status.j2 + dest: /etc/nginx/sites-available/status + notify: Restart nginx + +- name: Activate nginx site + file: + src: /etc/nginx/sites-available/status + dest: /etc/nginx/sites-enabled/status + state: link + notify: Restart nginx diff --git a/roles/prometheus-nginx-exporter/templates/nginx/status.j2 b/roles/prometheus-nginx-exporter/templates/nginx/status.j2 new file mode 100644 index 00000000..d7051980 --- /dev/null +++ b/roles/prometheus-nginx-exporter/templates/nginx/status.j2 @@ -0,0 +1,8 @@ +{{ ansible_header | comment }} + +server { + listen 127.0.0.1:8080; + location = /stub_status { + stub_status; + } +} diff --git a/roles/prometheus-node-exporter/templates/default/prometheus-node-exporter.j2 b/roles/prometheus-node-exporter/templates/default/prometheus-node-exporter.j2 index 9610d2d4..819d243a 100644 --- a/roles/prometheus-node-exporter/templates/default/prometheus-node-exporter.j2 +++ b/roles/prometheus-node-exporter/templates/default/prometheus-node-exporter.j2 @@ -4,7 +4,7 @@ # Due to shell scaping, to pass backslashes for regexes, you need to double # them (\\d for \d). If running under systemd, you need to double them again # (\\\\d to mean \d), and escape newlines too. -ARGS="--web.listen-address={{ query('ldap', 'ip', ansible_hostname, 10) | ipv4 | first }}:9100" +ARGS="--web.listen-address={{ adm_ipv4 }}:9100" # Prometheus-node-exporter supports the following options: # diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index e3bfc5bc..65a5fe05 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -46,6 +46,14 @@ mode: 0644 when: prometheus.unifi_snmp_targets is defined +# We don't need to restart Prometheus when updating nodes +- name: Configure Prometheus NGINX targets + copy: + content: "{{ [{'targets': prometheus.nginx_targets}] | to_nice_json }}" + dest: /etc/prometheus/targets_nginx.json + mode: 0644 + when: prometheus.nginx_targets is defined + # We don't need to restart Prometheus when updating nodes - name: Configure Prometheus Apache targets copy: diff --git a/roles/prometheus/templates/prometheus/prometheus.yml.j2 b/roles/prometheus/templates/prometheus/prometheus.yml.j2 index e7e9a44e..d99e0742 100644 --- a/roles/prometheus/templates/prometheus/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus/prometheus.yml.j2 @@ -78,6 +78,20 @@ scrape_configs: replacement: 127.0.0.1:9116 {% endif %} +{% if prometheus.nginx_targets is defined %} + - job_name: nginx + file_sd_configs: + - files: + - '/etc/prometheus/targets_nginx.json' + relabel_configs: + # Do not put :9117 in instance name, rather here + - source_labels: [__address__] + target_label: instance + - source_labels: [instance] + target_label: __address__ + replacement: '$1:9117' +{% endif %} + {% if prometheus.apache_targets is defined %} - job_name: apache file_sd_configs: