Monitor NGINX with Prometheus

plays/monitoring.yml

@@ -16,6 +16,9 @@
         - https://grafana.crans.org
         - https://wiki.crans.org
         - https://pad.crans.org
+      nginx_targets:
+        - hodaur.adm.crans.org
+        - charybde.adm.crans.org
       apache_targets: []  # [zamok.adm.crans.org]
 
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
@@ -63,9 +66,15 @@
 # Monitor all hosts
 - hosts: server,test_vm
   vars:
-    adm_ipv4: "{{ ansible_all_ipv4_addresses | ipaddr(adm_subnet) | first }}"
+    adm_ipv4: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ipv4 | first }}"
   roles: ["prometheus-node-exporter"]
 
+# Export nginx metrics
+- hosts: charybde.adm.crans.org,hodaur.adm.crans.org
+  vars:
+    adm_ipv4: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ipv4 | first }}"
+  roles: ["prometheus-nginx-exporter"]
+
 # Export apache metrics
 #- hosts: zamok.adm.crans.org
 #  vars:

roles/prometheus-nginx-exporter/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Restart prometheus-nginx-exporter
+  service:
+    name: prometheus-nginx-exporter
+    state: restarted
+
+- name: Restart nginx
+  service:
+    name: nginx
+    state: restarted

roles/prometheus-nginx-exporter/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Install prometheus-nginx-exporter
+  apt:
+    update_cache: true
+    name: prometheus-nginx-exporter
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
+
+- name: Make prometheus-nginx-exporter listen on adm only
+  lineinfile:
+    path: /etc/default/prometheus-nginx-exporter
+    regexp: '^ARGS='
+    line: |
+      ARGS="-web.listen-address={{ adm_ipv4 }}:9117"
+  notify: Restart prometheus-nginx-exporter
+
+- name: Configure nginx
+  template:
+    src: nginx/status.j2
+    dest: /etc/nginx/sites-available/status
+  notify: Restart nginx
+
+- name: Activate nginx site
+  file:
+    src: /etc/nginx/sites-available/status
+    dest: /etc/nginx/sites-enabled/status
+    state: link
+  notify: Restart nginx

roles/prometheus-nginx-exporter/templates/nginx/status.j2

@@ -0,0 +1,8 @@
+{{ ansible_header | comment }}
+
+server {
+    listen 127.0.0.1:8080;
+    location = /stub_status {
+        stub_status;
+    }
+}

roles/prometheus-node-exporter/templates/default/prometheus-node-exporter.j2

@@ -4,7 +4,7 @@
 # Due to shell scaping, to pass backslashes for regexes, you need to double
 # them (\\d for \d). If running under systemd, you need to double them again
 # (\\\\d to mean \d), and escape newlines too.
-ARGS="--web.listen-address={{ query('ldap', 'ip', ansible_hostname, 10) | ipv4 | first }}:9100"
+ARGS="--web.listen-address={{ adm_ipv4 }}:9100"
 
 # Prometheus-node-exporter supports the following options:
 #

roles/prometheus/tasks/main.yml

@@ -46,6 +46,14 @@
     mode: 0644
   when: prometheus.unifi_snmp_targets is defined
 
+# We don't need to restart Prometheus when updating nodes
+- name: Configure Prometheus NGINX targets
+  copy:
+    content: "{{ [{'targets': prometheus.nginx_targets}] | to_nice_json }}"
+    dest: /etc/prometheus/targets_nginx.json
+    mode: 0644
+  when: prometheus.nginx_targets is defined
+
 # We don't need to restart Prometheus when updating nodes
 - name: Configure Prometheus Apache targets
   copy:

roles/prometheus/templates/prometheus/prometheus.yml.j2

@@ -78,6 +78,20 @@ scrape_configs:
         replacement: 127.0.0.1:9116
 {% endif %}
 
+{% if prometheus.nginx_targets is defined %}
+  - job_name: nginx
+    file_sd_configs:
+      - files:
+        - '/etc/prometheus/targets_nginx.json'
+    relabel_configs:
+      # Do not put :9117 in instance name, rather here
+      - source_labels: [__address__]
+        target_label: instance
+      - source_labels: [instance]
+        target_label: __address__
+        replacement: '$1:9117'
+{% endif %}
+
 {% if prometheus.apache_targets is defined %}
   - job_name: apache
     file_sd_configs: