Alert ninjabot

roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2

@@ -28,14 +28,15 @@ route:
 
   # When the first notification was sent, wait 'group_interval' to send a batch
   # of new alerts that started firing for that group.
-  group_interval: 5m
+  group_interval: 1m
 
   # If an alert has successfully been sent, wait 'repeat_interval' to
   # resend them.
   repeat_interval: 3h 
 
   # A default receiver
-  receiver: team-roots-mails
+  #receiver: team-roots-mails
+  receiver: webhook-ninjabot
 
 
 # Inhibition rules allow to mute a set of alerts given that another alert is
@@ -55,4 +56,7 @@ receivers:
 - name: 'team-roots-mails'
   email_configs:
   - to: 'roots@crans.org'
-
+- name: 'webhook-ninjabot'
+  webhook_configs:
+  - url: 'http://zamok.adm.crans.org:5000/'
+    send_resolved: true

roles/prometheus/templates/prometheus/alert.rules.yml.j2

@@ -59,6 +59,8 @@ groups:
   - alert: APIHighRequestLatency
     expr: api_http_request_latencies_second{quantile="0.5"} > 1
     for: 10m
+    labels:
+      severity: warning
     annotations:
       summary: "Latence élevée sur {{ $labels.instance }}"
       description: "{{ $labels.instance }} a un temps de réponse médian au dessus d'une seconde (actuellement : {{ $value }}s)."
@@ -77,7 +79,7 @@ groups:
     expr: node_ntp_drift_seconds > 0.01
     for: 1m
     labels:
-      severity: moderate
+      severity: warning
     annotations:
       description: "Décalage NTP élevé sur {{ $labels.instance }}"
       summary: "Le décalage NTP est élevé ({{ $value }} > 0.01)"