[prometheus] Monitoring rules, Yay!

roles/prometheus/templates/prometheus/alert.rules.yml.j2

@@ -2,24 +2,83 @@
 {# As this is also Jinja2 it will conflict without a raw block #}
 {% raw %}
 groups:
-- name: example
+- name: alert.rules
   rules:
 
-  # Alert for any instance that is unreachable for >5 minutes.
+  # Alert for any instance that is unreachable for >3 minutes.
   - alert: InstanceDown
     expr: up == 0
-    for: 5m
+    for: 3m
     labels:
-      severity: page
+      severity: critical
     annotations:
-      summary: "Instance {{ $labels.instance }} down"
-      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+      summary: "Le serveur {{ $labels.instance }} est down"
+      description: "Le serveur {{ $labels.instance }} est down depuis plus de 3 minutes."
+
+  # Alert for out of memory
+  - alert: OutOfMemory
+    expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10
+    for: 30m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Presque plus de mémoire vive sur {{ $labels.instance }}"
+      description: "La mémoire est en train de se remplir ({{ $value }}% restant)"
+
+  # Alert for out of disk space
+  - alert: OutOfDiskSpace
+    expr: node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10
+    for: 30m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Presque plus de stockage sur {{ $labels.instance }}"
+      description: "Le disque est presque rempli ({{ $value }}% restant)"
+
+  # Alert for out of inode space on disk
+  - alert: OutOfInodes
+    expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10
+    for: 30m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Presque plus d'inodes disque sur {{ $labels.instance }}"
+      description: "Le disque est presque à courre d'inodes disponibles ({{ $value }}% restant)"
+
+  # Alert for high CPU usage
+  - alert: CpuLoad
+    expr: node_load15 / (count without (cpu, mode) (node_cpu_seconds_total{mode="system"})) > 2
+    for: 30m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Charge CPU élevée sur {{ $labels.instance }}"
+      description: "La charge CPU (moyenne de 15mn) est élevée."
 
   # Alert for any instance that has a median request latency >1s.
   - alert: APIHighRequestLatency
     expr: api_http_request_latencies_second{quantile="0.5"} > 1
     for: 10m
     annotations:
-      summary: "High request latency on {{ $labels.instance }}"
-      description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)"
+      summary: "Latence élevée sur {{ $labels.instance }}"
+      description: "{{ $labels.instance }} a un temps de réponse médian au dessus d'une seconde (actuellement : {{ $value }}s)."
+
+  # NTP
+  - alert: ntp_drifting
+    expr: node_ntp_drift_seconds > 0.05
+    for: 3m
+    labels:
+      severity: critical
+    annotations:
+      description: "Décalage NTP trop élevé sur {{ $labels.instance }}"
+      summary: "Le décalage NTP est trop élevé ({{ $value }} > 0.05)"
+
+  - alert: ntp_drifting
+    expr: node_ntp_drift_seconds > 0.01
+    for: 1m
+    labels:
+      severity: moderate
+    annotations:
+      description: "Décalage NTP élevé sur {{ $labels.instance }}"
+      summary: "Le décalage NTP est élevé ({{ $value }} > 0.01)"
 {% endraw %}