ansible/roles/prometheus/templates/prometheus/alert.rules.yml.j2

{{ ansible_header | comment }}
{# As this is also using brackets it will conflict without a raw block #}
{% raw %}
# Synced with https://awesome-prometheus-alerts.grep.to/rules.html on 2021-06-07
# We remove descriptions as we only send summary on IRC.
# UPS, APT and RADIUS configuration is made by Crans.

groups:
- name: alert.rules
  rules:

  ##############################
  # Prometheus self-monitoring #
  ##############################

  - alert: PrometheusJobMissing
    expr: absent(up{job="prometheus"})
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Prometheus job missing (instance {{ $labels.instance }})

  - alert: PrometheusTargetMissing
    expr: up == 0
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: "{{ $labels.instance }} ({{ $labels.job }}) est manquant"

  - alert: PrometheusConfigurationReloadFailure
    expr: prometheus_config_last_reload_successful != 1
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Prometheus configuration reload failure (instance {{ $labels.instance }})

  - alert: PrometheusTooManyRestarts
    expr: changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Prometheus too many restarts (instance {{ $labels.instance }})

  - alert: PrometheusRuleEvaluationFailures
    expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Prometheus rule evaluation failures (instance {{ $labels.instance }})

  - alert: PrometheusTargetEmpty
    expr: prometheus_sd_discovered_targets == 0
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Prometheus target empty (instance {{ $labels.instance }})

  # This already happened in 2021 at Crans
  - alert: PrometheusTsdbCompactionsFailed
    expr: increase(prometheus_tsdb_compactions_failed_total[1m]) > 0
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Prometheus TSDB compactions failed (instance {{ $labels.instance }})

  #####################
  # Host and hardware #
  #####################

  # Alert for out of memory
  # Do not take into account memory not used by apps
  - alert: HostOutOfMemory
    expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes + node_memory_PageTables_bytes + node_memory_VmallocUsed_bytes + node_memory_SwapCached_bytes + node_memory_Slab_bytes) / node_memory_MemTotal_bytes * 100 < 10
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: La mémoire vive de {{ $labels.instance }} arrive à saturation ({{ $value }}%)

  - alert: HostUnusualDiskReadRate
    expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Host unusual disk read rate (instance {{ $labels.instance }})

  - alert: HostUnusualDiskWriteRate
    expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Host unusual disk write rate (instance {{ $labels.instance }})

  - alert: HostOutOfDiskSpace
    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{ $labels.mountpoint }} sur {{ $labels.instance }} arrive à saturation ({{ $value }}%)"

  - alert: HostDiskWillFillIn24Hours
    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})

  - alert: HostOutOfInodes
    expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Presque plus d'inodes disponibles ({{ $value }}% restant) dans {{ $labels.mountpoint }} sur {{ $labels.instance }}

  - alert: HostHighCpuLoad
    expr: node_load5 > 9
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: Charge sur {{ $labels.instance }} à {{ $value }}

  - alert: HostSystemdServiceCrashed
    expr: node_systemd_unit_state{state="failed"} == 1
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: "{{ $labels.name }} a crashé sur {{ $labels.instance }}"

  # 0B is so hot
  # En pratique c'est mauvais de tourner des disques trop chauds
  - alert: HostPhysicalComponentTooHot
    expr: node_hwmon_temp_celsius > 75
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Host physical component too hot (instance {{ $labels.instance }})

  - alert: HostNodeOvertemperatureAlarm
    expr: node_hwmon_temp_crit_alarm_celsius == 1
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Host node overtemperature alarm (instance {{ $labels.instance }})

  - alert: HostRaidDiskFailure
    expr: node_md_disks{state="failed"} > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Le RAID sur {{ $labels.instance }} a perdu {{ $value }} disque(s)

  - alert: HostOomKillDetected
    expr: increase(node_vmstat_oom_kill[1m]) > 0
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Host OOM kill detected (instance {{ $labels.instance }})

  - alert: HostEdacCorrectableErrorsDetected
    expr: increase(node_edac_correctable_errors_total[1m]) > 0
    for: 0m
    labels:
      severity: info
    annotations:
      summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})

  - alert: HostEdacUncorrectableErrorsDetected
    expr: node_edac_uncorrectable_errors_total > 0
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})

  # This happend in June 2021 at Crans
  - alert: HostConntrackLimit
    expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Host conntrack limit (instance {{ $labels.instance }})

  ############
  # Blackbox #
  ############

  - alert: BlackboxProbeFailed
    expr: probe_success == 0
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: Blackbox probe failed (instance {{ $labels.instance }})

  - alert: BlackboxSlowProbe
    expr: avg_over_time(probe_duration_seconds[1m]) > 1
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: Blackbox slow probe (instance {{ $labels.instance }})

  - alert: BlackboxSslCertificateWillExpireSoon
    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 20
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }})

  #######
  # UPS #
  #######

  # Check UPS
  - alert: UpsOutputSourceChanged
    expr: upsOutputSource != 3
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: La source d'alimentation de {{ $labels.instance }} a changé !

  - alert: UpsBatteryStatusChanged
    expr: upsBatteryStatus != 2
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: L'état de la batterie de {{ $labels.instance }} a changé !

  - alert: UpsTemperatureWarning
    expr: (xupsEnvRemoteTemp < 10) or (xupsEnvRemoteTemp > 26)
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: La température autour de {{ $labels.instance }} est de {{ $value }}°C

  - alert: UpsTemperatureCritical
    expr: (xupsEnvRemoteTemp < 0) or (xupsEnvRemoteTemp > 30)
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: La température autour de {{ $labels.instance }} est de {{ $value }}°C

  - alert: UpsHighHumidity
    expr: xupsEnvRemoteHumidity > 65
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: L'humidité autour de {{ $labels.instance }} est de {{ $value }}%

  - alert: UpsVeryHighHumidity
    expr: xupsEnvRemoteHumidity > 85
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: L'humidité autour de {{ $labels.instance }} est de {{ $value }}%

  - alert: UpsHighLoad
    expr: upsOutputPercentLoad > 70
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: La charge de {{ $labels.instance }} est de {{ $value }}%

  - alert: UpsWrongInputVoltage
    expr: (upsInputVoltage < 210) or (upsInputVoltage > 250)
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: La tension d'entrée de {{ $labels.instance }} est de {{ $value }}V

  - alert: UpsWrongOutputVoltage
    expr: (upsOutputVoltage < 215) or (upsOutputVoltage > 245)
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: La tension de sortie de {{ $labels.instance }} est de {{ $value }}V

  #########
  # Other #
  #########

  - alert: AptAutoremovePending
    expr: apt_autoremove_pending > 0
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "{{ $value }} paquet(s) APT sont inutile(s) sur {{ $labels.instance }}"

  - alert: AptOrphans
    expr: apt_orphans > 10
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "{{ $value }} paquet(s) APT sont orphelins sur {{ $labels.instance }}"

  - alert: MailqNotEmpty
    expr: postfix_mailq_length > 25
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "{{ $value }} mails dans la mailq sur {{ $labels.instance }}"

  - alert: NoRadiusLogin
    expr: rate(radiusd_access_ok[3m]) == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: Personne ne vient taper le RADIUS

  - alert: TooManyReallocatedSectors
    expr: smartmon_reallocated_sector_ct_raw_value > 1e3
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "{{ $labels.disk }} sur {{ $labels.instance }} a {{ $value }} secteurs réalloués"

  - alert: TooManyUDPErrors
    expr: irate(node_netstat_Udp_InErrors[5m]) > 100
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "{{ $labels.instance }} a plus de {{ $value }} connexions UDP en erreur. Quelque chose spam!"

{% endraw %}