{ critical, warning, ... }: { # Memory alerts.NodeOutOfMemory = { expr = '' ( ( node_memory_MemAvailable_bytes + ( clamp_min(node_zfs_arc_size - node_zfs_arc_c_min, 0) or node_memory_MemAvailable_bytes * 0 ) ) / node_memory_MemTotal_bytes ) < 0.1 ''; for = "3m"; labels = critical; annotations.Available = "{{ $value | humanizePercentage }}"; }; alerts.NodeUnderMemoryPressure = { expr = "rate(node_vmstat_pgmajfault[5m]) > 1000"; labels = critical; for = "0m"; annotations.Pressure = "{{ $value | humanize }}"; }; alerts.NodeSwapIsFillingUp = { expr = '' (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.75 ''; for = "1m"; labels = critical; annotations.UsedSwap = "{{ $value | humanizePercentage }}"; }; alerts.NodeOomKillDetected = { expr = "increase(node_vmstat_oom_kill[1m]) > 0"; for = "0m"; labels = critical; }; # CPU alerts.NodeCpuUsage = { expr = '' (avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8 ''; for = "10m"; labels = warning; annotations.AverageUsage = "{{ $value | humanizePercentage }}"; }; alerts.NodeCpuStealNoisyNeighbor = { expr = '' avg by (instance) ( rate(node_cpu_seconds_total{mode="steal"}[2m]) ) > 0.1 ''; for = "10m"; labels = warning; annotations.Steal = "{{ $value | humanizePercentage }}"; }; # Network alerts.NodeLinkHighUsageIn = { expr = '' (rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80 ''; labels = warning; for = "3m"; annotations = { Usage = "{{ $value | humanizePercentage }}"; Device = "{{ $labels.device }}"; }; }; alerts.NodeLinkHighUsageOut = { expr = '' (rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80 ''; labels = warning; for = "3m"; annotations = { Usage = "{{ $value | humanizePercentage }}"; Device = "{{ $labels.device }}"; }; }; alerts.NodeConntrackLimit = { expr = '' (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.8 ''; for = "5m"; labels = warning; annotations.Filled = "{{ $value | humanizePercentage }}"; }; alerts.NodeNetworkReceiveErrors = { expr = '' rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01 ''; for = "2m"; labels = warning; annotations = { Errors = "{{ $value | humanizePercentage }}"; Device = "{{ $labels.device }}"; }; }; alerts.NodeNetworkTransmitErrors = { expr = '' rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01 ''; for = "2m"; labels = warning; annotations = { Errors = "{{ $value | humanizePercentage }}"; Device = "{{ $labels.device }}"; }; }; alerts.NodeNetworkBondDegraded = { expr = "node_bonding_active - node_bonding_slaves != 0"; for = "2m"; labels = warning; annotations.Device = "{{ $labels.device }}"; }; # Temperature alerts.NodePhysicalComponentTooHot = { expr = '' node_hwmon_temp_celsius > clamp_max(node_hwmon_temp_max_celsius, 79) ''; for = "0m"; labels = critical; annotations = { Temperature = "{{ $value | humanize }} °C"; Chip = "{{ $labels.chip }}"; Sensor = "{{ $labels.sensor }}"; }; }; alerts.NodeNodeOvertemperatureAlarm = { expr = "node_hwmon_temp_crit_alarm_celsius == 1"; for = "0m"; labels = critical; annotations = { Chip = "{{ $labels.chip }}"; Sensor = "{{ $labels.sensor }}"; }; }; # Storage and disks alerts.NodeRaidArrayGotInactive = { expr = '' node_md_state{state="inactive"} > 0 ''; for = "0m"; labels = critical; annotations = { Device = "{{ $labels.device }}"; }; }; alerts.NodeRaidDiskFailure = { expr = '' node_md_disks{state="failed"} > 0 ''; for = "0m"; labels = critical; annotations = { Device = "{{ $labels.md_device }}"; }; }; alerts.NodeOutOfDiskSpace = { expr = '' (node_filesystem_free_bytes / node_filesystem_size_bytes < 0.1) and on (instance, device, mountpoint) (node_filesystem_readonly) == 0 ''; for = "5m"; labels = critical; annotations = { Mountpoint = "{{ $labels.mountpoint }}"; FreeSpace = "{{ $value | humanizePercentage }}"; }; }; alerts.NodeOutOfInodes = { expr = "node_filesystem_files_free / node_filesystem_files < 0.1"; for = "3m"; labels = critical; annotations = { Mountpoint = "{{ $labels.mountpoint }}"; FreeInodes = "{{ $value | humanizePercentage }}"; }; }; alerts.NodeUnhealthyDisk = { expr = "smartmon_device_smart_healthy < 1"; for = "10m"; labels = critical; annotations.Disk = "{{ $labels.disk }}"; }; alerts.NodeZfsWrongState = { expr = '' node_zfs_zpool_state{state!="online"} > 0 ''; for = "5m"; labels = critical; annotations = { State = "{{ $labels.state }}"; ZPool = "{{ $labels.zpool }}"; }; }; # Clock alerts.NodeClockSkew = { expr = '' (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0) ''; for = "2m"; labels = warning; }; alerts.NodeClockNotSynchronising = { expr = '' min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16 ''; for = "2m"; labels = warning; }; # EDAC alerts.NodeEdacCorrectableErrorsDetected = { expr = '' increase(node_edac_correctable_errors_total[1m]) > 0 ''; for = "0m"; labels = warning; annotations.CorrectedErrors = "{{ $value }}"; }; alerts.NodeEdacUncorrectableErrorsDetected = { expr = '' increase(node_edac_uncorrectable_errors_total[1m]) > 0 ''; for = "0m"; labels = critical; annotations.DetectedErrors = "{{ $value }}"; }; # Misc alerts.NodeLoad5Usage = { expr = '' node_load5 / ( count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.5 ''; for = "1m"; labels = warning; annotations.Load5PerCore = "{{ $value | humanize }}"; }; alerts.NodeSystemdUnitFailed = { expr = '' node_systemd_unit_state{state="failed"} == 1 ''; for = "5m"; labels = warning; annotations.Unit = "{{ $labels.name }}"; }; alerts.NodeBorgmaticLastTooOld = { expr = '' (time() - (node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"} ) > 2 * 24 * 60 * 60) ''; for = "0m"; labels = warning; annotations.Last = "{{ $value | humanizeDuration }}"; }; alerts.NodeBorgmaticAbsent = { expr = '' up{job="node"} == 1 unless on(instance) node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"} ''; for = "0s"; labels = warning; }; alerts.NodeRequiresReboot = { expr = "node_reboot_required > 0"; for = "5m"; labels = warning; }; }