nixos/hosts/vm/victoriametrics/rules/node.nix

{ critical, warning, ... }:
{
  # Memory

  alerts.NodeOutOfMemory = {
    expr = ''
      (
        (
          node_memory_MemAvailable_bytes
          + (
            clamp_min(node_zfs_arc_size - node_zfs_arc_c_min, 0)
            or node_memory_MemAvailable_bytes * 0
          )
        )
        / node_memory_MemTotal_bytes
      ) < 0.1
    '';
    for = "3m";
    labels = critical;
    annotations.Available = "{{ $value | humanizePercentage }}";
  };

  alerts.NodeUnderMemoryPressure = {
    expr = "rate(node_vmstat_pgmajfault[5m]) > 1000";
    labels = critical;
    for = "0m";
    annotations.Pressure = "{{ $value | humanize }}";
  };

  alerts.NodeSwapIsFillingUp = {
    expr = ''
      (1 - (node_memory_SwapFree_bytes
           / node_memory_SwapTotal_bytes)) > 0.75
    '';
    for = "1m";
    labels = critical;
    annotations.UsedSwap = "{{ $value | humanizePercentage }}";
  };

  alerts.NodeOomKillDetected = {
    expr = "increase(node_vmstat_oom_kill[1m]) > 0";
    for = "0m";
    labels = critical;
  };

  # CPU

  alerts.NodeCpuUsage = {
    expr = ''
      (avg by (instance)
        (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8
    '';
    for = "10m";
    labels = warning;
    annotations.AverageUsage = "{{ $value | humanizePercentage }}";
  };

  alerts.NodeCpuStealNoisyNeighbor = {
    expr = ''
      avg by (instance) (
        rate(node_cpu_seconds_total{mode="steal"}[2m])
      ) > 0.1
    '';
    for = "10m";
    labels = warning;
    annotations.Steal = "{{ $value | humanizePercentage }}";
  };

  # Network

  alerts.NodeLinkHighUsageIn = {
    expr = ''
      (rate(node_network_receive_bytes_total[5m])
       / on(instance, device) node_network_speed_bytes) > .80
    '';
    labels = warning;
    for = "3m";
    annotations = {
      Usage = "{{ $value | humanizePercentage }}";
      Device = "{{ $labels.device }}";
    };
  };

  alerts.NodeLinkHighUsageOut = {
    expr = ''
      (rate(node_network_transmit_bytes_total[5m])
       / on(instance, device) node_network_speed_bytes) > .80
    '';
    labels = warning;
    for = "3m";
    annotations = {
      Usage = "{{ $value | humanizePercentage }}";
      Device = "{{ $labels.device }}";
    };
  };

  alerts.NodeConntrackLimit = {
    expr = ''
      (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.8
    '';
    for = "5m";
    labels = warning;
    annotations.Filled = "{{ $value | humanizePercentage }}";
  };

  alerts.NodeNetworkReceiveErrors = {
    expr = ''
      rate(node_network_receive_errs_total[2m])
        / rate(node_network_receive_packets_total[2m]) > 0.01
    '';
    for = "2m";
    labels = warning;
    annotations = {
      Errors = "{{ $value | humanizePercentage }}";
      Device = "{{ $labels.device }}";
    };
  };

  alerts.NodeNetworkTransmitErrors = {
    expr = ''
      rate(node_network_transmit_errs_total[2m])
        / rate(node_network_transmit_packets_total[2m]) > 0.01
    '';
    for = "2m";
    labels = warning;
    annotations = {
      Errors = "{{ $value | humanizePercentage }}";
      Device = "{{ $labels.device }}";
    };
  };

  alerts.NodeNetworkBondDegraded = {
    expr = "node_bonding_active - node_bonding_slaves != 0";
    for = "2m";
    labels = warning;
    annotations.Device = "{{ $labels.device }}";
  };

  # Temperature

  alerts.NodePhysicalComponentTooHot = {
    expr = ''
      node_hwmon_temp_celsius > clamp_max(node_hwmon_temp_max_celsius, 79)
    '';
    for = "0m";
    labels = critical;
    annotations = {
      Temperature = "{{ $value | humanize }} °C";
      Chip = "{{ $labels.chip }}";
      Sensor = "{{ $labels.sensor }}";
    };
  };

  alerts.NodeNodeOvertemperatureAlarm = {
    expr = "node_hwmon_temp_crit_alarm_celsius == 1";
    for = "0m";
    labels = critical;
    annotations = {
      Chip = "{{ $labels.chip }}";
      Sensor = "{{ $labels.sensor }}";
    };
  };

  # Storage and disks

  alerts.NodeRaidArrayGotInactive = {
    expr = ''
      node_md_state{state="inactive"} > 0
    '';
    for = "0m";
    labels = critical;
    annotations = {
      Device = "{{ $labels.device }}";
    };
  };

  alerts.NodeRaidDiskFailure = {
    expr = ''
      node_md_disks{state="failed"} > 0
    '';
    for = "0m";
    labels = critical;
    annotations = {
      Device = "{{ $labels.md_device }}";
    };
  };

  alerts.NodeOutOfDiskSpace = {
    expr = ''
      (node_filesystem_free_bytes / node_filesystem_size_bytes < 0.1)
      and on (instance, device, mountpoint) (node_filesystem_readonly) == 0
    '';
    for = "5m";
    labels = critical;
    annotations = {
      Mountpoint = "{{ $labels.mountpoint }}";
      FreeSpace = "{{ $value | humanizePercentage }}";
    };
  };

  alerts.NodeOutOfInodes = {
    expr = "node_filesystem_files_free / node_filesystem_files < 0.1";
    for = "3m";
    labels = critical;
    annotations = {
      Mountpoint = "{{ $labels.mountpoint }}";
      FreeInodes = "{{ $value | humanizePercentage }}";
    };
  };

  alerts.NodeUnhealthyDisk = {
    expr = "smartmon_device_smart_healthy < 1";
    for = "10m";
    labels = critical;
    annotations.Disk = "{{ $labels.disk }}";
  };

  alerts.NodeZfsWrongState = {
    expr = ''
      node_zfs_zpool_state{state!="online"} > 0
    '';
    for = "5m";
    labels = critical;
    annotations = {
      State = "{{ $labels.state }}";
      ZPool = "{{ $labels.zpool }}";
    };
  };

  # Clock

  alerts.NodeClockSkew = {
    expr = ''
      (node_timex_offset_seconds > 0.05
        and deriv(node_timex_offset_seconds[5m]) >= 0)
      or (node_timex_offset_seconds < -0.05
        and deriv(node_timex_offset_seconds[5m]) <= 0)
    '';
    for = "2m";
    labels = warning;
  };

  alerts.NodeClockNotSynchronising = {
    expr = ''
      min_over_time(node_timex_sync_status[1m]) == 0
      and node_timex_maxerror_seconds >= 16
    '';
    for = "2m";
    labels = warning;
  };

  # EDAC

  alerts.NodeEdacCorrectableErrorsDetected = {
    expr = ''
      increase(node_edac_correctable_errors_total[1m]) > 0
    '';
    for = "0m";
    labels = warning;
    annotations.CorrectedErrors = "{{ $value }}";
  };

  alerts.NodeEdacUncorrectableErrorsDetected = {
    expr = ''
      increase(node_edac_uncorrectable_errors_total[1m]) > 0
    '';
    for = "0m";
    labels = critical;
    annotations.DetectedErrors = "{{ $value }}";
  };

  # Misc

  alerts.NodeLoad5Usage = {
    expr = ''
      node_load5 / (
        count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.5
    '';
    for = "1m";
    labels = warning;
    annotations.Load5PerCore = "{{ $value | humanize }}";
  };

  alerts.NodeSystemdUnitFailed = {
    expr = ''
      node_systemd_unit_state{state="failed"} == 1
    '';
    for = "5m";
    labels = warning;
    annotations.Unit = "{{ $labels.name }}";
  };

  alerts.NodeBorgmaticLastTooOld = {
    expr = ''
      (time()
      - (node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"}
      ) > 2 * 24 * 60 * 60)
    '';
    for = "0m";
    labels = warning;
    annotations.Last = "{{ $value | humanizeDuration }}";
  };

  alerts.NodeBorgmaticAbsent = {
    expr = ''
      up{job="node"} == 1 unless on(instance) node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"}
    '';
    for = "0s";
    labels = warning;
  };

  alerts.NodeRequiresReboot = {
    expr = "node_reboot_required > 0";
    for = "5m";
    labels = warning;
  };
}