Merge branch 'victoriametrics' into 'main'

Draft: mise en place de la config de victoriametrics

See merge request nounous/nixos!68

flake.nix

@@ -99,6 +99,12 @@
               specialArgs = inputs;
               modules = [ ./hosts/vm/vaultwarden ] ++ baseModules;
             };
+
+            victoriametrics = nixosSystem {
+              specialArgs = inputs;
+              modules = [ ./hosts/vm/victoriametrics ] ++ baseModules;
+            };
+
             cransIso = nixosSystem {
               system = "x86_64-linux";
               specialArgs = inputs;

hosts/vm/victoriametrics/default.nix

@@ -0,0 +1,28 @@
+
+{ ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./victoriametrics.nix
+    ];
+
+  networking.hostName = "victoriametrics";
+  boot.loader.grub.devices = [ "/dev/sda" ];
+
+  crans = {
+    enable = true;
+
+    networking = {
+      id = 153;
+      srvNat = {
+        enable = true;
+        interface = "ens19";
+      };
+    };
+
+    resticClient.enable = false;
+  };
+
+  system.stateVersion = "26.05";
+}

hosts/vm/victoriametrics/hardware-configuration.nix

@@ -0,0 +1,30 @@
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/1be0ad06-95af-4630-9bd5-77fc34dd0c60";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens19.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens20.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/vm/victoriametrics/rules/bird.nix

@@ -0,0 +1,163 @@
+{ critical, warning, ... }:
+
+{
+  records."bird:protocol_up:bgp_all".expr = ''
+    label_replace(
+      bird_protocol_up{proto="BGP"},
+      "group", "$1",
+      "instance", "^([^0-9\\.]+)-[0-9]+.*"
+    )
+  '';
+
+  # FIXME: sessions en cours d'installation, pas encore monitorées
+  records."bird:protocol_up:bgp".expr = ''
+    bird:protocol_up:bgp_all
+    unless
+    bird:protocol_up:bgp_all{
+      group="edge",
+      name=~"^(viarezo|isp[12]|rezel)[46]$"
+    }
+  '';
+
+  # Sessions qui ne sont volontairement pas redondées
+  # au sein d'un groupe
+  records."bird:protocol_up:bgp:non_redundant".expr = ''
+    bird:protocol_up:bgp{
+      group="edge",
+      name=~"^(oti|crans|legacy|edge)[46]$"
+    }
+  '';
+
+  # Sessions qui le sont
+  records."bird:protocol_up:bgp:redundant".expr = ''
+    bird:protocol_up:bgp
+    unless
+    bird:protocol_up:bgp:non_redundant
+  '';
+
+  alerts.BirdBGPRedundancyDegraded = {
+    expr = ''
+      (
+        count by (group, name) (
+          bird:protocol_up:bgp:redundant{state="Established"}
+        ) + (
+          count by (group, name) (
+            bird:protocol_up:bgp:redundant{state!="Established"} * 0
+          )
+        )
+      ) < 2
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Session = "{{ $labels.name }}";
+      Count = "{{ $value }}";
+      Group = "{{ $labels.group }}";
+    };
+  };
+
+  alerts.BirdBGPDown = {
+    expr = ''
+      (
+        count by (group, name) (
+          bird:protocol_up:bgp{state="Established"}
+        ) + (
+          count by (group, name) (
+            bird:protocol_up:bgp{state!="Established"} * 0
+          )
+        )
+      ) == 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Session = "{{ $labels.name }}";
+      Group = "{{ $labels.group }}";
+    };
+  };
+
+  # TODO: warning pour redondant ?
+  alerts.BirdBGPNoExportedPrefixRedundant = {
+    expr = ''
+      bird_protocol_prefix_export_count{
+        export_filter!="REJECT",
+      } * on (instance, name) group_left (group) (
+        bird:protocol_up:bgp:redundant{state="Established"}
+      ) == 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Session = "{{ $labels.name }}";
+      Group = "{{ $labels.group }}";
+    };
+  };
+
+  alerts.BirdBGPNoImportedPrefixRedundant = {
+    expr = ''
+      bird_protocol_prefix_import_count{
+        import_filter!="REJECT",
+      } * on (instance, name) group_left (group) (
+        bird:protocol_up:bgp:redundant{state="Established"}
+      ) == 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Session = "{{ $labels.name }}";
+      Group = "{{ $labels.group }}";
+    };
+  };
+
+  alerts.BirdBGPNoExportedPrefixNonRedundant = {
+    expr = ''
+      sum by (group) (
+        bird_protocol_prefix_export_count{
+          export_filter!="REJECT",
+        } * on (instance, name) group_left (group) (
+          bird:protocol_up:bgp:non_redundant{state="Established"}
+        )
+      ) == 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Session = "{{ $labels.name }}";
+      Group = "{{ $labels.group }}";
+    };
+  };
+
+  alerts.BirdBGPNoImportedPrefixNonRedundant = {
+    expr = ''
+      sum by (group) (
+        bird_protocol_prefix_import_count{
+          import_filter!="REJECT",
+        } * on (instance, name) group_left (group) (
+          bird:protocol_up:bgp:non_redundant{state="Established"}
+        )
+      ) == 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Session = "{{ $labels.name }}";
+      Group = "{{ $labels.group }}";
+    };
+  };
+
+  alerts.BirdOSPFNeighboursChange = {
+    expr = ''
+      changes(bird_ospf_neighbor_count[5m]) > 0
+      or changes(bird_ospfv3_neighbor_count[5m]) > 0
+    '';
+    for = "0m";
+    labels = warning;
+  };
+
+  alerts.BirdOSPFDown = {
+    expr = "bird_ospf_running == 0";
+    for = "0m";
+    labels = critical;
+    annotations.Instance = "{{ $labels.name }}";
+  };
+}

hosts/vm/victoriametrics/rules/ceph.nix

@@ -0,0 +1,149 @@
+{ critical, warning, ... }:
+
+{
+  alerts.CephErrorState = {
+    expr = ''
+      ceph_health_status > 1
+    '';
+    for = "5m";
+    labels = critical;
+  };
+
+  alerts.CephMonitorAvailableStorageCritical = {
+    expr = ''
+      ceph_monitor_avail_percent < 15
+    '';
+    for = "1m";
+    labels = critical;
+  };
+
+  alerts.CephMonitorAvailableStorageWarning = {
+    expr = ''
+      ceph_monitor_avail_percent < 30
+    '';
+    for = "1m";
+    labels = warning;
+  };
+
+  alerts.CephMonitorClockSkewTooHigh = {
+    expr = ''
+      abs(ceph_monitor_clock_skew_seconds) > 0.1
+    '';
+    for = "1m";
+    labels = warning;
+  };  
+
+  alerts.CephOSDApplyLatencyTooHigh = {
+    expr = ''
+      ceph_osd_perf_apply_latency_seconds > 10
+    '';
+    for = "1m";
+    labels = warning;
+    annotations = {
+      Latency = "{{ $value }}";
+      Device = "{{ $labels.device }}";
+    };
+  };
+
+  alerts.CephOSDDown = {
+    expr = ''
+      ceph_osd_up == 0
+    '';
+    for = "1m";
+    labels = critical;
+  };
+
+  alerts.CephOSDReweighted = {
+    expr = ''
+      ceph_osd_weight < 1
+    '';
+    for = "1h";
+    labels = warning;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephOSDUtilization = {
+    expr = ''
+      ceph_osd_utilization > 90
+    '';
+    for = "1m";
+    labels = critical;
+  };
+
+  alerts.CephPgActivating = {
+    expr = ''
+      ceph_pg_activating > 0
+    '';
+    for = "5m";
+    labels = critical;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephPgDown = {
+    expr = ''
+      ceph_pg_down > 0
+    '';
+    for = "3m";
+    labels = critical;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephPgIncomplete = {
+    expr = ''
+      ceph_pg_incomplete > 0
+    '';
+    for = "2m";
+    labels = critical;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephPgInconsistent = {
+    expr = ''
+      ceph_pg_inconsistent > 0
+    '';
+    for = "1m";
+    labels = warning;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephPgBackfillTooFull = {
+    expr = ''
+      ceph_pg_backfill_toofull > 0
+    '';
+    for = "5m";
+    labels = warning;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephPgUnavailable = {
+    expr = ''
+      ceph_pg_total - ceph_pg_active > 0
+    '';
+    for = "5m";
+    labels = critical;
+    annotations.Instance = "{{ $labels.cluster }}";
+    annotations.Database = "{{ $value }}";
+  };
+
+  alerts.CephTargetDown = {
+    expr = ''
+      up{job="ceph"} == 0
+    '';
+    for = "10m";
+    labels = critical;
+  };
+
+  alerts.CephWarnState = {
+    expr = ''
+      ceph_health_status == 1
+    '';
+    for = "30m";
+    labels = warning;
+  };
+}

hosts/vm/victoriametrics/rules/default.nix

@@ -0,0 +1,13 @@
+{ ... }:
+
+{
+  bird = ./bird.nix;
+  ceph = ./ceph.nix;
+  ilo = ./ilo.nix;
+  jitsi = ./jitsi.nix;
+  node = ./node.nix;
+  postgresql = ./postgresql.nix;
+  switch = ./switch.nix;
+  unbound = ./unbound.nix;
+  ups = ./ups.nix;
+}

hosts/vm/victoriametrics/rules/ilo.nix

@@ -0,0 +1,140 @@
+{ critical, warning, ... }:
+
+{
+  # cpqHeFltTolFanTable
+
+  alerts.IloFanDegraded = {
+    expr = ''
+      cpqHeFltTolFanCondition{
+        cpqHeFltTolFanCondition="degraded"
+      } == 1
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Chassis = "{{ $labels.cpqHeFltTolFanChassis }}";
+      Index = "{{ $labels.cpqHeFltTolFanIndex }}";
+    };
+  };
+
+  alerts.IloFanFailed = {
+    expr = ''
+      cpqHeFltTolFanCondition{
+        cpqHeFltTolFanCondition="failed"
+      } == 1
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Chassis = "{{ $labels.cpqHeFltTolFanChassis }}";
+      Index = "{{ $labels.cpqHeFltTolFanIndex }}";
+    };
+  };
+
+  # cpqHeFltTolPowerSupplyTable
+
+  alerts.IloHighPSUUsage = {
+    expr = ''
+      cpqHeFltTolPowerSupplyCapacityUsed
+      / (cpqHeFltTolPowerSupplyCapacityMaximum > 0) > 0.5
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Usage = "{{ $value | humanizePercentage }}";
+      Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}";
+      Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}";
+    };
+  };
+
+  alerts.IloPSUDegraded = {
+    expr = ''
+      cpqHeFltTolPowerSupplyCondition{
+        cpqHeFltTolPowerSupplyCondition="degraded"
+      } == 1
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}";
+      Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}";
+    };
+  };
+
+  alerts.IloPSUFailed = {
+    expr = ''
+      (
+        cpqHeFltTolPowerSupplyCondition{
+          cpqHeFltTolPowerSupplyCondition="failed"
+        } == 1
+      ) * on (
+        instance,
+        cpqHeFltTolPowerSupplyChassis,
+        cpqHeFltTolPowerSupplyBay
+      ) group_left (
+        cpqHeFltTolPowerSupplyErrorCondition
+      )
+      (cpqHeFltTolPowerSupplyErrorCondition == 1)
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}";
+      Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}";
+      Error = "{{ $labels.cpqHeFltTolPowerSupplyErrorCondition }}";
+    };
+  };
+
+  # cpqHeTemperatureTable
+
+  alerts.IloTooHot = {
+    expr = ''
+      (
+        cpqHeTemperatureCondition{
+          cpqHeTemperatureCondition!="ok"
+        } == 1
+      ) * on (
+        instance,
+        cpqHeTemperatureChassis,
+        cpqHeTemperatureIndex
+      ) group_left (cpqHeTemperatureLocale)
+      cpqHeTemperatureLocale_info
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Chassis = "{{ $labels.cpqHeTemperatureChassis }}";
+      Index = "{{ $labels.cpqHeTemperatureIndex }}";
+      Locale = "{{ $labels.cpqHeTemperatureLocale }}";
+      Condition = "{{ $labels.cpqHeTemperatureCondition }}";
+    };
+  };
+
+  # cpqHeResMem2ModuleTable
+
+  alerts.IloRamHS = {
+    expr = ''
+      cpqHeResMem2ModuleStatus{
+        cpqHeResMem2ModuleStatus!~"good|notPresent"
+      } == 1
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Module = "{{ $labels.cpqHeResMem2Module }}";
+      Status = "{{ $labels.cpqHeResMem2ModuleStatus }}";
+    };
+  };
+
+  alerts.IloUneSeuleRam = {
+    expr = ''
+      count by (instance) (
+        cpqHeResMem2ModuleStatus{
+          cpqHeResMem2ModuleStatus="good"
+        } == 1
+      ) <= 1
+    '';
+    for = "0m";
+    labels = critical;
+  };
+}

hosts/vm/victoriametrics/rules/jitsi.nix

@@ -0,0 +1,9 @@
+{ critical, ... }:
+
+{
+  alerts.JitsiHighStressLevel = {
+    expr = "jitsi_stress_level > 0.7";
+    for = "3m";
+    labels = critical;
+  };
+}

hosts/vm/victoriametrics/rules/node.nix

@@ -0,0 +1,318 @@
+{ critical, warning, ... }:
+{
+  # Memory
+
+  alerts.NodeOutOfMemory = {
+    expr = ''
+      (
+        (
+          node_memory_MemAvailable_bytes
+          + (
+            clamp_min(node_zfs_arc_size - node_zfs_arc_c_min, 0)
+            or node_memory_MemAvailable_bytes * 0
+          )
+        )
+        / node_memory_MemTotal_bytes
+      ) < 0.1
+    '';
+    for = "3m";
+    labels = critical;
+    annotations.Available = "{{ $value | humanizePercentage }}";
+  };
+
+  alerts.NodeUnderMemoryPressure = {
+    expr = "rate(node_vmstat_pgmajfault[5m]) > 1000";
+    labels = critical;
+    for = "0m";
+    annotations.Pressure = "{{ $value | humanize }}";
+  };
+
+  alerts.NodeSwapIsFillingUp = {
+    expr = ''
+      (1 - (node_memory_SwapFree_bytes
+           / node_memory_SwapTotal_bytes)) > 0.75
+    '';
+    for = "1m";
+    labels = critical;
+    annotations.UsedSwap = "{{ $value | humanizePercentage }}";
+  };
+
+  alerts.NodeOomKillDetected = {
+    expr = "increase(node_vmstat_oom_kill[1m]) > 0";
+    for = "0m";
+    labels = critical;
+  };
+
+  # CPU
+
+  alerts.NodeCpuUsage = {
+    expr = ''
+      (avg by (instance)
+        (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8
+    '';
+    for = "10m";
+    labels = warning;
+    annotations.AverageUsage = "{{ $value | humanizePercentage }}";
+  };
+
+  alerts.NodeCpuStealNoisyNeighbor = {
+    expr = ''
+      avg by (instance) (
+        rate(node_cpu_seconds_total{mode="steal"}[2m])
+      ) > 0.1
+    '';
+    for = "10m";
+    labels = warning;
+    annotations.Steal = "{{ $value | humanizePercentage }}";
+  };
+
+  # Network
+
+  alerts.NodeLinkHighUsageIn = {
+    expr = ''
+      (rate(node_network_receive_bytes_total[5m])
+       / on(instance, device) node_network_speed_bytes) > .80
+    '';
+    labels = warning;
+    for = "3m";
+    annotations = {
+      Usage = "{{ $value | humanizePercentage }}";
+      Device = "{{ $labels.device }}";
+    };
+  };
+
+  alerts.NodeLinkHighUsageOut = {
+    expr = ''
+      (rate(node_network_transmit_bytes_total[5m])
+       / on(instance, device) node_network_speed_bytes) > .80
+    '';
+    labels = warning;
+    for = "3m";
+    annotations = {
+      Usage = "{{ $value | humanizePercentage }}";
+      Device = "{{ $labels.device }}";
+    };
+  };
+
+  alerts.NodeConntrackLimit = {
+    expr = ''
+      (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.8
+    '';
+    for = "5m";
+    labels = warning;
+    annotations.Filled = "{{ $value | humanizePercentage }}";
+  };
+
+  alerts.NodeNetworkReceiveErrors = {
+    expr = ''
+      rate(node_network_receive_errs_total[2m])
+        / rate(node_network_receive_packets_total[2m]) > 0.01
+    '';
+    for = "2m";
+    labels = warning;
+    annotations = {
+      Errors = "{{ $value | humanizePercentage }}";
+      Device = "{{ $labels.device }}";
+    };
+  };
+
+  alerts.NodeNetworkTransmitErrors = {
+    expr = ''
+      rate(node_network_transmit_errs_total[2m])
+        / rate(node_network_transmit_packets_total[2m]) > 0.01
+    '';
+    for = "2m";
+    labels = warning;
+    annotations = {
+      Errors = "{{ $value | humanizePercentage }}";
+      Device = "{{ $labels.device }}";
+    };
+  };
+
+  alerts.NodeNetworkBondDegraded = {
+    expr = "node_bonding_active - node_bonding_slaves != 0";
+    for = "2m";
+    labels = warning;
+    annotations.Device = "{{ $labels.device }}";
+  };
+
+  # Temperature
+
+  alerts.NodePhysicalComponentTooHot = {
+    expr = ''
+      node_hwmon_temp_celsius > clamp_max(node_hwmon_temp_max_celsius, 79)
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Temperature = "{{ $value | humanize }} °C";
+      Chip = "{{ $labels.chip }}";
+      Sensor = "{{ $labels.sensor }}";
+    };
+  };
+
+  alerts.NodeNodeOvertemperatureAlarm = {
+    expr = "node_hwmon_temp_crit_alarm_celsius == 1";
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Chip = "{{ $labels.chip }}";
+      Sensor = "{{ $labels.sensor }}";
+    };
+  };
+
+  # Storage and disks
+
+  alerts.NodeRaidArrayGotInactive = {
+    expr = ''
+      node_md_state{state="inactive"} > 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Device = "{{ $labels.device }}";
+    };
+  };
+
+  alerts.NodeRaidDiskFailure = {
+    expr = ''
+      node_md_disks{state="failed"} > 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Device = "{{ $labels.md_device }}";
+    };
+  };
+
+  alerts.NodeOutOfDiskSpace = {
+    expr = ''
+      (node_filesystem_free_bytes / node_filesystem_size_bytes < 0.1)
+      and on (instance, device, mountpoint) (node_filesystem_readonly) == 0
+    '';
+    for = "5m";
+    labels = critical;
+    annotations = {
+      Mountpoint = "{{ $labels.mountpoint }}";
+      FreeSpace = "{{ $value | humanizePercentage }}";
+    };
+  };
+
+  alerts.NodeOutOfInodes = {
+    expr = "node_filesystem_files_free / node_filesystem_files < 0.1";
+    for = "3m";
+    labels = critical;
+    annotations = {
+      Mountpoint = "{{ $labels.mountpoint }}";
+      FreeInodes = "{{ $value | humanizePercentage }}";
+    };
+  };
+
+  alerts.NodeUnhealthyDisk = {
+    expr = "smartmon_device_smart_healthy < 1";
+    for = "10m";
+    labels = critical;
+    annotations.Disk = "{{ $labels.disk }}";
+  };
+
+  alerts.NodeZfsWrongState = {
+    expr = ''
+      node_zfs_zpool_state{state!="online"} > 0
+    '';
+    for = "5m";
+    labels = critical;
+    annotations = {
+      State = "{{ $labels.state }}";
+      ZPool = "{{ $labels.zpool }}";
+    };
+  };
+
+  # Clock
+
+  alerts.NodeClockSkew = {
+    expr = ''
+      (node_timex_offset_seconds > 0.05
+        and deriv(node_timex_offset_seconds[5m]) >= 0)
+      or (node_timex_offset_seconds < -0.05
+        and deriv(node_timex_offset_seconds[5m]) <= 0)
+    '';
+    for = "2m";
+    labels = warning;
+  };
+
+  alerts.NodeClockNotSynchronising = {
+    expr = ''
+      min_over_time(node_timex_sync_status[1m]) == 0
+      and node_timex_maxerror_seconds >= 16
+    '';
+    for = "2m";
+    labels = warning;
+  };
+
+  # EDAC
+
+  alerts.NodeEdacCorrectableErrorsDetected = {
+    expr = ''
+      increase(node_edac_correctable_errors_total[1m]) > 0
+    '';
+    for = "0m";
+    labels = warning;
+    annotations.CorrectedErrors = "{{ $value }}";
+  };
+
+  alerts.NodeEdacUncorrectableErrorsDetected = {
+    expr = ''
+      increase(node_edac_uncorrectable_errors_total[1m]) > 0
+    '';
+    for = "0m";
+    labels = critical;
+    annotations.DetectedErrors = "{{ $value }}";
+  };
+
+  # Misc
+
+  alerts.NodeLoad5Usage = {
+    expr = ''
+      node_load5 / (
+        count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.5
+    '';
+    for = "1m";
+    labels = warning;
+    annotations.Load5PerCore = "{{ $value | humanize }}";
+  };
+
+  alerts.NodeSystemdUnitFailed = {
+    expr = ''
+      node_systemd_unit_state{state="failed"} == 1
+    '';
+    for = "5m";
+    labels = warning;
+    annotations.Unit = "{{ $labels.name }}";
+  };
+
+  alerts.NodeBorgmaticLastTooOld = {
+    expr = ''
+      (time()
+      - (node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"}
+      ) > 2 * 24 * 60 * 60)
+    '';
+    for = "0m";
+    labels = warning;
+    annotations.Last = "{{ $value | humanizeDuration }}";
+  };
+
+  alerts.NodeBorgmaticAbsent = {
+    expr = ''
+      up{job="node"} == 1 unless on(instance) node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"}
+    '';
+    for = "0s";
+    labels = warning;
+  };
+
+  alerts.NodeRequiresReboot = {
+    expr = "node_reboot_required > 0";
+    for = "5m";
+    labels = warning;
+  };
+}
+

hosts/vm/victoriametrics/rules/postgresql.nix

@@ -0,0 +1,180 @@
+{ critical, warning, ... }:
+
+{
+  alerts.PostgresqlDown = {
+    expr = "pg_up == 0";
+    for = "0m";
+    labels = critical;
+  };
+
+  alerts.PostgresqlExporterError = {
+    expr = "pg_exporter_last_scrape_error > 0";
+    for = "0m";
+    labels = critical;
+  };
+
+  alerts.PostgresqlReplicationLag = {
+    expr = ''
+      pg_replication_lag > 30
+      and on(instance)
+      pg_replication_is_replica == 1
+    '';
+    for = "0m";
+    labels = critical;
+    annotations = {
+      Lag = "{{ $value }}";
+      DB = "{{ $labels.datname }}";
+    };
+  };
+
+  alerts.PostgresqlTableNotVacuumed = {
+    expr = ''
+      time() - pg_stat_user_tables_last_autovacuum
+      > 60 * 60 * 24
+    '';
+    for = "0m";
+    labels = warning;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlTableNotAnalyzed = {
+    expr = ''
+      time() - pg_stat_user_tables_last_autoanalyze
+      > 60 * 60 * 24
+    '';
+    for = "0m";
+    labels = warning;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlTooManyConnections = {
+    expr = ''
+      (
+        sum by (datname)
+          (pg_stat_activity_count{datname!~"template.*|postgres"})
+      ) * 100
+      > pg_settings_max_connections * 80
+    '';
+    for = "2m";
+    labels = warning;
+    annotations = {
+      UsageLevel = "{{ $value | humanizePercentage }}";
+      DB = "{{ $labels.datname }}";
+    };
+  };
+
+  alerts.PostgresqlDeadLocks = {
+    expr = ''
+      increase(
+        pg_stat_database_deadlocks{
+          datname!~"template.*|postgres"
+        }[1m]
+      ) > 5
+    '';
+    for = "0m";
+    labels = warning;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlSlowQueries = {
+    expr = "pg_slow_queries > 0";
+    for = "2m";
+    labels = warning;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlHighRollbackRate = {
+    expr = ''
+      (
+        rate(
+          pg_stat_database_xact_rollback{datname!~"template.*"}[3m]
+        ) / rate(
+        pg_stat_database_xact_commit{datname!~"template.*"}[3m]
+        )
+      ) > 0.2
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      RollbackRate = "{{ $value | humanizePercentage }}";
+      DB = "{{ $labels.datname }}";
+    };
+  };
+
+  alerts.PostgresqlWalReplicationStopped = {
+    expr = "rate(pg_xlog_position_bytes[1m]) == 0";
+    for = "0m";
+    labels = critical;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlHighRateStatementTimeout = {
+    expr = ''
+      rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3
+    '';
+    for = "0m";
+    labels = critical;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlHighRateDeadlock = {
+    expr = ''
+      increase(
+        postgresql_errors_total{type="deadlock_detected"}[1m]
+      ) > 1
+    '';
+    for = "0m";
+    labels = critical;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alert.PostgresqlTooManyDeadTuples = {
+    expr = ''
+      (
+        (pg_stat_user_tables_n_dead_tup > 10000)
+        / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)
+      ) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)
+    '';
+    for = "2m";
+    labels = warning;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alert.PostgresqlSplitBrain = {
+    expr = ''
+      count(pg_replication_is_replica == 0) != 1
+    '';
+    for = "0m";
+    labels = critical;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlPromotedNode = {
+    expr = ''
+      pg_replication_is_replica
+      and
+      changes(pg_replication_is_replica[1m]) > 0
+    '';
+    for = "0m";
+    labels = warning;
+    annotations.DB = "{{ $labels.datname }}";
+  };
+
+  alerts.PostgresqlTooManyLocksAcquired = {
+    expr = ''
+      (
+        (sum (pg_locks_count))
+        / (
+          pg_settings_max_locks_per_transaction
+          * pg_settings_max_connections
+        )
+      ) * 100 > 20
+    '';
+    for = "2m";
+    labels = critical;
+    annotations = {
+      DB = "{{ $labels.datname }}";
+      Deadlocks = "{{ $value | humanizePercentage }}";
+    };
+  };
+}

hosts/vm/victoriametrics/rules/switch.nix

@@ -0,0 +1,81 @@
+{ critical, warning, ... }:
+
+{
+  alerts.SwitchPromiscuousChange = {
+    expr = ''
+      changes(last_over_time(ifPromiscuousMode[1h])[5m]) > 0
+    '';
+    for = "0m";
+    labels = warning;
+    annotations = {
+      Interface = "{{ $labels.ifName }}";
+      Alias = "{{ $labels.ifAlias }}";
+    };
+  };
+
+  alerts.SwitchInErrors = {
+    expr = ''
+      rate(ifInErrors[15m]) / (
+        rate(ifInUcastPkts[15m])
+        + rate(ifInNUcastPkts[15m])
+        + rate(ifInErrors[15m])
+      ) > 0.0001
+    '';
+    for = "15m";
+    labels = warning;
+    annotations = {
+      ErrorRate = "{{ $value | humanizePercentage }}";
+      Interface = "{{ $labels.ifName }}";
+      Alias = "{{ $labels.ifAlias }}";
+    };
+  };
+
+  alerts.SwitchOutErrors = {
+    expr = ''
+      rate(ifOutErrors[15m]) / (
+        rate(ifOutUcastPkts[15m])
+        + rate(ifOutNUcastPkts[15m])
+        + rate(ifOutErrors[15m])
+      ) > 0.0001
+    '';
+    for = "15m";
+    labels = warning;
+    annotations = {
+      ErrorRate = "{{ $value | humanizePercentage }}";
+      Interface = "{{ $labels.ifName }}";
+      Alias = "{{ $labels.ifAlias }}";
+    };
+  };
+
+  alerts.SwitchInLinkUsage = {
+    expr = ''
+      rate(ifHCInOctets[5m]) / (
+        (ifHighSpeed > 0) * 1000000 / 8
+      ) > 0.8
+    '';
+    for = "5m";
+    keep_firing_for = "10m";
+    labels = warning;
+    annotations = {
+      Usage = "{{ $value | humanizePercentage }}";
+      Interface = "{{ $labels.ifName }}";
+      Alias = "{{ $labels.ifAlias }}";
+    };
+  };
+
+  alerts.SwitchOutLinkUsage = {
+    expr = ''
+      rate(ifHCOutOctets[5m]) / (
+        (ifHighSpeed > 0) * 1000000 / 8
+      ) > 0.8
+    '';
+    for = "5m";
+    keep_firing_for = "10m";
+    labels = warning;
+    annotations = {
+      Usage = "{{ $value | humanizePercentage }}";
+      Interface = "{{ $labels.ifName }}";
+      Alias = "{{ $labels.ifAlias }}";
+    };
+  };
+}

hosts/vm/victoriametrics/rules/unbound.nix

@@ -0,0 +1,11 @@
+{ critical, warning, ... }:
+
+{
+  alerts.UnboundDown = { 
+    expr = ''
+      (up{job="unbound"} unless unbound_up) == 0
+    '';
+    for = "0m";
+    labels = critical; 
+  };
+}

hosts/vm/victoriametrics/rules/ups.nix

@@ -0,0 +1,19 @@
+{ critical, warning, ... }:
+
+{
+  alerts.UpsBatteryFailure = { # s'appelait eatonBatteryFailure avant
+    expr = ''
+      xupsBatteryFailure{xupsBatteryFailure="yes"} == 1
+    '';
+    for = "0m";
+    labels = critical;
+  };
+
+  alerts.UpsBatteryLow = { # idem
+    expr = ''
+      xupsBatTimeRemaining < 1800
+    '';
+    for = "0m";
+    labels = critical;
+  };
+}

hosts/vm/victoriametrics/scrape.nix

@@ -0,0 +1,118 @@
+{ ... }:
+
+{
+
+  bird = {
+    replacementAddress = "$1:9324";
+    targets = [
+      #"infra-1.back.infra.auro.re"
+      #"infra-2.back.infra.auro.re"
+      #"edge-1.back.infra.auro.re"
+      #"edge-2.back.infra.auro.re"
+    ];
+  };
+
+  ceph = {
+    replacementAddress = "$1:9283";
+    targets = import ./targets/ceph.nix {};
+  };
+
+  ilo = {
+    path = "/snmp";
+    params = {
+      module = [
+        "uptime"
+        "ilo_memory"
+        "ilo_power"
+        "ilo_temp_fans"
+      ];
+      auth = [ "ilo" ];
+    };
+    replacementAddress = "127.0.0.1:9116";
+    # Some get-bulk requests are left unanswered when using IPv6,
+    # so we force IPv4
+    replacementTarget = "udp4://$1";
+    targets = import ./targets/ilo.nix {};
+  };
+
+  jitsi = {
+    replacementAddress = "$1:9700";
+    targets = [ "jitsi.adm.crans.org" ];
+  };
+
+  #kea = { # pour quand on en aura un 
+  #  replacementAddress = "$1:9547";
+  #  targets = [
+  #    "dhcp-1.isp.infra.auro.re"
+  #    "dhcp-2.isp.infra.auro.re"
+  #  ];
+  #};
+
+  node = {
+    replacementAddress = "$1:9100";
+    targets = import ./targets/node.nix {};
+  };
+
+  postgresql = {
+    replacementAddress = "$1:9187";
+    targets = import ./targets/postgresql.nix {};
+  };
+
+  #procurve = {
+  #  path = "/snmp";
+  #  params = {
+  #    module = [ "switch" "uptime" ];
+  #    auth = [ "procurve" ];
+  #  };
+    # Procurve switches are cheap…
+  #  interval = "2m";
+  #  timeout = "30s";
+  #  replacementAddress = "127.0.0.1:9116";
+  #  replacementTarget = "$1";
+  #  targets = import ./targets/procurve.nix {};
+  #};
+
+  #quanta = {
+  #  path = "/snmp";
+  #  params = {
+  #    module = [ "switch" "uptime" "quanta" ];
+  #    auth = [ "quanta" ];
+  #  };
+  #  replacementAddress = "127.0.0.1:9116";
+  #  replacementTarget = "$1";
+  #  # Quanta switches are cheap…
+  #  interval = "3m";
+  #  timeout = "45s";
+  #  targets = import ./targets/quanta.nix {};
+  #};
+
+  #snmp-exporter = { # pour le jour où on aura deux victoriametrics pour se surveiller l'un l'autre
+  #  replacementAddress = "$1:9116";
+  #  targets = [
+  #    "prometheus-1.monit.infra.auro.re"
+  #    "prometheus-2.monit.infra.auro.re"
+  #  ];
+  #};
+
+  unbound = {
+    replacementAddress = "$1:9167";
+    targets = import ./targets/unbound.nix {};
+  };
+
+  ups = {
+    path = "/snmp";
+    params.module = [ "ups" "uptime" ];
+    params.auth = [ "ups" ];
+    replacementAddress = "127.0.0.1:9116";
+    replacementTarget = "$1";
+    targets = import ./targets/ups.nix {};
+  };
+
+#  victoria-metrics = { # pour le jour où on aura deux victoriametrics pour se surveiller l'un l'autre
+#    replacementAddress = "$1:8428";
+#    targets = [
+#      "victoriametrics.adm.crans.org"
+#     # "prometheus-2.monit.infra.auro.re"
+#    ];
+#  };
+}

hosts/vm/victoriametrics/snmp.nix

@@ -0,0 +1,218 @@
+{ ... }:
+
+{
+  auths = {
+    ilo = {
+      version = 3;
+      username = "root"; # A VERIFIER
+      security_level = "authPriv";
+      auth_protocol = "SHA";
+      password = "\${ILO_AUTH}";
+      priv_protocol = "AES";
+      priv_password = "\${ILO_PRIV}";
+    };
+    ups = {
+      version = 1;
+      community = "prometheus";
+    };
+    ## Tenté, échoue sur 10.201.3.1 (celui dans le local serveurs de EDC). :(
+    #ups = {
+    #  version = 3;
+    #  username = "prometheus";
+    #  security_level = "authPriv";
+    #  auth_protocol = "SHA";
+    #  password = "\${ups_AUTH}";
+    #  priv_protocol = "AES";
+    #  priv_password = "\${ups_PRIV}";
+    #};
+  };
+
+  modules.ups = {
+    walk = [
+      "xupsInput"
+      "xupsOutput"
+      "xupsBypass"
+      "xupsEnvironment"
+      "xupsBattery"
+      "xupsConfig"
+    ];
+    lookups = [
+      {
+        lookup = "xupsInputName";
+        source_indexes = [ "xupsInputPhase" ];
+      }
+      {
+        lookup = "xupsOutputName";
+        source_indexes = [ "xupsOutputPhase" ];
+      }
+      {
+        lookup = "xupsBypassName";
+        source_indexes = [ "xupsBypassPhase" ];
+      }
+    ];
+    overrides = {
+      upsBatteryStatus.type = "EnumAsStateSet";
+      xupsBatteryAbmStatus.type = "EnumAsStateSet";
+      xupsBypassId.type = "EnumAsStateSet";
+      xupsContactType.type = "EnumAsStateSet";
+      xupsInputId.type = "EnumAsStateSet";
+      xupsOutputId.type = "EnumAsStateSet";
+      xupsOutputSource.type = "EnumAsStateSet";
+      xupsBatteryFailure.type = "EnumAsStateSet";
+    };
+  };
+
+  modules.ilo_temp_fans = {
+    walk = [
+      #"cpqHeThermalFanTable"
+      "cpqHeFltTolFanTable"
+      "cpqHeTemperatureTable"
+      #"cpqHeCoolingModuleTable"
+    ];
+    timeout = "2s";
+    retries = 10;
+    overrides = {
+      # cpqHeThermalFanTable
+      # cpqHeFltTolFanTable
+      cpqHeFltTolFanChassis.ignore = true;
+      cpqHeFltTolFanHotPlug.type = "EnumAsInfo";
+      cpqHeFltTolFanIndex.ignore = true;
+      cpqHeFltTolFanLocale.type = "EnumAsInfo";
+      cpqHeFltTolFanPresent.type = "EnumAsStateSet";
+      cpqHeFltTolFanType.type = "EnumAsInfo";
+      cpqHeFltTolFanSpeed.type = "EnumAsStateSet";
+      cpqHeFltTolFanRedundant.type = "EnumAsStateSet";
+      cpqHeFltTolFanCondition.type = "EnumAsStateSet";
+      # cpqHeTemperatureTable
+      cpqHeTemperatureChassis.ignore = true;
+      cpqHeTemperatureIndex.ignore = true;
+      cpqHeTemperatureLocale.type = "EnumAsInfo";
+      cpqHeTemperatureCondition.type = "EnumAsStateSet";
+      cpqHeTemperatureThresholdType.type = "EnumAsInfo";
+    };
+  };
+
+  modules.ilo_power = {
+    walk = [
+      "cpqHeFltTolPowerSupplyTable"
+      #"cpqHePowerMeter"
+    ];
+    timeout = "2s";
+    retries = 10;
+    overrides = {
+      # cpqHeFltTolPowerSupplyTable
+      cpqHeFltTolPowerSupplyChassis.ignore = true;
+      cpqHeFltTolPowerSupplyHotPlug.type = "EnumAsInfo";
+      cpqHeFltTolPowerSupplyErrorCondition.type = "EnumAsStateSet";
+      cpqHeFltTolPowerSupplyBay.ignore = true;
+      cpqHeFltTolPowerSupplyPresent.type = "EnumAsStateSet";
+      cpqHeFltTolPowerSupplyCondition.type = "EnumAsStateSet";
+      cpqHeFltTolPowerSupplyStatus.type = "EnumAsStateSet";
+      cpqHeFltTolPowerSupplyRedundant.type = "EnumAsStateSet";
+    };
+  };
+
+  modules.ilo_memory = {
+    walk = [
+      #"cpqHeCorrectableMemory"
+      #"cpqHePowerFailure"
+      #"cpqHeCriticalError"
+      "cpqHeResMem2BoardTable"
+      "cpqHeResMem2ModuleTable"
+      #"cpqHeHWBios"
+      #"cpqHeSysBackupBattery"
+      #"cpqHeResilientMemory"
+      #"cpqSm2Cntlr"
+      #"cpqHostOs"
+      #"cpqSeCpuTable"
+      #"cpqDaCntlr"
+    ];
+    timeout = "2s";
+    retries = 10;
+    overrides = {
+      #cpqHeResMem2BoardTable
+      cpqHeResMem2BoardCondition.type = "EnumAsStateSet"; 
+      cpqHeResMem2BoardHotPlug.type = "EnumAsInfo";
+      cpqHeResMem2BoardOnlineStatus.type = "EnumAsStateSet";
+      cpqHeResMem2BoardErrorStatus.type = "EnumAsStateSet";
+      cpqHeResMem2BoardLocked.type = "EnumAsStateSet";
+      #cpqHeResMem2ModuleTable
+      cpqHeResMem2ModuleCellStatus.type = "EnumAsStateSet";
+      cpqHeResMem2ModuleCondition.type = "EnumAsStateSet";
+      cpqHeResMem2ModuleSmartMemory.type = "EnumAsInfo"; 
+      cpqHeResMem2ModuleStatus.type = "EnumAsStateSet";
+      cpqHeResMem2ModuleTechnology.type = "EnumAsInfo";
+      cpqHeResMem2ModuleType.type = "EnumAsInfo";
+    }; 
+  };
+
+  modules.switch = {
+    walk = [
+      "interfaces"
+      "ifXTable"
+      "dot3StatsTable"
+      "dot3HCStatsTable"
+      "dot3Errors"
+      "dot3Tests"
+      "dot3CollTable"
+      "lldpLocChassisId"
+      "lldpRemTable"
+      "lldpLocPortTable"
+      #"dot1dBasePort"
+    ];
+    lookups = [
+      {
+        lookup = "ifAlias";
+        source_indexes = [ "ifIndex" ];
+      }
+      {
+        lookup = "ifDescr";
+        source_indexes = [ "ifIndex" ];
+      }
+      {
+        lookup = "ifName";
+        source_indexes = [ "ifIndex" ];
+      }
+      {
+        lookup = "ifAlias";
+        source_indexes = [ "dot3StatsIndex" ];
+      }
+      {
+        lookup = "ifDescr";
+        source_indexes = [ "dot3StatsIndex" ];
+      }
+      {
+        lookup = "ifName";
+        source_indexes = [ "dot3StatsIndex" ];
+      }
+      {
+        lookup = "lldpRemChassisId";
+        source_indexes = [
+          "lldpRemTimeMark"
+          "lldpRemLocalPortNum"
+          "lldpRemIndex"
+        ];
+      }
+    ];  
+    overrides = {
+      dot3StatsDuplexStatus.type = "EnumAsStateSet";
+      dot3StatsEtherChipSet.ignore = true;
+      dot3StatsIndex.ignore = true;
+      ifAdminStatus.type = "EnumAsStateSet";
+      ifAlias.ignore = true;
+      ifDescr.ignore = true;
+      ifIndex.ignore = true;
+      ifName.ignore = true;
+      ifOperStatus.type = "EnumAsStateSet";
+      ifType.type = "EnumAsInfo";
+      lldpLocPortIdSubtype.type = "EnumAsInfo";
+      lldpRemPortIdSubtype.type = "EnumAsInfo";
+    };
+    timeout = "5s";
+    retries = 10;
+  };
+
+  modules.uptime = {
+    walk = [ "sysUpTime" ];
+  };
+} 

hosts/vm/victoriametrics/targets/ceph.nix

@@ -0,0 +1,5 @@
+{ ... }:
+
+[
+    "cephiroth.adm.crans.org"
+]

hosts/vm/victoriametrics/targets/ilo.nix

@@ -0,0 +1,13 @@
+{ ... }:
+
+[
+    "ilo-cephiroth.adm.crans.org"
+    "ilo-daniel.adm.crans.org"
+    "ilo-gulp.adm.crans.org"
+    "ilo-jack.adm.crans.org"
+    "ilo-odlyd.adm.crans.org"
+    "ilo-sam.adm.crans.org"
+    "ilo-stitch.adm.crans.org"
+    "ilo-thot.adm.crans.org"
+    "ilo-zamok.adm.crans.org"
+]

hosts/vm/victoriametrics/targets/node.nix

@@ -0,0 +1,57 @@
+{ ... }:
+
+[
+    "backup-ft.adm.crans.org"
+    "boeing.adm.crans.org"
+    "cameron.adm.crans.org"
+    "cas.adm.crans.org"
+    "collabora.adm.crans.org"
+    "daniel.adm.crans.org"
+    "ecilis.adm.crans.org"
+    "eclat.adm.crans.org"
+    "en7.adm.crans.org"
+    "ethercalc.adm.crans.org"
+    "flirt.adm.crans.org"
+    "ft.adm.crans.org"
+    "fyre.adm.crans.org"
+    "gitlab-ci.adm.crans.org"
+    "gitzly.adm.crans.org"
+    "gulp.adm.crans.org"
+    "helloworld.adm.crans.org"
+    "hodaur.adm.crans.org"
+    "irc.adm.crans.org"
+    "jack.adm.crans.org"
+    "jitsi.adm.crans.org"
+    "kenobi.adm.crans.org"
+    "kiwi.adm.crans.org"
+    "mailman.adm.crans.org"
+    "neo.adm.crans.org"
+    "neree.adm.crans.org"
+    "nextcloud.adm.crans.org"
+    "owl.adm.crans.org"
+    "owncloud.adm.crans.org"
+    "periodique.adm.crans.org"
+    "proxy-pve-adh.adm.crans.org"
+    "ptf.adm.crans.org"
+    "re2o.adm.crans.org"
+    "redisdead.adm.crans.org"
+    "reverseproxy.adm.crans.org"
+    "romanesco.adm.crans.org"
+    "roundcube.adm.crans.org"
+    "routeur-2754.adm.crans.org"
+    "routeur-daniel.adm.crans.org"
+    "routeur-ft.adm.crans.org"
+    "routeur-jack.adm.crans.org"
+    "routeur-sam.adm.crans.org"
+    "sam.adm.crans.org"
+    "silice.adm.crans.org"
+    "sputnik.adm.crans.org"
+    "stitch.adm.crans.org"
+    "tealc.adm.crans.org"
+    "thot.adm.crans.org"
+    "victoriametrics.adm.crans.org"
+    "voyager.adm.crans.org"
+    "wall-e.adm.crans.org"
+    "yson-partou.adm.crans.org"
+    "zamok.adm.crans.org"
+]

hosts/vm/victoriametrics/targets/postgresql.nix

@@ -0,0 +1,8 @@
+{ ... }:
+
+[	
+    "daniel.adm.crans.org"
+    "jack.adm.crans.org"
+    "sam.adm.crans.org"
+    "tealc.adm.crans.org"
+]

hosts/vm/victoriametrics/targets/unbound.nix

@@ -0,0 +1,5 @@
+{...}:
+[
+    "silice.adm.crans.org"
+    "sputnik.adm.crans.org"
+]

hosts/vm/victoriametrics/targets/ups.nix

@@ -0,0 +1,5 @@
+{ ... }:
+
+[
+    "excalibur.adm.crans.org"
+]

hosts/vm/victoriametrics/victoriametrics.nix

@@ -0,0 +1,103 @@
+{ config, lib, pkgs, ... }:
+
+let 
+  mkScrapeConfig = name: config: let
+    relabelAddress = lib.optional (builtins.hasAttr "replacementAddress" config) {
+      source_labels = [ "__address__" ];
+      target_label = "__address__";
+      replacement = config.replacementAddress;
+    };
+    relabelTarget = lib.optional (builtins.hasAttr "replacementTarget" config) {
+      source_labels = [ "__address__" ];
+      target_label = "__param_target";
+      replacement = config.replacementTarget;
+    };
+  in {
+    job_name = name;
+    metrics_path = config.path or "/metrics";
+    static_configs = [ { targets = config.targets; } ];
+    params = config.params or { };
+    relabel_configs = [
+      { source_labels = [ "__address__" ]; target_label = "instance"; }
+#      { source_labels = [ "__address__" ]; target_label = "__param_target"; }
+#      { source_labels = [ "__param_target" ]; target_label = "instance"; }
+    ] ++ relabelTarget ++ relabelAddress;
+    scrape_timeout = config.timeout or "15s";
+    scrape_interval = config.interval or "1m";
+  };
+  mkScrapeConfigs = lib.attrsets.mapAttrsToList mkScrapeConfig;
+  critical = { severity = "critical"; };
+  warning = { severity = "warning"; };
+  mkRuleGroups = lib.attrsets.mapAttrsToList (name: path: {
+    inherit name;
+    rules = let
+      rules = import path { inherit critical warning; };
+      alerts = lib.attrsets.mapAttrsToList
+        (alert: attrs: attrs // { inherit alert; })
+        (rules.alerts or {});
+      records = lib.attrsets.mapAttrsToList
+        (record: attrs: attrs // { inherit record; })
+        (rules.records or {});
+    in alerts ++ records;
+  });
+in
+{
+  age.secrets.snmp-env = {
+    file = ../../../secrets/victoriametrics/snmp.age;
+    owner = "root"; 
+    group = "root";
+  };
+
+  networking.firewall.allowedTCPPorts = [ 8428 9116 ];
+
+  services.victoriametrics = let
+    scrapeConfigs = mkScrapeConfigs (import ./scrape.nix {});
+  in {
+    enable = true;
+    retentionPeriod = "30d";
+    extraOptions = [
+      "-enableTCP6"
+      # Compatibilité avec la datasource Prometheus de Grafana
+      "-vmalert.proxyURL=http://localhost:8880"
+    ];
+    prometheusConfig.scrape_configs = scrapeConfigs;
+  };
+
+  services.vmalert.instances.default = {
+    enable = true;
+    rules.groups = mkRuleGroups (import ./rules {});
+    settings = let
+      victoriametricsUrl = "http://localhost:8428";
+    in {
+      "datasource.url" = victoriametricsUrl;
+      "remoteWrite.url" = victoriametricsUrl;
+      "remoteRead.url" = victoriametricsUrl;
+      "enableTCP6" = true;
+      #"notifier.url" = [ "http://docker-ovh.adm.auro.re:9093" ];  # A MODIFIER #####################################
+    };
+  };
+
+  services.prometheus.exporters.snmp = let
+    generator = import ./snmp.nix {};
+    configSnmp = pkgs.runCommand "snmp.yml" { # Nous avons des doutes sur ce fichier? faut-il l'importer?
+      nativeBuildInputs = [ pkgs.prometheus-snmp-exporter ];
+      generator = builtins.toJSON generator;
+      passAsFile = [ "generator" ];
+    } ''
+      ${lib.getExe' pkgs.prometheus-snmp-exporter "generator"} \
+        generate \
+        --mibs-dir=${./mibs} \
+        --generator-path=$generatorPath \
+        --output-path=$out
+    '';
+  in {
+    enable = true;
+    enableConfigCheck = false;
+    environmentFile = config.age.secrets.snmp-env.path;
+    configurationPath = configSnmp;
+  }; 
+
+  environment.systemPackages = with pkgs; [
+    net-snmp
+  ];
+}