diff --git a/hosts/vm/victoriametrics/default.nix b/hosts/vm/victoriametrics/default.nix index 203fddf..658d45e 100644 --- a/hosts/vm/victoriametrics/default.nix +++ b/hosts/vm/victoriametrics/default.nix @@ -4,7 +4,8 @@ { imports = [ ./hardware-configuration.nix - ]; + ./victoriametrics.nix + ]; networking.hostName = "victoriametrics"; boot.loader.grub.devices = [ "/dev/sda" ]; diff --git a/hosts/vm/victoriametrics/rules/bird.nix b/hosts/vm/victoriametrics/rules/bird.nix new file mode 100644 index 0000000..199db80 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/bird.nix @@ -0,0 +1,163 @@ +{ critical, warning, ... }: + +{ + records."bird:protocol_up:bgp_all".expr = '' + label_replace( + bird_protocol_up{proto="BGP"}, + "group", "$1", + "instance", "^([^0-9\\.]+)-[0-9]+.*" + ) + ''; + + # FIXME: sessions en cours d'installation, pas encore monitorées + records."bird:protocol_up:bgp".expr = '' + bird:protocol_up:bgp_all + unless + bird:protocol_up:bgp_all{ + group="edge", + name=~"^(viarezo|isp[12]|rezel)[46]$" + } + ''; + + # Sessions qui ne sont volontairement pas redondées + # au sein d'un groupe + records."bird:protocol_up:bgp:non_redundant".expr = '' + bird:protocol_up:bgp{ + group="edge", + name=~"^(oti|crans|legacy|edge)[46]$" + } + ''; + + # Sessions qui le sont + records."bird:protocol_up:bgp:redundant".expr = '' + bird:protocol_up:bgp + unless + bird:protocol_up:bgp:non_redundant + ''; + + alerts.BirdBGPRedundancyDegraded = { + expr = '' + ( + count by (group, name) ( + bird:protocol_up:bgp:redundant{state="Established"} + ) + ( + count by (group, name) ( + bird:protocol_up:bgp:redundant{state!="Established"} * 0 + ) + ) + ) < 2 + ''; + for = "0m"; + labels = warning; + annotations = { + Session = "{{ $labels.name }}"; + Count = "{{ $value }}"; + Group = "{{ $labels.group }}"; + }; + }; + + alerts.BirdBGPDown = { + expr = '' + ( + count by (group, name) ( + bird:protocol_up:bgp{state="Established"} + ) + ( + count by (group, name) ( + bird:protocol_up:bgp{state!="Established"} * 0 + ) + ) + ) == 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Session = "{{ $labels.name }}"; + Group = "{{ $labels.group }}"; + }; + }; + + # TODO: warning pour redondant ? + alerts.BirdBGPNoExportedPrefixRedundant = { + expr = '' + bird_protocol_prefix_export_count{ + export_filter!="REJECT", + } * on (instance, name) group_left (group) ( + bird:protocol_up:bgp:redundant{state="Established"} + ) == 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Session = "{{ $labels.name }}"; + Group = "{{ $labels.group }}"; + }; + }; + + alerts.BirdBGPNoImportedPrefixRedundant = { + expr = '' + bird_protocol_prefix_import_count{ + import_filter!="REJECT", + } * on (instance, name) group_left (group) ( + bird:protocol_up:bgp:redundant{state="Established"} + ) == 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Session = "{{ $labels.name }}"; + Group = "{{ $labels.group }}"; + }; + }; + + alerts.BirdBGPNoExportedPrefixNonRedundant = { + expr = '' + sum by (group) ( + bird_protocol_prefix_export_count{ + export_filter!="REJECT", + } * on (instance, name) group_left (group) ( + bird:protocol_up:bgp:non_redundant{state="Established"} + ) + ) == 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Session = "{{ $labels.name }}"; + Group = "{{ $labels.group }}"; + }; + }; + + alerts.BirdBGPNoImportedPrefixNonRedundant = { + expr = '' + sum by (group) ( + bird_protocol_prefix_import_count{ + import_filter!="REJECT", + } * on (instance, name) group_left (group) ( + bird:protocol_up:bgp:non_redundant{state="Established"} + ) + ) == 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Session = "{{ $labels.name }}"; + Group = "{{ $labels.group }}"; + }; + }; + + alerts.BirdOSPFNeighboursChange = { + expr = '' + changes(bird_ospf_neighbor_count[5m]) > 0 + or changes(bird_ospfv3_neighbor_count[5m]) > 0 + ''; + for = "0m"; + labels = warning; + }; + + alerts.BirdOSPFDown = { + expr = "bird_ospf_running == 0"; + for = "0m"; + labels = critical; + annotations.Instance = "{{ $labels.name }}"; + }; +} diff --git a/hosts/vm/victoriametrics/rules/ceph.nix b/hosts/vm/victoriametrics/rules/ceph.nix new file mode 100644 index 0000000..2ceea17 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/ceph.nix @@ -0,0 +1,149 @@ +{ critical, warning, ... }: + +{ + alerts.CephErrorState = { + expr = '' + ceph_health_status > 1 + ''; + for = "5m"; + labels = critical; + }; + + alerts.CephMonitorAvailableStorageCritical = { + expr = '' + ceph_monitor_avail_percent < 15 + ''; + for = "1m"; + labels = critical; + }; + + alerts.CephMonitorAvailableStorageWarning = { + expr = '' + ceph_monitor_avail_percent < 30 + ''; + for = "1m"; + labels = warning; + }; + + alerts.CephMonitorClockSkewTooHigh = { + expr = '' + abs(ceph_monitor_clock_skew_seconds) > 0.1 + ''; + for = "1m"; + labels = warning; + }; + + alerts.CephOSDApplyLatencyTooHigh = { + expr = '' + ceph_osd_perf_apply_latency_seconds > 10 + ''; + for = "1m"; + labels = warning; + annotations = { + Latency = "{{ $value }}"; + Device = "{{ $labels.device }}"; + }; + }; + + alerts.CephOSDDown = { + expr = '' + ceph_osd_up == 0 + ''; + for = "1m"; + labels = critical; + }; + + alerts.CephOSDReweighted = { + expr = '' + ceph_osd_weight < 1 + ''; + for = "1h"; + labels = warning; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephOSDUtilization = { + expr = '' + ceph_osd_utilization > 90 + ''; + for = "1m"; + labels = critical; + }; + + alerts.CephPgActivating = { + expr = '' + ceph_pg_activating > 0 + ''; + for = "5m"; + labels = critical; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephPgDown = { + expr = '' + ceph_pg_down > 0 + ''; + for = "3m"; + labels = critical; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephPgIncomplete = { + expr = '' + ceph_pg_incomplete > 0 + ''; + for = "2m"; + labels = critical; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephPgInconsistent = { + expr = '' + ceph_pg_inconsistent > 0 + ''; + for = "1m"; + labels = warning; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephPgBackfillTooFull = { + expr = '' + ceph_pg_backfill_toofull > 0 + ''; + for = "5m"; + labels = warning; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephPgUnavailable = { + expr = '' + ceph_pg_total - ceph_pg_active > 0 + ''; + for = "5m"; + labels = critical; + annotations.Instance = "{{ $labels.cluster }}"; + annotations.Database = "{{ $value }}"; + }; + + alerts.CephTargetDown = { + expr = '' + up{job="ceph"} == 0 + ''; + for = "10m"; + labels = critical; + }; + + alerts.CephWarnState = { + expr = '' + ceph_health_status == 1 + ''; + for = "30m"; + labels = warning; + }; +} diff --git a/hosts/vm/victoriametrics/rules/default.nix b/hosts/vm/victoriametrics/rules/default.nix new file mode 100644 index 0000000..a3dee8d --- /dev/null +++ b/hosts/vm/victoriametrics/rules/default.nix @@ -0,0 +1,13 @@ +{ ... }: + +{ + bird = ./bird.nix; + ceph = ./ceph.nix; + eaton = ./eaton.nix; + ilo = ./ilo.nix; + jitsi = ./jitsi.nix; + node = ./node.nix; + postgresql = ./postgresql.nix; + switch = ./switch.nix; + unbound = ./unbound.nix; +} diff --git a/hosts/vm/victoriametrics/rules/eaton.nix b/hosts/vm/victoriametrics/rules/eaton.nix new file mode 100644 index 0000000..f7b4743 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/eaton.nix @@ -0,0 +1,19 @@ +{ critical, warning, ... }: + +{ + alerts.EatonBatteryFailure = { + expr = '' + xupsBatteryFailure{xupsBatteryFailure="yes"} == 1 + ''; + for = "0m"; + labels = critical; + }; + + alerts.EatonBatteryLow = { + expr = '' + xupsBatTimeRemaining < 1800 + ''; + for = "0m"; + labels = critical; + }; +} diff --git a/hosts/vm/victoriametrics/rules/ilo.nix b/hosts/vm/victoriametrics/rules/ilo.nix new file mode 100644 index 0000000..ca7a3cf --- /dev/null +++ b/hosts/vm/victoriametrics/rules/ilo.nix @@ -0,0 +1,140 @@ +{ critical, warning, ... }: + +{ + # cpqHeFltTolFanTable + + alerts.IloFanDegraded = { + expr = '' + cpqHeFltTolFanCondition{ + cpqHeFltTolFanCondition="degraded" + } == 1 + ''; + for = "0m"; + labels = warning; + annotations = { + Chassis = "{{ $labels.cpqHeFltTolFanChassis }}"; + Index = "{{ $labels.cpqHeFltTolFanIndex }}"; + }; + }; + + alerts.IloFanFailed = { + expr = '' + cpqHeFltTolFanCondition{ + cpqHeFltTolFanCondition="failed" + } == 1 + ''; + for = "0m"; + labels = critical; + annotations = { + Chassis = "{{ $labels.cpqHeFltTolFanChassis }}"; + Index = "{{ $labels.cpqHeFltTolFanIndex }}"; + }; + }; + + # cpqHeFltTolPowerSupplyTable + + alerts.IloHighPSUUsage = { + expr = '' + cpqHeFltTolPowerSupplyCapacityUsed + / (cpqHeFltTolPowerSupplyCapacityMaximum > 0) > 0.5 + ''; + for = "0m"; + labels = warning; + annotations = { + Usage = "{{ $value | humanizePercentage }}"; + Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}"; + Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}"; + }; + }; + + alerts.IloPSUDegraded = { + expr = '' + cpqHeFltTolPowerSupplyCondition{ + cpqHeFltTolPowerSupplyCondition="degraded" + } == 1 + ''; + for = "0m"; + labels = warning; + annotations = { + Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}"; + Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}"; + }; + }; + + alerts.IloPSUFailed = { + expr = '' + ( + cpqHeFltTolPowerSupplyCondition{ + cpqHeFltTolPowerSupplyCondition="failed" + } == 1 + ) * on ( + instance, + cpqHeFltTolPowerSupplyChassis, + cpqHeFltTolPowerSupplyBay + ) group_left ( + cpqHeFltTolPowerSupplyErrorCondition + ) + (cpqHeFltTolPowerSupplyErrorCondition == 1) + ''; + for = "0m"; + labels = warning; + annotations = { + Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}"; + Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}"; + Error = "{{ $labels.cpqHeFltTolPowerSupplyErrorCondition }}"; + }; + }; + + # cpqHeTemperatureTable + + alerts.IloTooHot = { + expr = '' + ( + cpqHeTemperatureCondition{ + cpqHeTemperatureCondition!="ok" + } == 1 + ) * on ( + instance, + cpqHeTemperatureChassis, + cpqHeTemperatureIndex + ) group_left (cpqHeTemperatureLocale) + cpqHeTemperatureLocale_info + ''; + for = "0m"; + labels = warning; + annotations = { + Chassis = "{{ $labels.cpqHeTemperatureChassis }}"; + Index = "{{ $labels.cpqHeTemperatureIndex }}"; + Locale = "{{ $labels.cpqHeTemperatureLocale }}"; + Condition = "{{ $labels.cpqHeTemperatureCondition }}"; + }; + }; + + # cpqHeResMem2ModuleTable + + alerts.IloRamHS = { + expr = '' + cpqHeResMem2ModuleStatus{ + cpqHeResMem2ModuleStatus!~"good|notPresent" + } == 1 + ''; + for = "0m"; + labels = critical; + annotations = { + Module = "{{ $labels.cpqHeResMem2Module }}"; + Status = "{{ $labels.cpqHeResMem2ModuleStatus }}"; + }; + }; + + alerts.IloUneSeuleRam = { + expr = '' + count by (instance) ( + cpqHeResMem2ModuleStatus{ + cpqHeResMem2ModuleStatus="good" + } == 1 + ) <= 1 + ''; + for = "0m"; + labels = critical; + }; +} diff --git a/hosts/vm/victoriametrics/rules/jitsi.nix b/hosts/vm/victoriametrics/rules/jitsi.nix new file mode 100644 index 0000000..f77cc83 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/jitsi.nix @@ -0,0 +1,9 @@ +{ critical, ... }: + +{ + alerts.JitsiHighStressLevel = { + expr = "jitsi_stress_level > 0.7"; + for = "3m"; + labels = critical; + }; +} diff --git a/hosts/vm/victoriametrics/rules/node.nix b/hosts/vm/victoriametrics/rules/node.nix new file mode 100644 index 0000000..38034d3 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/node.nix @@ -0,0 +1,318 @@ +{ critical, warning, ... }: +{ + # Memory + + alerts.NodeOutOfMemory = { + expr = '' + ( + ( + node_memory_MemAvailable_bytes + + ( + clamp_min(node_zfs_arc_size - node_zfs_arc_c_min, 0) + or node_memory_MemAvailable_bytes * 0 + ) + ) + / node_memory_MemTotal_bytes + ) < 0.1 + ''; + for = "3m"; + labels = critical; + annotations.Available = "{{ $value | humanizePercentage }}"; + }; + + alerts.NodeUnderMemoryPressure = { + expr = "rate(node_vmstat_pgmajfault[5m]) > 1000"; + labels = critical; + for = "0m"; + annotations.Pressure = "{{ $value | humanize }}"; + }; + + alerts.NodeSwapIsFillingUp = { + expr = '' + (1 - (node_memory_SwapFree_bytes + / node_memory_SwapTotal_bytes)) > 0.75 + ''; + for = "1m"; + labels = critical; + annotations.UsedSwap = "{{ $value | humanizePercentage }}"; + }; + + alerts.NodeOomKillDetected = { + expr = "increase(node_vmstat_oom_kill[1m]) > 0"; + for = "0m"; + labels = critical; + }; + + # CPU + + alerts.NodeCpuUsage = { + expr = '' + (avg by (instance) + (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8 + ''; + for = "10m"; + labels = warning; + annotations.AverageUsage = "{{ $value | humanizePercentage }}"; + }; + + alerts.NodeCpuStealNoisyNeighbor = { + expr = '' + avg by (instance) ( + rate(node_cpu_seconds_total{mode="steal"}[2m]) + ) > 0.1 + ''; + for = "10m"; + labels = warning; + annotations.Steal = "{{ $value | humanizePercentage }}"; + }; + + # Network + + alerts.NodeLinkHighUsageIn = { + expr = '' + (rate(node_network_receive_bytes_total[5m]) + / on(instance, device) node_network_speed_bytes) > .80 + ''; + labels = warning; + for = "3m"; + annotations = { + Usage = "{{ $value | humanizePercentage }}"; + Device = "{{ $labels.device }}"; + }; + }; + + alerts.NodeLinkHighUsageOut = { + expr = '' + (rate(node_network_transmit_bytes_total[5m]) + / on(instance, device) node_network_speed_bytes) > .80 + ''; + labels = warning; + for = "3m"; + annotations = { + Usage = "{{ $value | humanizePercentage }}"; + Device = "{{ $labels.device }}"; + }; + }; + + alerts.NodeConntrackLimit = { + expr = '' + (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.8 + ''; + for = "5m"; + labels = warning; + annotations.Filled = "{{ $value | humanizePercentage }}"; + }; + + alerts.NodeNetworkReceiveErrors = { + expr = '' + rate(node_network_receive_errs_total[2m]) + / rate(node_network_receive_packets_total[2m]) > 0.01 + ''; + for = "2m"; + labels = warning; + annotations = { + Errors = "{{ $value | humanizePercentage }}"; + Device = "{{ $labels.device }}"; + }; + }; + + alerts.NodeNetworkTransmitErrors = { + expr = '' + rate(node_network_transmit_errs_total[2m]) + / rate(node_network_transmit_packets_total[2m]) > 0.01 + ''; + for = "2m"; + labels = warning; + annotations = { + Errors = "{{ $value | humanizePercentage }}"; + Device = "{{ $labels.device }}"; + }; + }; + + alerts.NodeNetworkBondDegraded = { + expr = "node_bonding_active - node_bonding_slaves != 0"; + for = "2m"; + labels = warning; + annotations.Device = "{{ $labels.device }}"; + }; + + # Temperature + + alerts.NodePhysicalComponentTooHot = { + expr = '' + node_hwmon_temp_celsius > clamp_max(node_hwmon_temp_max_celsius, 79) + ''; + for = "0m"; + labels = critical; + annotations = { + Temperature = "{{ $value | humanize }} °C"; + Chip = "{{ $labels.chip }}"; + Sensor = "{{ $labels.sensor }}"; + }; + }; + + alerts.NodeNodeOvertemperatureAlarm = { + expr = "node_hwmon_temp_crit_alarm_celsius == 1"; + for = "0m"; + labels = critical; + annotations = { + Chip = "{{ $labels.chip }}"; + Sensor = "{{ $labels.sensor }}"; + }; + }; + + # Storage and disks + + alerts.NodeRaidArrayGotInactive = { + expr = '' + node_md_state{state="inactive"} > 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Device = "{{ $labels.device }}"; + }; + }; + + alerts.NodeRaidDiskFailure = { + expr = '' + node_md_disks{state="failed"} > 0 + ''; + for = "0m"; + labels = critical; + annotations = { + Device = "{{ $labels.md_device }}"; + }; + }; + + alerts.NodeOutOfDiskSpace = { + expr = '' + (node_filesystem_free_bytes / node_filesystem_size_bytes < 0.1) + and on (instance, device, mountpoint) (node_filesystem_readonly) == 0 + ''; + for = "5m"; + labels = critical; + annotations = { + Mountpoint = "{{ $labels.mountpoint }}"; + FreeSpace = "{{ $value | humanizePercentage }}"; + }; + }; + + alerts.NodeOutOfInodes = { + expr = "node_filesystem_files_free / node_filesystem_files < 0.1"; + for = "3m"; + labels = critical; + annotations = { + Mountpoint = "{{ $labels.mountpoint }}"; + FreeInodes = "{{ $value | humanizePercentage }}"; + }; + }; + + alerts.NodeUnhealthyDisk = { + expr = "smartmon_device_smart_healthy < 1"; + for = "10m"; + labels = critical; + annotations.Disk = "{{ $labels.disk }}"; + }; + + alerts.NodeZfsWrongState = { + expr = '' + node_zfs_zpool_state{state!="online"} > 0 + ''; + for = "5m"; + labels = critical; + annotations = { + State = "{{ $labels.state }}"; + ZPool = "{{ $labels.zpool }}"; + }; + }; + + # Clock + + alerts.NodeClockSkew = { + expr = '' + (node_timex_offset_seconds > 0.05 + and deriv(node_timex_offset_seconds[5m]) >= 0) + or (node_timex_offset_seconds < -0.05 + and deriv(node_timex_offset_seconds[5m]) <= 0) + ''; + for = "2m"; + labels = warning; + }; + + alerts.NodeClockNotSynchronising = { + expr = '' + min_over_time(node_timex_sync_status[1m]) == 0 + and node_timex_maxerror_seconds >= 16 + ''; + for = "2m"; + labels = warning; + }; + + # EDAC + + alerts.NodeEdacCorrectableErrorsDetected = { + expr = '' + increase(node_edac_correctable_errors_total[1m]) > 0 + ''; + for = "0m"; + labels = warning; + annotations.CorrectedErrors = "{{ $value }}"; + }; + + alerts.NodeEdacUncorrectableErrorsDetected = { + expr = '' + increase(node_edac_uncorrectable_errors_total[1m]) > 0 + ''; + for = "0m"; + labels = critical; + annotations.DetectedErrors = "{{ $value }}"; + }; + + # Misc + + alerts.NodeLoad5Usage = { + expr = '' + node_load5 / ( + count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.5 + ''; + for = "1m"; + labels = warning; + annotations.Load5PerCore = "{{ $value | humanize }}"; + }; + + alerts.NodeSystemdUnitFailed = { + expr = '' + node_systemd_unit_state{state="failed"} == 1 + ''; + for = "5m"; + labels = warning; + annotations.Unit = "{{ $labels.name }}"; + }; + + alerts.NodeBorgmaticLastTooOld = { + expr = '' + (time() + - (node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"} + ) > 2 * 24 * 60 * 60) + ''; + for = "0m"; + labels = warning; + annotations.Last = "{{ $value | humanizeDuration }}"; + }; + + alerts.NodeBorgmaticAbsent = { + expr = '' + up{job="node"} == 1 unless on(instance) node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"} + ''; + for = "0s"; + labels = warning; + }; + + alerts.NodeRequiresReboot = { + expr = "node_reboot_required > 0"; + for = "5m"; + labels = warning; + }; +} + diff --git a/hosts/vm/victoriametrics/rules/postgresql.nix b/hosts/vm/victoriametrics/rules/postgresql.nix new file mode 100644 index 0000000..249b8d0 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/postgresql.nix @@ -0,0 +1,180 @@ +{ critical, warning, ... }: + +{ + alerts.PostgresqlDown = { + expr = "pg_up == 0"; + for = "0m"; + labels = critical; + }; + + alerts.PostgresqlExporterError = { + expr = "pg_exporter_last_scrape_error > 0"; + for = "0m"; + labels = critical; + }; + + alerts.PostgresqlReplicationLag = { + expr = '' + pg_replication_lag > 30 + and on(instance) + pg_replication_is_replica == 1 + ''; + for = "0m"; + labels = critical; + annotations = { + Lag = "{{ $value }}"; + DB = "{{ $labels.datname }}"; + }; + }; + + alerts.PostgresqlTableNotVacuumed = { + expr = '' + time() - pg_stat_user_tables_last_autovacuum + > 60 * 60 * 24 + ''; + for = "0m"; + labels = warning; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlTableNotAnalyzed = { + expr = '' + time() - pg_stat_user_tables_last_autoanalyze + > 60 * 60 * 24 + ''; + for = "0m"; + labels = warning; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlTooManyConnections = { + expr = '' + ( + sum by (datname) + (pg_stat_activity_count{datname!~"template.*|postgres"}) + ) * 100 + > pg_settings_max_connections * 80 + ''; + for = "2m"; + labels = warning; + annotations = { + UsageLevel = "{{ $value | humanizePercentage }}"; + DB = "{{ $labels.datname }}"; + }; + }; + + alerts.PostgresqlDeadLocks = { + expr = '' + increase( + pg_stat_database_deadlocks{ + datname!~"template.*|postgres" + }[1m] + ) > 5 + ''; + for = "0m"; + labels = warning; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlSlowQueries = { + expr = "pg_slow_queries > 0"; + for = "2m"; + labels = warning; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlHighRollbackRate = { + expr = '' + ( + rate( + pg_stat_database_xact_rollback{datname!~"template.*"}[3m] + ) / rate( + pg_stat_database_xact_commit{datname!~"template.*"}[3m] + ) + ) > 0.2 + ''; + for = "0m"; + labels = warning; + annotations = { + RollbackRate = "{{ $value | humanizePercentage }}"; + DB = "{{ $labels.datname }}"; + }; + }; + + alerts.PostgresqlWalReplicationStopped = { + expr = "rate(pg_xlog_position_bytes[1m]) == 0"; + for = "0m"; + labels = critical; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlHighRateStatementTimeout = { + expr = '' + rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3 + ''; + for = "0m"; + labels = critical; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlHighRateDeadlock = { + expr = '' + increase( + postgresql_errors_total{type="deadlock_detected"}[1m] + ) > 1 + ''; + for = "0m"; + labels = critical; + annotations.DB = "{{ $labels.datname }}"; + }; + + alert.PostgresqlTooManyDeadTuples = { + expr = '' + ( + (pg_stat_user_tables_n_dead_tup > 10000) + / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup) + ) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1) + ''; + for = "2m"; + labels = warning; + annotations.DB = "{{ $labels.datname }}"; + }; + + alert.PostgresqlSplitBrain = { + expr = '' + count(pg_replication_is_replica == 0) != 1 + ''; + for = "0m"; + labels = critical; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlPromotedNode = { + expr = '' + pg_replication_is_replica + and + changes(pg_replication_is_replica[1m]) > 0 + ''; + for = "0m"; + labels = warning; + annotations.DB = "{{ $labels.datname }}"; + }; + + alerts.PostgresqlTooManyLocksAcquired = { + expr = '' + ( + (sum (pg_locks_count)) + / ( + pg_settings_max_locks_per_transaction + * pg_settings_max_connections + ) + ) * 100 > 20 + ''; + for = "2m"; + labels = critical; + annotations = { + DB = "{{ $labels.datname }}"; + Deadlocks = "{{ $value | humanizePercentage }}"; + }; + }; +} diff --git a/hosts/vm/victoriametrics/rules/switch.nix b/hosts/vm/victoriametrics/rules/switch.nix new file mode 100644 index 0000000..4cfcaab --- /dev/null +++ b/hosts/vm/victoriametrics/rules/switch.nix @@ -0,0 +1,81 @@ +{ critical, warning, ... }: + +{ + alerts.SwitchPromiscuousChange = { + expr = '' + changes(last_over_time(ifPromiscuousMode[1h])[5m]) > 0 + ''; + for = "0m"; + labels = warning; + annotations = { + Interface = "{{ $labels.ifName }}"; + Alias = "{{ $labels.ifAlias }}"; + }; + }; + + alerts.SwitchInErrors = { + expr = '' + rate(ifInErrors[15m]) / ( + rate(ifInUcastPkts[15m]) + + rate(ifInNUcastPkts[15m]) + + rate(ifInErrors[15m]) + ) > 0.0001 + ''; + for = "15m"; + labels = warning; + annotations = { + ErrorRate = "{{ $value | humanizePercentage }}"; + Interface = "{{ $labels.ifName }}"; + Alias = "{{ $labels.ifAlias }}"; + }; + }; + + alerts.SwitchOutErrors = { + expr = '' + rate(ifOutErrors[15m]) / ( + rate(ifOutUcastPkts[15m]) + + rate(ifOutNUcastPkts[15m]) + + rate(ifOutErrors[15m]) + ) > 0.0001 + ''; + for = "15m"; + labels = warning; + annotations = { + ErrorRate = "{{ $value | humanizePercentage }}"; + Interface = "{{ $labels.ifName }}"; + Alias = "{{ $labels.ifAlias }}"; + }; + }; + + alerts.SwitchInLinkUsage = { + expr = '' + rate(ifHCInOctets[5m]) / ( + (ifHighSpeed > 0) * 1000000 / 8 + ) > 0.8 + ''; + for = "5m"; + keep_firing_for = "10m"; + labels = warning; + annotations = { + Usage = "{{ $value | humanizePercentage }}"; + Interface = "{{ $labels.ifName }}"; + Alias = "{{ $labels.ifAlias }}"; + }; + }; + + alerts.SwitchOutLinkUsage = { + expr = '' + rate(ifHCOutOctets[5m]) / ( + (ifHighSpeed > 0) * 1000000 / 8 + ) > 0.8 + ''; + for = "5m"; + keep_firing_for = "10m"; + labels = warning; + annotations = { + Usage = "{{ $value | humanizePercentage }}"; + Interface = "{{ $labels.ifName }}"; + Alias = "{{ $labels.ifAlias }}"; + }; + }; +} diff --git a/hosts/vm/victoriametrics/rules/unbound.nix b/hosts/vm/victoriametrics/rules/unbound.nix new file mode 100644 index 0000000..9244681 --- /dev/null +++ b/hosts/vm/victoriametrics/rules/unbound.nix @@ -0,0 +1,11 @@ +{ critical, warning, ... }: + +{ + alerts.UnboundDown = { + expr = '' + (up{job="unbound"} unless unbound_up) == 0 + ''; + for = "0m"; + labels = critical; + }; +} diff --git a/hosts/vm/victoriametrics/scrape.nix b/hosts/vm/victoriametrics/scrape.nix new file mode 100644 index 0000000..0ac1dc7 --- /dev/null +++ b/hosts/vm/victoriametrics/scrape.nix @@ -0,0 +1,118 @@ +{ ... }: + +{ + + bird = { + replacementAddress = "$1:9324"; + targets = [ + #"infra-1.back.infra.auro.re" + #"infra-2.back.infra.auro.re" + #"edge-1.back.infra.auro.re" + #"edge-2.back.infra.auro.re" + ]; + }; + + ceph = { + replacementAddress = "$1:9283"; + targets = import ./targets/ceph.nix {}; + }; + + eaton = { + path = "/snmp"; + params.module = [ "eaton" "uptime" ]; + params.auth = [ "eaton" ]; + replacementAddress = "127.0.0.1:9116"; + replacementTarget = "$1"; + targets = import ./targets/eaton.nix {}; + }; + + ilo = { + path = "/snmp"; + params = { + module = [ + "uptime" + "ilo_memory" + "ilo_power" + "ilo_temp_fans" + ]; + auth = [ "ilo" ]; + }; + replacementAddress = "127.0.0.1:9116"; + # Some get-bulk requests are left unanswered when using IPv6, + # so we force IPv4 + replacementTarget = "udp4://$1"; + targets = import ./targets/ilo.nix {}; + }; + + jitsi = { + replacementAddress = "$1:9700"; + targets = [ "jitsi.adm.crans.org" ]; + }; + + #kea = { # pour quand on en aura un + # replacementAddress = "$1:9547"; + # targets = [ + # "dhcp-1.isp.infra.auro.re" + # "dhcp-2.isp.infra.auro.re" + # ]; + #}; + + node = { + replacementAddress = "$1:9100"; + targets = import ./targets/node.nix {}; + }; + + postgresql = { + replacementAddress = "$1:9187"; + targets = import ./targets/postgresql.nix {}; + }; + + #procurve = { + # path = "/snmp"; + # params = { + # module = [ "switch" "uptime" ]; + # auth = [ "procurve" ]; + # }; + # Procurve switches are cheap… + # interval = "2m"; + # timeout = "30s"; + # replacementAddress = "127.0.0.1:9116"; + # replacementTarget = "$1"; + # targets = import ./targets/procurve.nix {}; + #}; + + #quanta = { + # path = "/snmp"; + # params = { + # module = [ "switch" "uptime" "quanta" ]; + # auth = [ "quanta" ]; + # }; + # replacementAddress = "127.0.0.1:9116"; + # replacementTarget = "$1"; + # # Quanta switches are cheap… + # interval = "3m"; + # timeout = "45s"; + # targets = import ./targets/quanta.nix {}; + #}; + + #snmp-exporter = { # pour le jour où on aura deux victoriametrics pour se surveiller l'un l'autre + # replacementAddress = "$1:9116"; + # targets = [ + # "prometheus-1.monit.infra.auro.re" + # "prometheus-2.monit.infra.auro.re" + # ]; + #}; + + unbound = { + replacementAddress = "$1:9167"; + targets = import ./targets/unbound.nix {}; + }; + +# victoria-metrics = { # pour le jour où on aura deux victoriametrics pour se surveiller l'un l'autre +# replacementAddress = "$1:8428"; +# targets = [ +# "victoriametrics.adm.crans.org" +# # "prometheus-2.monit.infra.auro.re" +# ]; +# }; +} diff --git a/hosts/vm/victoriametrics/snmp.nix b/hosts/vm/victoriametrics/snmp.nix new file mode 100644 index 0000000..924eb48 --- /dev/null +++ b/hosts/vm/victoriametrics/snmp.nix @@ -0,0 +1,218 @@ +{ ... }: + +{ + auths = { + ilo = { + version = 3; + username = "root"; # A VERIFIER + security_level = "authPriv"; + auth_protocol = "SHA"; + password = "\${ILO_AUTH}"; + priv_protocol = "AES"; + priv_password = "\${ILO_PRIV}"; + }; + eaton = { + version = 1; + community = "prometheus"; + }; + ## Tenté, échoue sur 10.201.3.1 (celui dans le local serveurs de EDC). :( + #eaton = { + # version = 3; + # username = "prometheus"; + # security_level = "authPriv"; + # auth_protocol = "SHA"; + # password = "\${EATON_AUTH}"; + # priv_protocol = "AES"; + # priv_password = "\${EATON_PRIV}"; + #}; + }; + + modules.eaton = { + walk = [ + "xupsInput" + "xupsOutput" + "xupsBypass" + "xupsEnvironment" + "xupsBattery" + "xupsConfig" + ]; + lookups = [ + { + lookup = "xupsInputName"; + source_indexes = [ "xupsInputPhase" ]; + } + { + lookup = "xupsOutputName"; + source_indexes = [ "xupsOutputPhase" ]; + } + { + lookup = "xupsBypassName"; + source_indexes = [ "xupsBypassPhase" ]; + } + ]; + overrides = { + upsBatteryStatus.type = "EnumAsStateSet"; + xupsBatteryAbmStatus.type = "EnumAsStateSet"; + xupsBypassId.type = "EnumAsStateSet"; + xupsContactType.type = "EnumAsStateSet"; + xupsInputId.type = "EnumAsStateSet"; + xupsOutputId.type = "EnumAsStateSet"; + xupsOutputSource.type = "EnumAsStateSet"; + xupsBatteryFailure.type = "EnumAsStateSet"; + }; + }; + + modules.ilo_temp_fans = { + walk = [ + #"cpqHeThermalFanTable" + "cpqHeFltTolFanTable" + "cpqHeTemperatureTable" + #"cpqHeCoolingModuleTable" + ]; + timeout = "2s"; + retries = 10; + overrides = { + # cpqHeThermalFanTable + # cpqHeFltTolFanTable + cpqHeFltTolFanChassis.ignore = true; + cpqHeFltTolFanHotPlug.type = "EnumAsInfo"; + cpqHeFltTolFanIndex.ignore = true; + cpqHeFltTolFanLocale.type = "EnumAsInfo"; + cpqHeFltTolFanPresent.type = "EnumAsStateSet"; + cpqHeFltTolFanType.type = "EnumAsInfo"; + cpqHeFltTolFanSpeed.type = "EnumAsStateSet"; + cpqHeFltTolFanRedundant.type = "EnumAsStateSet"; + cpqHeFltTolFanCondition.type = "EnumAsStateSet"; + # cpqHeTemperatureTable + cpqHeTemperatureChassis.ignore = true; + cpqHeTemperatureIndex.ignore = true; + cpqHeTemperatureLocale.type = "EnumAsInfo"; + cpqHeTemperatureCondition.type = "EnumAsStateSet"; + cpqHeTemperatureThresholdType.type = "EnumAsInfo"; + }; + }; + + modules.ilo_power = { + walk = [ + "cpqHeFltTolPowerSupplyTable" + #"cpqHePowerMeter" + ]; + timeout = "2s"; + retries = 10; + overrides = { + # cpqHeFltTolPowerSupplyTable + cpqHeFltTolPowerSupplyChassis.ignore = true; + cpqHeFltTolPowerSupplyHotPlug.type = "EnumAsInfo"; + cpqHeFltTolPowerSupplyErrorCondition.type = "EnumAsStateSet"; + cpqHeFltTolPowerSupplyBay.ignore = true; + cpqHeFltTolPowerSupplyPresent.type = "EnumAsStateSet"; + cpqHeFltTolPowerSupplyCondition.type = "EnumAsStateSet"; + cpqHeFltTolPowerSupplyStatus.type = "EnumAsStateSet"; + cpqHeFltTolPowerSupplyRedundant.type = "EnumAsStateSet"; + }; + }; + + modules.ilo_memory = { + walk = [ + #"cpqHeCorrectableMemory" + #"cpqHePowerFailure" + #"cpqHeCriticalError" + "cpqHeResMem2BoardTable" + "cpqHeResMem2ModuleTable" + #"cpqHeHWBios" + #"cpqHeSysBackupBattery" + #"cpqHeResilientMemory" + #"cpqSm2Cntlr" + #"cpqHostOs" + #"cpqSeCpuTable" + #"cpqDaCntlr" + ]; + timeout = "2s"; + retries = 10; + overrides = { + #cpqHeResMem2BoardTable + cpqHeResMem2BoardCondition.type = "EnumAsStateSet"; + cpqHeResMem2BoardHotPlug.type = "EnumAsInfo"; + cpqHeResMem2BoardOnlineStatus.type = "EnumAsStateSet"; + cpqHeResMem2BoardErrorStatus.type = "EnumAsStateSet"; + cpqHeResMem2BoardLocked.type = "EnumAsStateSet"; + #cpqHeResMem2ModuleTable + cpqHeResMem2ModuleCellStatus.type = "EnumAsStateSet"; + cpqHeResMem2ModuleCondition.type = "EnumAsStateSet"; + cpqHeResMem2ModuleSmartMemory.type = "EnumAsInfo"; + cpqHeResMem2ModuleStatus.type = "EnumAsStateSet"; + cpqHeResMem2ModuleTechnology.type = "EnumAsInfo"; + cpqHeResMem2ModuleType.type = "EnumAsInfo"; + }; + }; + + modules.switch = { + walk = [ + "interfaces" + "ifXTable" + "dot3StatsTable" + "dot3HCStatsTable" + "dot3Errors" + "dot3Tests" + "dot3CollTable" + "lldpLocChassisId" + "lldpRemTable" + "lldpLocPortTable" + #"dot1dBasePort" + ]; + lookups = [ + { + lookup = "ifAlias"; + source_indexes = [ "ifIndex" ]; + } + { + lookup = "ifDescr"; + source_indexes = [ "ifIndex" ]; + } + { + lookup = "ifName"; + source_indexes = [ "ifIndex" ]; + } + { + lookup = "ifAlias"; + source_indexes = [ "dot3StatsIndex" ]; + } + { + lookup = "ifDescr"; + source_indexes = [ "dot3StatsIndex" ]; + } + { + lookup = "ifName"; + source_indexes = [ "dot3StatsIndex" ]; + } + { + lookup = "lldpRemChassisId"; + source_indexes = [ + "lldpRemTimeMark" + "lldpRemLocalPortNum" + "lldpRemIndex" + ]; + } + ]; + overrides = { + dot3StatsDuplexStatus.type = "EnumAsStateSet"; + dot3StatsEtherChipSet.ignore = true; + dot3StatsIndex.ignore = true; + ifAdminStatus.type = "EnumAsStateSet"; + ifAlias.ignore = true; + ifDescr.ignore = true; + ifIndex.ignore = true; + ifName.ignore = true; + ifOperStatus.type = "EnumAsStateSet"; + ifType.type = "EnumAsInfo"; + lldpLocPortIdSubtype.type = "EnumAsInfo"; + lldpRemPortIdSubtype.type = "EnumAsInfo"; + }; + timeout = "5s"; + retries = 10; + }; + + modules.uptime = { + walk = [ "sysUpTime" ]; + }; +} diff --git a/hosts/vm/victoriametrics/targets/ceph.nix b/hosts/vm/victoriametrics/targets/ceph.nix new file mode 100644 index 0000000..f0ad169 --- /dev/null +++ b/hosts/vm/victoriametrics/targets/ceph.nix @@ -0,0 +1,5 @@ +{ ... }: + +[ + "cephiroth.adm.crans.org" +] diff --git a/hosts/vm/victoriametrics/targets/ilo.nix b/hosts/vm/victoriametrics/targets/ilo.nix new file mode 100644 index 0000000..d3351ca --- /dev/null +++ b/hosts/vm/victoriametrics/targets/ilo.nix @@ -0,0 +1,13 @@ +{ ... }: + +[ + "ilo-cephiroth.adm.crans.org" + "ilo-daniel.adm.crans.org" + "ilo-gulp.adm.crans.org" + "ilo-jack.adm.crans.org" + "ilo-odlyd.adm.crans.org" + "ilo-sam.adm.crans.org" + "ilo-stitch.adm.crans.org" + "ilo-thot.adm.crans.org" + "ilo-zamok.adm.crans.org" +] diff --git a/hosts/vm/victoriametrics/targets/node.nix b/hosts/vm/victoriametrics/targets/node.nix new file mode 100644 index 0000000..788783c --- /dev/null +++ b/hosts/vm/victoriametrics/targets/node.nix @@ -0,0 +1,57 @@ +{ ... }: + +[ + "backup-ft.adm.crans.org" + "boeing.adm.crans.org" + "cameron.adm.crans.org" + "cas.adm.crans.org" + "collabora.adm.crans.org" + "daniel.adm.crans.org" + "ecilis.adm.crans.org" + "eclat.adm.crans.org" + "en7.adm.crans.org" + "ethercalc.adm.crans.org" + "flirt.adm.crans.org" + "ft.adm.crans.org" + "fyre.adm.crans.org" + "gitlab-ci.adm.crans.org" + "gitzly.adm.crans.org" + "gulp.adm.crans.org" + "helloworld.adm.crans.org" + "hodaur.adm.crans.org" + "irc.adm.crans.org" + "jack.adm.crans.org" + "jitsi.adm.crans.org" + "kenobi.adm.crans.org" + "kiwi.adm.crans.org" + "mailman.adm.crans.org" + "neo.adm.crans.org" + "neree.adm.crans.org" + "nextcloud.adm.crans.org" + "owl.adm.crans.org" + "owncloud.adm.crans.org" + "periodique.adm.crans.org" + "proxy-pve-adh.adm.crans.org" + "ptf.adm.crans.org" + "re2o.adm.crans.org" + "redisdead.adm.crans.org" + "reverseproxy.adm.crans.org" + "romanesco.adm.crans.org" + "roundcube.adm.crans.org" + "routeur-2754.adm.crans.org" + "routeur-daniel.adm.crans.org" + "routeur-ft.adm.crans.org" + "routeur-jack.adm.crans.org" + "routeur-sam.adm.crans.org" + "sam.adm.crans.org" + "silice.adm.crans.org" + "sputnik.adm.crans.org" + "stitch.adm.crans.org" + "tealc.adm.crans.org" + "thot.adm.crans.org" + "victoriametrics.adm.crans.org" + "voyager.adm.crans.org" + "wall-e.adm.crans.org" + "yson-partou.adm.crans.org" + "zamok.adm.crans.org" +] diff --git a/hosts/vm/victoriametrics/targets/postgresql.nix b/hosts/vm/victoriametrics/targets/postgresql.nix new file mode 100644 index 0000000..37a36fb --- /dev/null +++ b/hosts/vm/victoriametrics/targets/postgresql.nix @@ -0,0 +1,8 @@ +{ ... }: + +[ + "daniel.adm.crans.org" + "jack.adm.crans.org" + "sam.adm.crans.org" + "tealc.adm.crans.org" +] diff --git a/hosts/vm/victoriametrics/targets/unbound.nix b/hosts/vm/victoriametrics/targets/unbound.nix new file mode 100644 index 0000000..795bf27 --- /dev/null +++ b/hosts/vm/victoriametrics/targets/unbound.nix @@ -0,0 +1,5 @@ +{...}: +[ + "silice.adm.crans.org" + "sputnik.adm.crans.org" +] diff --git a/hosts/vm/victoriametrics/targets/ups.nix b/hosts/vm/victoriametrics/targets/ups.nix new file mode 100644 index 0000000..8947467 --- /dev/null +++ b/hosts/vm/victoriametrics/targets/ups.nix @@ -0,0 +1,5 @@ +{ ... }: + +[ + "excalibur.adm.crans.org" +] diff --git a/hosts/vm/victoriametrics/victoriametrics.nix b/hosts/vm/victoriametrics/victoriametrics.nix new file mode 100644 index 0000000..a686f4c --- /dev/null +++ b/hosts/vm/victoriametrics/victoriametrics.nix @@ -0,0 +1,103 @@ +{ config, lib, pkgs, ... }: + +let + mkScrapeConfig = name: config: let + relabelAddress = lib.optional (builtins.hasAttr "replacementAddress" config) { + source_labels = [ "__address__" ]; + target_label = "__address__"; + replacement = config.replacementAddress; + }; + relabelTarget = lib.optional (builtins.hasAttr "replacementTarget" config) { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + replacement = config.replacementTarget; + }; + in { + job_name = name; + metrics_path = config.path or "/metrics"; + static_configs = [ { targets = config.targets; } ]; + params = config.params or { }; + relabel_configs = [ + { source_labels = [ "__address__" ]; target_label = "instance"; } +# { source_labels = [ "__address__" ]; target_label = "__param_target"; } +# { source_labels = [ "__param_target" ]; target_label = "instance"; } + ] ++ relabelTarget ++ relabelAddress; + scrape_timeout = config.timeout or "15s"; + scrape_interval = config.interval or "1m"; + }; + mkScrapeConfigs = lib.attrsets.mapAttrsToList mkScrapeConfig; + critical = { severity = "critical"; }; + warning = { severity = "warning"; }; + mkRuleGroups = lib.attrsets.mapAttrsToList (name: path: { + inherit name; + rules = let + rules = import path { inherit critical warning; }; + alerts = lib.attrsets.mapAttrsToList + (alert: attrs: attrs // { inherit alert; }) + (rules.alerts or {}); + records = lib.attrsets.mapAttrsToList + (record: attrs: attrs // { inherit record; }) + (rules.records or {}); + in alerts ++ records; + }); +in +{ + age.secrets.snmp-env = { + file = ../../../secrets/victoriametrics/snmp.age; + owner = "root"; + group = "root"; + }; + + networking.firewall.allowedTCPPorts = [ 8428 9116 ]; + + services.victoriametrics = let + scrapeConfigs = mkScrapeConfigs (import ./scrape.nix {}); + in { + enable = true; + retentionPeriod = "30d"; + extraOptions = [ + "-enableTCP6" + # Compatibilité avec la datasource Prometheus de Grafana + "-vmalert.proxyURL=http://localhost:8880" + ]; + prometheusConfig.scrape_configs = scrapeConfigs; + }; + + services.vmalert.instances.default = { + enable = true; + rules.groups = mkRuleGroups (import ./rules {}); + settings = let + victoriametricsUrl = "http://localhost:8428"; + in { + "datasource.url" = victoriametricsUrl; + "remoteWrite.url" = victoriametricsUrl; + "remoteRead.url" = victoriametricsUrl; + "enableTCP6" = true; + #"notifier.url" = [ "http://docker-ovh.adm.auro.re:9093" ]; # A MODIFIER ##################################### + }; + }; + + services.prometheus.exporters.snmp = let + generator = import ./snmp.nix {}; + configSnmp = pkgs.runCommand "snmp.yml" { # Nous avons des doutes sur ce fichier? faut-il l'importer? + nativeBuildInputs = [ pkgs.prometheus-snmp-exporter ]; + generator = builtins.toJSON generator; + passAsFile = [ "generator" ]; + } '' + ${lib.getExe' pkgs.prometheus-snmp-exporter "generator"} \ + generate \ + --mibs-dir=${./mibs} \ + --generator-path=$generatorPath \ + --output-path=$out + ''; + in { + enable = true; + enableConfigCheck = false; + environmentFile = config.age.secrets.snmp-env.path; + configurationPath = configSnmp; + }; + + environment.systemPackages = with pkgs; [ + net-snmp + ]; +}