mirror of https://gitlab.crans.org/nounous/nixos
mise en place de la config de victoriametrics, on s'est fortement inspiré de ce qu'Aurore proposait
LapinPancake
parent
67eb99bb35
commit
c14721d7cb
|
|
@ -4,7 +4,8 @@
|
|||
{
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
];
|
||||
./victoriametrics.nix
|
||||
];
|
||||
|
||||
networking.hostName = "victoriametrics";
|
||||
boot.loader.grub.devices = [ "/dev/sda" ];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,163 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
records."bird:protocol_up:bgp_all".expr = ''
|
||||
label_replace(
|
||||
bird_protocol_up{proto="BGP"},
|
||||
"group", "$1",
|
||||
"instance", "^([^0-9\\.]+)-[0-9]+.*"
|
||||
)
|
||||
'';
|
||||
|
||||
# FIXME: sessions en cours d'installation, pas encore monitorées
|
||||
records."bird:protocol_up:bgp".expr = ''
|
||||
bird:protocol_up:bgp_all
|
||||
unless
|
||||
bird:protocol_up:bgp_all{
|
||||
group="edge",
|
||||
name=~"^(viarezo|isp[12]|rezel)[46]$"
|
||||
}
|
||||
'';
|
||||
|
||||
# Sessions qui ne sont volontairement pas redondées
|
||||
# au sein d'un groupe
|
||||
records."bird:protocol_up:bgp:non_redundant".expr = ''
|
||||
bird:protocol_up:bgp{
|
||||
group="edge",
|
||||
name=~"^(oti|crans|legacy|edge)[46]$"
|
||||
}
|
||||
'';
|
||||
|
||||
# Sessions qui le sont
|
||||
records."bird:protocol_up:bgp:redundant".expr = ''
|
||||
bird:protocol_up:bgp
|
||||
unless
|
||||
bird:protocol_up:bgp:non_redundant
|
||||
'';
|
||||
|
||||
alerts.BirdBGPRedundancyDegraded = {
|
||||
expr = ''
|
||||
(
|
||||
count by (group, name) (
|
||||
bird:protocol_up:bgp:redundant{state="Established"}
|
||||
) + (
|
||||
count by (group, name) (
|
||||
bird:protocol_up:bgp:redundant{state!="Established"} * 0
|
||||
)
|
||||
)
|
||||
) < 2
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Session = "{{ $labels.name }}";
|
||||
Count = "{{ $value }}";
|
||||
Group = "{{ $labels.group }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.BirdBGPDown = {
|
||||
expr = ''
|
||||
(
|
||||
count by (group, name) (
|
||||
bird:protocol_up:bgp{state="Established"}
|
||||
) + (
|
||||
count by (group, name) (
|
||||
bird:protocol_up:bgp{state!="Established"} * 0
|
||||
)
|
||||
)
|
||||
) == 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Session = "{{ $labels.name }}";
|
||||
Group = "{{ $labels.group }}";
|
||||
};
|
||||
};
|
||||
|
||||
# TODO: warning pour redondant ?
|
||||
alerts.BirdBGPNoExportedPrefixRedundant = {
|
||||
expr = ''
|
||||
bird_protocol_prefix_export_count{
|
||||
export_filter!="REJECT",
|
||||
} * on (instance, name) group_left (group) (
|
||||
bird:protocol_up:bgp:redundant{state="Established"}
|
||||
) == 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Session = "{{ $labels.name }}";
|
||||
Group = "{{ $labels.group }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.BirdBGPNoImportedPrefixRedundant = {
|
||||
expr = ''
|
||||
bird_protocol_prefix_import_count{
|
||||
import_filter!="REJECT",
|
||||
} * on (instance, name) group_left (group) (
|
||||
bird:protocol_up:bgp:redundant{state="Established"}
|
||||
) == 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Session = "{{ $labels.name }}";
|
||||
Group = "{{ $labels.group }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.BirdBGPNoExportedPrefixNonRedundant = {
|
||||
expr = ''
|
||||
sum by (group) (
|
||||
bird_protocol_prefix_export_count{
|
||||
export_filter!="REJECT",
|
||||
} * on (instance, name) group_left (group) (
|
||||
bird:protocol_up:bgp:non_redundant{state="Established"}
|
||||
)
|
||||
) == 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Session = "{{ $labels.name }}";
|
||||
Group = "{{ $labels.group }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.BirdBGPNoImportedPrefixNonRedundant = {
|
||||
expr = ''
|
||||
sum by (group) (
|
||||
bird_protocol_prefix_import_count{
|
||||
import_filter!="REJECT",
|
||||
} * on (instance, name) group_left (group) (
|
||||
bird:protocol_up:bgp:non_redundant{state="Established"}
|
||||
)
|
||||
) == 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Session = "{{ $labels.name }}";
|
||||
Group = "{{ $labels.group }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.BirdOSPFNeighboursChange = {
|
||||
expr = ''
|
||||
changes(bird_ospf_neighbor_count[5m]) > 0
|
||||
or changes(bird_ospfv3_neighbor_count[5m]) > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
};
|
||||
|
||||
alerts.BirdOSPFDown = {
|
||||
expr = "bird_ospf_running == 0";
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations.Instance = "{{ $labels.name }}";
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
alerts.CephErrorState = {
|
||||
expr = ''
|
||||
ceph_health_status > 1
|
||||
'';
|
||||
for = "5m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.CephMonitorAvailableStorageCritical = {
|
||||
expr = ''
|
||||
ceph_monitor_avail_percent < 15
|
||||
'';
|
||||
for = "1m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.CephMonitorAvailableStorageWarning = {
|
||||
expr = ''
|
||||
ceph_monitor_avail_percent < 30
|
||||
'';
|
||||
for = "1m";
|
||||
labels = warning;
|
||||
};
|
||||
|
||||
alerts.CephMonitorClockSkewTooHigh = {
|
||||
expr = ''
|
||||
abs(ceph_monitor_clock_skew_seconds) > 0.1
|
||||
'';
|
||||
for = "1m";
|
||||
labels = warning;
|
||||
};
|
||||
|
||||
alerts.CephOSDApplyLatencyTooHigh = {
|
||||
expr = ''
|
||||
ceph_osd_perf_apply_latency_seconds > 10
|
||||
'';
|
||||
for = "1m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Latency = "{{ $value }}";
|
||||
Device = "{{ $labels.device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.CephOSDDown = {
|
||||
expr = ''
|
||||
ceph_osd_up == 0
|
||||
'';
|
||||
for = "1m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.CephOSDReweighted = {
|
||||
expr = ''
|
||||
ceph_osd_weight < 1
|
||||
'';
|
||||
for = "1h";
|
||||
labels = warning;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephOSDUtilization = {
|
||||
expr = ''
|
||||
ceph_osd_utilization > 90
|
||||
'';
|
||||
for = "1m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.CephPgActivating = {
|
||||
expr = ''
|
||||
ceph_pg_activating > 0
|
||||
'';
|
||||
for = "5m";
|
||||
labels = critical;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephPgDown = {
|
||||
expr = ''
|
||||
ceph_pg_down > 0
|
||||
'';
|
||||
for = "3m";
|
||||
labels = critical;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephPgIncomplete = {
|
||||
expr = ''
|
||||
ceph_pg_incomplete > 0
|
||||
'';
|
||||
for = "2m";
|
||||
labels = critical;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephPgInconsistent = {
|
||||
expr = ''
|
||||
ceph_pg_inconsistent > 0
|
||||
'';
|
||||
for = "1m";
|
||||
labels = warning;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephPgBackfillTooFull = {
|
||||
expr = ''
|
||||
ceph_pg_backfill_toofull > 0
|
||||
'';
|
||||
for = "5m";
|
||||
labels = warning;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephPgUnavailable = {
|
||||
expr = ''
|
||||
ceph_pg_total - ceph_pg_active > 0
|
||||
'';
|
||||
for = "5m";
|
||||
labels = critical;
|
||||
annotations.Instance = "{{ $labels.cluster }}";
|
||||
annotations.Database = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.CephTargetDown = {
|
||||
expr = ''
|
||||
up{job="ceph"} == 0
|
||||
'';
|
||||
for = "10m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.CephWarnState = {
|
||||
expr = ''
|
||||
ceph_health_status == 1
|
||||
'';
|
||||
for = "30m";
|
||||
labels = warning;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{ ... }:
|
||||
|
||||
{
|
||||
bird = ./bird.nix;
|
||||
ceph = ./ceph.nix;
|
||||
eaton = ./eaton.nix;
|
||||
ilo = ./ilo.nix;
|
||||
jitsi = ./jitsi.nix;
|
||||
node = ./node.nix;
|
||||
postgresql = ./postgresql.nix;
|
||||
switch = ./switch.nix;
|
||||
unbound = ./unbound.nix;
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
alerts.EatonBatteryFailure = {
|
||||
expr = ''
|
||||
xupsBatteryFailure{xupsBatteryFailure="yes"} == 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.EatonBatteryLow = {
|
||||
expr = ''
|
||||
xupsBatTimeRemaining < 1800
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
# cpqHeFltTolFanTable
|
||||
|
||||
alerts.IloFanDegraded = {
|
||||
expr = ''
|
||||
cpqHeFltTolFanCondition{
|
||||
cpqHeFltTolFanCondition="degraded"
|
||||
} == 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Chassis = "{{ $labels.cpqHeFltTolFanChassis }}";
|
||||
Index = "{{ $labels.cpqHeFltTolFanIndex }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.IloFanFailed = {
|
||||
expr = ''
|
||||
cpqHeFltTolFanCondition{
|
||||
cpqHeFltTolFanCondition="failed"
|
||||
} == 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Chassis = "{{ $labels.cpqHeFltTolFanChassis }}";
|
||||
Index = "{{ $labels.cpqHeFltTolFanIndex }}";
|
||||
};
|
||||
};
|
||||
|
||||
# cpqHeFltTolPowerSupplyTable
|
||||
|
||||
alerts.IloHighPSUUsage = {
|
||||
expr = ''
|
||||
cpqHeFltTolPowerSupplyCapacityUsed
|
||||
/ (cpqHeFltTolPowerSupplyCapacityMaximum > 0) > 0.5
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Usage = "{{ $value | humanizePercentage }}";
|
||||
Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}";
|
||||
Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.IloPSUDegraded = {
|
||||
expr = ''
|
||||
cpqHeFltTolPowerSupplyCondition{
|
||||
cpqHeFltTolPowerSupplyCondition="degraded"
|
||||
} == 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}";
|
||||
Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.IloPSUFailed = {
|
||||
expr = ''
|
||||
(
|
||||
cpqHeFltTolPowerSupplyCondition{
|
||||
cpqHeFltTolPowerSupplyCondition="failed"
|
||||
} == 1
|
||||
) * on (
|
||||
instance,
|
||||
cpqHeFltTolPowerSupplyChassis,
|
||||
cpqHeFltTolPowerSupplyBay
|
||||
) group_left (
|
||||
cpqHeFltTolPowerSupplyErrorCondition
|
||||
)
|
||||
(cpqHeFltTolPowerSupplyErrorCondition == 1)
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Chassis = "{{ $labels.cpqHeFltTolPowerSupplyChassis }}";
|
||||
Bay = "{{ $labels.cpqHeFltTolPowerSupplyBay }}";
|
||||
Error = "{{ $labels.cpqHeFltTolPowerSupplyErrorCondition }}";
|
||||
};
|
||||
};
|
||||
|
||||
# cpqHeTemperatureTable
|
||||
|
||||
alerts.IloTooHot = {
|
||||
expr = ''
|
||||
(
|
||||
cpqHeTemperatureCondition{
|
||||
cpqHeTemperatureCondition!="ok"
|
||||
} == 1
|
||||
) * on (
|
||||
instance,
|
||||
cpqHeTemperatureChassis,
|
||||
cpqHeTemperatureIndex
|
||||
) group_left (cpqHeTemperatureLocale)
|
||||
cpqHeTemperatureLocale_info
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Chassis = "{{ $labels.cpqHeTemperatureChassis }}";
|
||||
Index = "{{ $labels.cpqHeTemperatureIndex }}";
|
||||
Locale = "{{ $labels.cpqHeTemperatureLocale }}";
|
||||
Condition = "{{ $labels.cpqHeTemperatureCondition }}";
|
||||
};
|
||||
};
|
||||
|
||||
# cpqHeResMem2ModuleTable
|
||||
|
||||
alerts.IloRamHS = {
|
||||
expr = ''
|
||||
cpqHeResMem2ModuleStatus{
|
||||
cpqHeResMem2ModuleStatus!~"good|notPresent"
|
||||
} == 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Module = "{{ $labels.cpqHeResMem2Module }}";
|
||||
Status = "{{ $labels.cpqHeResMem2ModuleStatus }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.IloUneSeuleRam = {
|
||||
expr = ''
|
||||
count by (instance) (
|
||||
cpqHeResMem2ModuleStatus{
|
||||
cpqHeResMem2ModuleStatus="good"
|
||||
} == 1
|
||||
) <= 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{ critical, ... }:
|
||||
|
||||
{
|
||||
alerts.JitsiHighStressLevel = {
|
||||
expr = "jitsi_stress_level > 0.7";
|
||||
for = "3m";
|
||||
labels = critical;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,318 @@
|
|||
{ critical, warning, ... }:
|
||||
{
|
||||
# Memory
|
||||
|
||||
alerts.NodeOutOfMemory = {
|
||||
expr = ''
|
||||
(
|
||||
(
|
||||
node_memory_MemAvailable_bytes
|
||||
+ (
|
||||
clamp_min(node_zfs_arc_size - node_zfs_arc_c_min, 0)
|
||||
or node_memory_MemAvailable_bytes * 0
|
||||
)
|
||||
)
|
||||
/ node_memory_MemTotal_bytes
|
||||
) < 0.1
|
||||
'';
|
||||
for = "3m";
|
||||
labels = critical;
|
||||
annotations.Available = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
|
||||
alerts.NodeUnderMemoryPressure = {
|
||||
expr = "rate(node_vmstat_pgmajfault[5m]) > 1000";
|
||||
labels = critical;
|
||||
for = "0m";
|
||||
annotations.Pressure = "{{ $value | humanize }}";
|
||||
};
|
||||
|
||||
alerts.NodeSwapIsFillingUp = {
|
||||
expr = ''
|
||||
(1 - (node_memory_SwapFree_bytes
|
||||
/ node_memory_SwapTotal_bytes)) > 0.75
|
||||
'';
|
||||
for = "1m";
|
||||
labels = critical;
|
||||
annotations.UsedSwap = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
|
||||
alerts.NodeOomKillDetected = {
|
||||
expr = "increase(node_vmstat_oom_kill[1m]) > 0";
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
# CPU
|
||||
|
||||
alerts.NodeCpuUsage = {
|
||||
expr = ''
|
||||
(avg by (instance)
|
||||
(rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8
|
||||
'';
|
||||
for = "10m";
|
||||
labels = warning;
|
||||
annotations.AverageUsage = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
|
||||
alerts.NodeCpuStealNoisyNeighbor = {
|
||||
expr = ''
|
||||
avg by (instance) (
|
||||
rate(node_cpu_seconds_total{mode="steal"}[2m])
|
||||
) > 0.1
|
||||
'';
|
||||
for = "10m";
|
||||
labels = warning;
|
||||
annotations.Steal = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
|
||||
# Network
|
||||
|
||||
alerts.NodeLinkHighUsageIn = {
|
||||
expr = ''
|
||||
(rate(node_network_receive_bytes_total[5m])
|
||||
/ on(instance, device) node_network_speed_bytes) > .80
|
||||
'';
|
||||
labels = warning;
|
||||
for = "3m";
|
||||
annotations = {
|
||||
Usage = "{{ $value | humanizePercentage }}";
|
||||
Device = "{{ $labels.device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeLinkHighUsageOut = {
|
||||
expr = ''
|
||||
(rate(node_network_transmit_bytes_total[5m])
|
||||
/ on(instance, device) node_network_speed_bytes) > .80
|
||||
'';
|
||||
labels = warning;
|
||||
for = "3m";
|
||||
annotations = {
|
||||
Usage = "{{ $value | humanizePercentage }}";
|
||||
Device = "{{ $labels.device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeConntrackLimit = {
|
||||
expr = ''
|
||||
(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.8
|
||||
'';
|
||||
for = "5m";
|
||||
labels = warning;
|
||||
annotations.Filled = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
|
||||
alerts.NodeNetworkReceiveErrors = {
|
||||
expr = ''
|
||||
rate(node_network_receive_errs_total[2m])
|
||||
/ rate(node_network_receive_packets_total[2m]) > 0.01
|
||||
'';
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Errors = "{{ $value | humanizePercentage }}";
|
||||
Device = "{{ $labels.device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeNetworkTransmitErrors = {
|
||||
expr = ''
|
||||
rate(node_network_transmit_errs_total[2m])
|
||||
/ rate(node_network_transmit_packets_total[2m]) > 0.01
|
||||
'';
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Errors = "{{ $value | humanizePercentage }}";
|
||||
Device = "{{ $labels.device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeNetworkBondDegraded = {
|
||||
expr = "node_bonding_active - node_bonding_slaves != 0";
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
annotations.Device = "{{ $labels.device }}";
|
||||
};
|
||||
|
||||
# Temperature
|
||||
|
||||
alerts.NodePhysicalComponentTooHot = {
|
||||
expr = ''
|
||||
node_hwmon_temp_celsius > clamp_max(node_hwmon_temp_max_celsius, 79)
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Temperature = "{{ $value | humanize }} °C";
|
||||
Chip = "{{ $labels.chip }}";
|
||||
Sensor = "{{ $labels.sensor }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeNodeOvertemperatureAlarm = {
|
||||
expr = "node_hwmon_temp_crit_alarm_celsius == 1";
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Chip = "{{ $labels.chip }}";
|
||||
Sensor = "{{ $labels.sensor }}";
|
||||
};
|
||||
};
|
||||
|
||||
# Storage and disks
|
||||
|
||||
alerts.NodeRaidArrayGotInactive = {
|
||||
expr = ''
|
||||
node_md_state{state="inactive"} > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Device = "{{ $labels.device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeRaidDiskFailure = {
|
||||
expr = ''
|
||||
node_md_disks{state="failed"} > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Device = "{{ $labels.md_device }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeOutOfDiskSpace = {
|
||||
expr = ''
|
||||
(node_filesystem_free_bytes / node_filesystem_size_bytes < 0.1)
|
||||
and on (instance, device, mountpoint) (node_filesystem_readonly) == 0
|
||||
'';
|
||||
for = "5m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Mountpoint = "{{ $labels.mountpoint }}";
|
||||
FreeSpace = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeOutOfInodes = {
|
||||
expr = "node_filesystem_files_free / node_filesystem_files < 0.1";
|
||||
for = "3m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Mountpoint = "{{ $labels.mountpoint }}";
|
||||
FreeInodes = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.NodeUnhealthyDisk = {
|
||||
expr = "smartmon_device_smart_healthy < 1";
|
||||
for = "10m";
|
||||
labels = critical;
|
||||
annotations.Disk = "{{ $labels.disk }}";
|
||||
};
|
||||
|
||||
alerts.NodeZfsWrongState = {
|
||||
expr = ''
|
||||
node_zfs_zpool_state{state!="online"} > 0
|
||||
'';
|
||||
for = "5m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
State = "{{ $labels.state }}";
|
||||
ZPool = "{{ $labels.zpool }}";
|
||||
};
|
||||
};
|
||||
|
||||
# Clock
|
||||
|
||||
alerts.NodeClockSkew = {
|
||||
expr = ''
|
||||
(node_timex_offset_seconds > 0.05
|
||||
and deriv(node_timex_offset_seconds[5m]) >= 0)
|
||||
or (node_timex_offset_seconds < -0.05
|
||||
and deriv(node_timex_offset_seconds[5m]) <= 0)
|
||||
'';
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
};
|
||||
|
||||
alerts.NodeClockNotSynchronising = {
|
||||
expr = ''
|
||||
min_over_time(node_timex_sync_status[1m]) == 0
|
||||
and node_timex_maxerror_seconds >= 16
|
||||
'';
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
};
|
||||
|
||||
# EDAC
|
||||
|
||||
alerts.NodeEdacCorrectableErrorsDetected = {
|
||||
expr = ''
|
||||
increase(node_edac_correctable_errors_total[1m]) > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations.CorrectedErrors = "{{ $value }}";
|
||||
};
|
||||
|
||||
alerts.NodeEdacUncorrectableErrorsDetected = {
|
||||
expr = ''
|
||||
increase(node_edac_uncorrectable_errors_total[1m]) > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations.DetectedErrors = "{{ $value }}";
|
||||
};
|
||||
|
||||
# Misc
|
||||
|
||||
alerts.NodeLoad5Usage = {
|
||||
expr = ''
|
||||
node_load5 / (
|
||||
count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.5
|
||||
'';
|
||||
for = "1m";
|
||||
labels = warning;
|
||||
annotations.Load5PerCore = "{{ $value | humanize }}";
|
||||
};
|
||||
|
||||
alerts.NodeSystemdUnitFailed = {
|
||||
expr = ''
|
||||
node_systemd_unit_state{state="failed"} == 1
|
||||
'';
|
||||
for = "5m";
|
||||
labels = warning;
|
||||
annotations.Unit = "{{ $labels.name }}";
|
||||
};
|
||||
|
||||
alerts.NodeBorgmaticLastTooOld = {
|
||||
expr = ''
|
||||
(time()
|
||||
- (node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"}
|
||||
) > 2 * 24 * 60 * 60)
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations.Last = "{{ $value | humanizeDuration }}";
|
||||
};
|
||||
|
||||
alerts.NodeBorgmaticAbsent = {
|
||||
expr = ''
|
||||
up{job="node"} == 1 unless on(instance) node_systemd_timer_last_trigger_seconds{name="borgmatic.timer"}
|
||||
'';
|
||||
for = "0s";
|
||||
labels = warning;
|
||||
};
|
||||
|
||||
alerts.NodeRequiresReboot = {
|
||||
expr = "node_reboot_required > 0";
|
||||
for = "5m";
|
||||
labels = warning;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
alerts.PostgresqlDown = {
|
||||
expr = "pg_up == 0";
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.PostgresqlExporterError = {
|
||||
expr = "pg_exporter_last_scrape_error > 0";
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
|
||||
alerts.PostgresqlReplicationLag = {
|
||||
expr = ''
|
||||
pg_replication_lag > 30
|
||||
and on(instance)
|
||||
pg_replication_is_replica == 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
Lag = "{{ $value }}";
|
||||
DB = "{{ $labels.datname }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.PostgresqlTableNotVacuumed = {
|
||||
expr = ''
|
||||
time() - pg_stat_user_tables_last_autovacuum
|
||||
> 60 * 60 * 24
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlTableNotAnalyzed = {
|
||||
expr = ''
|
||||
time() - pg_stat_user_tables_last_autoanalyze
|
||||
> 60 * 60 * 24
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlTooManyConnections = {
|
||||
expr = ''
|
||||
(
|
||||
sum by (datname)
|
||||
(pg_stat_activity_count{datname!~"template.*|postgres"})
|
||||
) * 100
|
||||
> pg_settings_max_connections * 80
|
||||
'';
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
UsageLevel = "{{ $value | humanizePercentage }}";
|
||||
DB = "{{ $labels.datname }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.PostgresqlDeadLocks = {
|
||||
expr = ''
|
||||
increase(
|
||||
pg_stat_database_deadlocks{
|
||||
datname!~"template.*|postgres"
|
||||
}[1m]
|
||||
) > 5
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlSlowQueries = {
|
||||
expr = "pg_slow_queries > 0";
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlHighRollbackRate = {
|
||||
expr = ''
|
||||
(
|
||||
rate(
|
||||
pg_stat_database_xact_rollback{datname!~"template.*"}[3m]
|
||||
) / rate(
|
||||
pg_stat_database_xact_commit{datname!~"template.*"}[3m]
|
||||
)
|
||||
) > 0.2
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
RollbackRate = "{{ $value | humanizePercentage }}";
|
||||
DB = "{{ $labels.datname }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.PostgresqlWalReplicationStopped = {
|
||||
expr = "rate(pg_xlog_position_bytes[1m]) == 0";
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlHighRateStatementTimeout = {
|
||||
expr = ''
|
||||
rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlHighRateDeadlock = {
|
||||
expr = ''
|
||||
increase(
|
||||
postgresql_errors_total{type="deadlock_detected"}[1m]
|
||||
) > 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alert.PostgresqlTooManyDeadTuples = {
|
||||
expr = ''
|
||||
(
|
||||
(pg_stat_user_tables_n_dead_tup > 10000)
|
||||
/ (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)
|
||||
) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)
|
||||
'';
|
||||
for = "2m";
|
||||
labels = warning;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alert.PostgresqlSplitBrain = {
|
||||
expr = ''
|
||||
count(pg_replication_is_replica == 0) != 1
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlPromotedNode = {
|
||||
expr = ''
|
||||
pg_replication_is_replica
|
||||
and
|
||||
changes(pg_replication_is_replica[1m]) > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations.DB = "{{ $labels.datname }}";
|
||||
};
|
||||
|
||||
alerts.PostgresqlTooManyLocksAcquired = {
|
||||
expr = ''
|
||||
(
|
||||
(sum (pg_locks_count))
|
||||
/ (
|
||||
pg_settings_max_locks_per_transaction
|
||||
* pg_settings_max_connections
|
||||
)
|
||||
) * 100 > 20
|
||||
'';
|
||||
for = "2m";
|
||||
labels = critical;
|
||||
annotations = {
|
||||
DB = "{{ $labels.datname }}";
|
||||
Deadlocks = "{{ $value | humanizePercentage }}";
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
alerts.SwitchPromiscuousChange = {
|
||||
expr = ''
|
||||
changes(last_over_time(ifPromiscuousMode[1h])[5m]) > 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Interface = "{{ $labels.ifName }}";
|
||||
Alias = "{{ $labels.ifAlias }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.SwitchInErrors = {
|
||||
expr = ''
|
||||
rate(ifInErrors[15m]) / (
|
||||
rate(ifInUcastPkts[15m])
|
||||
+ rate(ifInNUcastPkts[15m])
|
||||
+ rate(ifInErrors[15m])
|
||||
) > 0.0001
|
||||
'';
|
||||
for = "15m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
ErrorRate = "{{ $value | humanizePercentage }}";
|
||||
Interface = "{{ $labels.ifName }}";
|
||||
Alias = "{{ $labels.ifAlias }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.SwitchOutErrors = {
|
||||
expr = ''
|
||||
rate(ifOutErrors[15m]) / (
|
||||
rate(ifOutUcastPkts[15m])
|
||||
+ rate(ifOutNUcastPkts[15m])
|
||||
+ rate(ifOutErrors[15m])
|
||||
) > 0.0001
|
||||
'';
|
||||
for = "15m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
ErrorRate = "{{ $value | humanizePercentage }}";
|
||||
Interface = "{{ $labels.ifName }}";
|
||||
Alias = "{{ $labels.ifAlias }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.SwitchInLinkUsage = {
|
||||
expr = ''
|
||||
rate(ifHCInOctets[5m]) / (
|
||||
(ifHighSpeed > 0) * 1000000 / 8
|
||||
) > 0.8
|
||||
'';
|
||||
for = "5m";
|
||||
keep_firing_for = "10m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Usage = "{{ $value | humanizePercentage }}";
|
||||
Interface = "{{ $labels.ifName }}";
|
||||
Alias = "{{ $labels.ifAlias }}";
|
||||
};
|
||||
};
|
||||
|
||||
alerts.SwitchOutLinkUsage = {
|
||||
expr = ''
|
||||
rate(ifHCOutOctets[5m]) / (
|
||||
(ifHighSpeed > 0) * 1000000 / 8
|
||||
) > 0.8
|
||||
'';
|
||||
for = "5m";
|
||||
keep_firing_for = "10m";
|
||||
labels = warning;
|
||||
annotations = {
|
||||
Usage = "{{ $value | humanizePercentage }}";
|
||||
Interface = "{{ $labels.ifName }}";
|
||||
Alias = "{{ $labels.ifAlias }}";
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{ critical, warning, ... }:
|
||||
|
||||
{
|
||||
alerts.UnboundDown = {
|
||||
expr = ''
|
||||
(up{job="unbound"} unless unbound_up) == 0
|
||||
'';
|
||||
for = "0m";
|
||||
labels = critical;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
{ ... }:
|
||||
|
||||
{
|
||||
|
||||
bird = {
|
||||
replacementAddress = "$1:9324";
|
||||
targets = [
|
||||
#"infra-1.back.infra.auro.re"
|
||||
#"infra-2.back.infra.auro.re"
|
||||
#"edge-1.back.infra.auro.re"
|
||||
#"edge-2.back.infra.auro.re"
|
||||
];
|
||||
};
|
||||
|
||||
ceph = {
|
||||
replacementAddress = "$1:9283";
|
||||
targets = import ./targets/ceph.nix {};
|
||||
};
|
||||
|
||||
eaton = {
|
||||
path = "/snmp";
|
||||
params.module = [ "eaton" "uptime" ];
|
||||
params.auth = [ "eaton" ];
|
||||
replacementAddress = "127.0.0.1:9116";
|
||||
replacementTarget = "$1";
|
||||
targets = import ./targets/eaton.nix {};
|
||||
};
|
||||
|
||||
ilo = {
|
||||
path = "/snmp";
|
||||
params = {
|
||||
module = [
|
||||
"uptime"
|
||||
"ilo_memory"
|
||||
"ilo_power"
|
||||
"ilo_temp_fans"
|
||||
];
|
||||
auth = [ "ilo" ];
|
||||
};
|
||||
replacementAddress = "127.0.0.1:9116";
|
||||
# Some get-bulk requests are left unanswered when using IPv6,
|
||||
# so we force IPv4
|
||||
replacementTarget = "udp4://$1";
|
||||
targets = import ./targets/ilo.nix {};
|
||||
};
|
||||
|
||||
jitsi = {
|
||||
replacementAddress = "$1:9700";
|
||||
targets = [ "jitsi.adm.crans.org" ];
|
||||
};
|
||||
|
||||
#kea = { # pour quand on en aura un
|
||||
# replacementAddress = "$1:9547";
|
||||
# targets = [
|
||||
# "dhcp-1.isp.infra.auro.re"
|
||||
# "dhcp-2.isp.infra.auro.re"
|
||||
# ];
|
||||
#};
|
||||
|
||||
node = {
|
||||
replacementAddress = "$1:9100";
|
||||
targets = import ./targets/node.nix {};
|
||||
};
|
||||
|
||||
postgresql = {
|
||||
replacementAddress = "$1:9187";
|
||||
targets = import ./targets/postgresql.nix {};
|
||||
};
|
||||
|
||||
#procurve = {
|
||||
# path = "/snmp";
|
||||
# params = {
|
||||
# module = [ "switch" "uptime" ];
|
||||
# auth = [ "procurve" ];
|
||||
# };
|
||||
# Procurve switches are cheap…
|
||||
# interval = "2m";
|
||||
# timeout = "30s";
|
||||
# replacementAddress = "127.0.0.1:9116";
|
||||
# replacementTarget = "$1";
|
||||
# targets = import ./targets/procurve.nix {};
|
||||
#};
|
||||
|
||||
#quanta = {
|
||||
# path = "/snmp";
|
||||
# params = {
|
||||
# module = [ "switch" "uptime" "quanta" ];
|
||||
# auth = [ "quanta" ];
|
||||
# };
|
||||
# replacementAddress = "127.0.0.1:9116";
|
||||
# replacementTarget = "$1";
|
||||
# # Quanta switches are cheap…
|
||||
# interval = "3m";
|
||||
# timeout = "45s";
|
||||
# targets = import ./targets/quanta.nix {};
|
||||
#};
|
||||
|
||||
#snmp-exporter = { # pour le jour où on aura deux victoriametrics pour se surveiller l'un l'autre
|
||||
# replacementAddress = "$1:9116";
|
||||
# targets = [
|
||||
# "prometheus-1.monit.infra.auro.re"
|
||||
# "prometheus-2.monit.infra.auro.re"
|
||||
# ];
|
||||
#};
|
||||
|
||||
unbound = {
|
||||
replacementAddress = "$1:9167";
|
||||
targets = import ./targets/unbound.nix {};
|
||||
};
|
||||
|
||||
# victoria-metrics = { # pour le jour où on aura deux victoriametrics pour se surveiller l'un l'autre
|
||||
# replacementAddress = "$1:8428";
|
||||
# targets = [
|
||||
# "victoriametrics.adm.crans.org"
|
||||
# # "prometheus-2.monit.infra.auro.re"
|
||||
# ];
|
||||
# };
|
||||
}
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
{ ... }:
|
||||
|
||||
{
|
||||
auths = {
|
||||
ilo = {
|
||||
version = 3;
|
||||
username = "root"; # A VERIFIER
|
||||
security_level = "authPriv";
|
||||
auth_protocol = "SHA";
|
||||
password = "\${ILO_AUTH}";
|
||||
priv_protocol = "AES";
|
||||
priv_password = "\${ILO_PRIV}";
|
||||
};
|
||||
eaton = {
|
||||
version = 1;
|
||||
community = "prometheus";
|
||||
};
|
||||
## Tenté, échoue sur 10.201.3.1 (celui dans le local serveurs de EDC). :(
|
||||
#eaton = {
|
||||
# version = 3;
|
||||
# username = "prometheus";
|
||||
# security_level = "authPriv";
|
||||
# auth_protocol = "SHA";
|
||||
# password = "\${EATON_AUTH}";
|
||||
# priv_protocol = "AES";
|
||||
# priv_password = "\${EATON_PRIV}";
|
||||
#};
|
||||
};
|
||||
|
||||
modules.eaton = {
|
||||
walk = [
|
||||
"xupsInput"
|
||||
"xupsOutput"
|
||||
"xupsBypass"
|
||||
"xupsEnvironment"
|
||||
"xupsBattery"
|
||||
"xupsConfig"
|
||||
];
|
||||
lookups = [
|
||||
{
|
||||
lookup = "xupsInputName";
|
||||
source_indexes = [ "xupsInputPhase" ];
|
||||
}
|
||||
{
|
||||
lookup = "xupsOutputName";
|
||||
source_indexes = [ "xupsOutputPhase" ];
|
||||
}
|
||||
{
|
||||
lookup = "xupsBypassName";
|
||||
source_indexes = [ "xupsBypassPhase" ];
|
||||
}
|
||||
];
|
||||
overrides = {
|
||||
upsBatteryStatus.type = "EnumAsStateSet";
|
||||
xupsBatteryAbmStatus.type = "EnumAsStateSet";
|
||||
xupsBypassId.type = "EnumAsStateSet";
|
||||
xupsContactType.type = "EnumAsStateSet";
|
||||
xupsInputId.type = "EnumAsStateSet";
|
||||
xupsOutputId.type = "EnumAsStateSet";
|
||||
xupsOutputSource.type = "EnumAsStateSet";
|
||||
xupsBatteryFailure.type = "EnumAsStateSet";
|
||||
};
|
||||
};
|
||||
|
||||
modules.ilo_temp_fans = {
|
||||
walk = [
|
||||
#"cpqHeThermalFanTable"
|
||||
"cpqHeFltTolFanTable"
|
||||
"cpqHeTemperatureTable"
|
||||
#"cpqHeCoolingModuleTable"
|
||||
];
|
||||
timeout = "2s";
|
||||
retries = 10;
|
||||
overrides = {
|
||||
# cpqHeThermalFanTable
|
||||
# cpqHeFltTolFanTable
|
||||
cpqHeFltTolFanChassis.ignore = true;
|
||||
cpqHeFltTolFanHotPlug.type = "EnumAsInfo";
|
||||
cpqHeFltTolFanIndex.ignore = true;
|
||||
cpqHeFltTolFanLocale.type = "EnumAsInfo";
|
||||
cpqHeFltTolFanPresent.type = "EnumAsStateSet";
|
||||
cpqHeFltTolFanType.type = "EnumAsInfo";
|
||||
cpqHeFltTolFanSpeed.type = "EnumAsStateSet";
|
||||
cpqHeFltTolFanRedundant.type = "EnumAsStateSet";
|
||||
cpqHeFltTolFanCondition.type = "EnumAsStateSet";
|
||||
# cpqHeTemperatureTable
|
||||
cpqHeTemperatureChassis.ignore = true;
|
||||
cpqHeTemperatureIndex.ignore = true;
|
||||
cpqHeTemperatureLocale.type = "EnumAsInfo";
|
||||
cpqHeTemperatureCondition.type = "EnumAsStateSet";
|
||||
cpqHeTemperatureThresholdType.type = "EnumAsInfo";
|
||||
};
|
||||
};
|
||||
|
||||
modules.ilo_power = {
|
||||
walk = [
|
||||
"cpqHeFltTolPowerSupplyTable"
|
||||
#"cpqHePowerMeter"
|
||||
];
|
||||
timeout = "2s";
|
||||
retries = 10;
|
||||
overrides = {
|
||||
# cpqHeFltTolPowerSupplyTable
|
||||
cpqHeFltTolPowerSupplyChassis.ignore = true;
|
||||
cpqHeFltTolPowerSupplyHotPlug.type = "EnumAsInfo";
|
||||
cpqHeFltTolPowerSupplyErrorCondition.type = "EnumAsStateSet";
|
||||
cpqHeFltTolPowerSupplyBay.ignore = true;
|
||||
cpqHeFltTolPowerSupplyPresent.type = "EnumAsStateSet";
|
||||
cpqHeFltTolPowerSupplyCondition.type = "EnumAsStateSet";
|
||||
cpqHeFltTolPowerSupplyStatus.type = "EnumAsStateSet";
|
||||
cpqHeFltTolPowerSupplyRedundant.type = "EnumAsStateSet";
|
||||
};
|
||||
};
|
||||
|
||||
modules.ilo_memory = {
|
||||
walk = [
|
||||
#"cpqHeCorrectableMemory"
|
||||
#"cpqHePowerFailure"
|
||||
#"cpqHeCriticalError"
|
||||
"cpqHeResMem2BoardTable"
|
||||
"cpqHeResMem2ModuleTable"
|
||||
#"cpqHeHWBios"
|
||||
#"cpqHeSysBackupBattery"
|
||||
#"cpqHeResilientMemory"
|
||||
#"cpqSm2Cntlr"
|
||||
#"cpqHostOs"
|
||||
#"cpqSeCpuTable"
|
||||
#"cpqDaCntlr"
|
||||
];
|
||||
timeout = "2s";
|
||||
retries = 10;
|
||||
overrides = {
|
||||
#cpqHeResMem2BoardTable
|
||||
cpqHeResMem2BoardCondition.type = "EnumAsStateSet";
|
||||
cpqHeResMem2BoardHotPlug.type = "EnumAsInfo";
|
||||
cpqHeResMem2BoardOnlineStatus.type = "EnumAsStateSet";
|
||||
cpqHeResMem2BoardErrorStatus.type = "EnumAsStateSet";
|
||||
cpqHeResMem2BoardLocked.type = "EnumAsStateSet";
|
||||
#cpqHeResMem2ModuleTable
|
||||
cpqHeResMem2ModuleCellStatus.type = "EnumAsStateSet";
|
||||
cpqHeResMem2ModuleCondition.type = "EnumAsStateSet";
|
||||
cpqHeResMem2ModuleSmartMemory.type = "EnumAsInfo";
|
||||
cpqHeResMem2ModuleStatus.type = "EnumAsStateSet";
|
||||
cpqHeResMem2ModuleTechnology.type = "EnumAsInfo";
|
||||
cpqHeResMem2ModuleType.type = "EnumAsInfo";
|
||||
};
|
||||
};
|
||||
|
||||
modules.switch = {
|
||||
walk = [
|
||||
"interfaces"
|
||||
"ifXTable"
|
||||
"dot3StatsTable"
|
||||
"dot3HCStatsTable"
|
||||
"dot3Errors"
|
||||
"dot3Tests"
|
||||
"dot3CollTable"
|
||||
"lldpLocChassisId"
|
||||
"lldpRemTable"
|
||||
"lldpLocPortTable"
|
||||
#"dot1dBasePort"
|
||||
];
|
||||
lookups = [
|
||||
{
|
||||
lookup = "ifAlias";
|
||||
source_indexes = [ "ifIndex" ];
|
||||
}
|
||||
{
|
||||
lookup = "ifDescr";
|
||||
source_indexes = [ "ifIndex" ];
|
||||
}
|
||||
{
|
||||
lookup = "ifName";
|
||||
source_indexes = [ "ifIndex" ];
|
||||
}
|
||||
{
|
||||
lookup = "ifAlias";
|
||||
source_indexes = [ "dot3StatsIndex" ];
|
||||
}
|
||||
{
|
||||
lookup = "ifDescr";
|
||||
source_indexes = [ "dot3StatsIndex" ];
|
||||
}
|
||||
{
|
||||
lookup = "ifName";
|
||||
source_indexes = [ "dot3StatsIndex" ];
|
||||
}
|
||||
{
|
||||
lookup = "lldpRemChassisId";
|
||||
source_indexes = [
|
||||
"lldpRemTimeMark"
|
||||
"lldpRemLocalPortNum"
|
||||
"lldpRemIndex"
|
||||
];
|
||||
}
|
||||
];
|
||||
overrides = {
|
||||
dot3StatsDuplexStatus.type = "EnumAsStateSet";
|
||||
dot3StatsEtherChipSet.ignore = true;
|
||||
dot3StatsIndex.ignore = true;
|
||||
ifAdminStatus.type = "EnumAsStateSet";
|
||||
ifAlias.ignore = true;
|
||||
ifDescr.ignore = true;
|
||||
ifIndex.ignore = true;
|
||||
ifName.ignore = true;
|
||||
ifOperStatus.type = "EnumAsStateSet";
|
||||
ifType.type = "EnumAsInfo";
|
||||
lldpLocPortIdSubtype.type = "EnumAsInfo";
|
||||
lldpRemPortIdSubtype.type = "EnumAsInfo";
|
||||
};
|
||||
timeout = "5s";
|
||||
retries = 10;
|
||||
};
|
||||
|
||||
modules.uptime = {
|
||||
walk = [ "sysUpTime" ];
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{ ... }:
|
||||
|
||||
[
|
||||
"cephiroth.adm.crans.org"
|
||||
]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{ ... }:
|
||||
|
||||
[
|
||||
"ilo-cephiroth.adm.crans.org"
|
||||
"ilo-daniel.adm.crans.org"
|
||||
"ilo-gulp.adm.crans.org"
|
||||
"ilo-jack.adm.crans.org"
|
||||
"ilo-odlyd.adm.crans.org"
|
||||
"ilo-sam.adm.crans.org"
|
||||
"ilo-stitch.adm.crans.org"
|
||||
"ilo-thot.adm.crans.org"
|
||||
"ilo-zamok.adm.crans.org"
|
||||
]
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
{ ... }:
|
||||
|
||||
[
|
||||
"backup-ft.adm.crans.org"
|
||||
"boeing.adm.crans.org"
|
||||
"cameron.adm.crans.org"
|
||||
"cas.adm.crans.org"
|
||||
"collabora.adm.crans.org"
|
||||
"daniel.adm.crans.org"
|
||||
"ecilis.adm.crans.org"
|
||||
"eclat.adm.crans.org"
|
||||
"en7.adm.crans.org"
|
||||
"ethercalc.adm.crans.org"
|
||||
"flirt.adm.crans.org"
|
||||
"ft.adm.crans.org"
|
||||
"fyre.adm.crans.org"
|
||||
"gitlab-ci.adm.crans.org"
|
||||
"gitzly.adm.crans.org"
|
||||
"gulp.adm.crans.org"
|
||||
"helloworld.adm.crans.org"
|
||||
"hodaur.adm.crans.org"
|
||||
"irc.adm.crans.org"
|
||||
"jack.adm.crans.org"
|
||||
"jitsi.adm.crans.org"
|
||||
"kenobi.adm.crans.org"
|
||||
"kiwi.adm.crans.org"
|
||||
"mailman.adm.crans.org"
|
||||
"neo.adm.crans.org"
|
||||
"neree.adm.crans.org"
|
||||
"nextcloud.adm.crans.org"
|
||||
"owl.adm.crans.org"
|
||||
"owncloud.adm.crans.org"
|
||||
"periodique.adm.crans.org"
|
||||
"proxy-pve-adh.adm.crans.org"
|
||||
"ptf.adm.crans.org"
|
||||
"re2o.adm.crans.org"
|
||||
"redisdead.adm.crans.org"
|
||||
"reverseproxy.adm.crans.org"
|
||||
"romanesco.adm.crans.org"
|
||||
"roundcube.adm.crans.org"
|
||||
"routeur-2754.adm.crans.org"
|
||||
"routeur-daniel.adm.crans.org"
|
||||
"routeur-ft.adm.crans.org"
|
||||
"routeur-jack.adm.crans.org"
|
||||
"routeur-sam.adm.crans.org"
|
||||
"sam.adm.crans.org"
|
||||
"silice.adm.crans.org"
|
||||
"sputnik.adm.crans.org"
|
||||
"stitch.adm.crans.org"
|
||||
"tealc.adm.crans.org"
|
||||
"thot.adm.crans.org"
|
||||
"victoriametrics.adm.crans.org"
|
||||
"voyager.adm.crans.org"
|
||||
"wall-e.adm.crans.org"
|
||||
"yson-partou.adm.crans.org"
|
||||
"zamok.adm.crans.org"
|
||||
]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{ ... }:
|
||||
|
||||
[
|
||||
"daniel.adm.crans.org"
|
||||
"jack.adm.crans.org"
|
||||
"sam.adm.crans.org"
|
||||
"tealc.adm.crans.org"
|
||||
]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{...}:
|
||||
[
|
||||
"silice.adm.crans.org"
|
||||
"sputnik.adm.crans.org"
|
||||
]
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{ ... }:
|
||||
|
||||
[
|
||||
"excalibur.adm.crans.org"
|
||||
]
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
mkScrapeConfig = name: config: let
|
||||
relabelAddress = lib.optional (builtins.hasAttr "replacementAddress" config) {
|
||||
source_labels = [ "__address__" ];
|
||||
target_label = "__address__";
|
||||
replacement = config.replacementAddress;
|
||||
};
|
||||
relabelTarget = lib.optional (builtins.hasAttr "replacementTarget" config) {
|
||||
source_labels = [ "__address__" ];
|
||||
target_label = "__param_target";
|
||||
replacement = config.replacementTarget;
|
||||
};
|
||||
in {
|
||||
job_name = name;
|
||||
metrics_path = config.path or "/metrics";
|
||||
static_configs = [ { targets = config.targets; } ];
|
||||
params = config.params or { };
|
||||
relabel_configs = [
|
||||
{ source_labels = [ "__address__" ]; target_label = "instance"; }
|
||||
# { source_labels = [ "__address__" ]; target_label = "__param_target"; }
|
||||
# { source_labels = [ "__param_target" ]; target_label = "instance"; }
|
||||
] ++ relabelTarget ++ relabelAddress;
|
||||
scrape_timeout = config.timeout or "15s";
|
||||
scrape_interval = config.interval or "1m";
|
||||
};
|
||||
mkScrapeConfigs = lib.attrsets.mapAttrsToList mkScrapeConfig;
|
||||
critical = { severity = "critical"; };
|
||||
warning = { severity = "warning"; };
|
||||
mkRuleGroups = lib.attrsets.mapAttrsToList (name: path: {
|
||||
inherit name;
|
||||
rules = let
|
||||
rules = import path { inherit critical warning; };
|
||||
alerts = lib.attrsets.mapAttrsToList
|
||||
(alert: attrs: attrs // { inherit alert; })
|
||||
(rules.alerts or {});
|
||||
records = lib.attrsets.mapAttrsToList
|
||||
(record: attrs: attrs // { inherit record; })
|
||||
(rules.records or {});
|
||||
in alerts ++ records;
|
||||
});
|
||||
in
|
||||
{
|
||||
age.secrets.snmp-env = {
|
||||
file = ../../../secrets/victoriametrics/snmp.age;
|
||||
owner = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 8428 9116 ];
|
||||
|
||||
services.victoriametrics = let
|
||||
scrapeConfigs = mkScrapeConfigs (import ./scrape.nix {});
|
||||
in {
|
||||
enable = true;
|
||||
retentionPeriod = "30d";
|
||||
extraOptions = [
|
||||
"-enableTCP6"
|
||||
# Compatibilité avec la datasource Prometheus de Grafana
|
||||
"-vmalert.proxyURL=http://localhost:8880"
|
||||
];
|
||||
prometheusConfig.scrape_configs = scrapeConfigs;
|
||||
};
|
||||
|
||||
services.vmalert.instances.default = {
|
||||
enable = true;
|
||||
rules.groups = mkRuleGroups (import ./rules {});
|
||||
settings = let
|
||||
victoriametricsUrl = "http://localhost:8428";
|
||||
in {
|
||||
"datasource.url" = victoriametricsUrl;
|
||||
"remoteWrite.url" = victoriametricsUrl;
|
||||
"remoteRead.url" = victoriametricsUrl;
|
||||
"enableTCP6" = true;
|
||||
#"notifier.url" = [ "http://docker-ovh.adm.auro.re:9093" ]; # A MODIFIER #####################################
|
||||
};
|
||||
};
|
||||
|
||||
services.prometheus.exporters.snmp = let
|
||||
generator = import ./snmp.nix {};
|
||||
configSnmp = pkgs.runCommand "snmp.yml" { # Nous avons des doutes sur ce fichier? faut-il l'importer?
|
||||
nativeBuildInputs = [ pkgs.prometheus-snmp-exporter ];
|
||||
generator = builtins.toJSON generator;
|
||||
passAsFile = [ "generator" ];
|
||||
} ''
|
||||
${lib.getExe' pkgs.prometheus-snmp-exporter "generator"} \
|
||||
generate \
|
||||
--mibs-dir=${./mibs} \
|
||||
--generator-path=$generatorPath \
|
||||
--output-path=$out
|
||||
'';
|
||||
in {
|
||||
enable = true;
|
||||
enableConfigCheck = false;
|
||||
environmentFile = config.age.secrets.snmp-env.path;
|
||||
configurationPath = configSnmp;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
net-snmp
|
||||
];
|
||||
}
|
||||
Loading…
Reference in New Issue