nixos/hosts/vm/victoriametrics/rules/ceph.nix

{ critical, warning, ... }:

{
  alerts.CephErrorState = {
    expr = ''
      ceph_health_status > 1
    '';
    for = "5m";
    labels = critical;
  };

  alerts.CephMonitorAvailableStorageCritical = {
    expr = ''
      ceph_monitor_avail_percent < 15
    '';
    for = "1m";
    labels = critical;
  };

  alerts.CephMonitorAvailableStorageWarning = {
    expr = ''
      ceph_monitor_avail_percent < 30
    '';
    for = "1m";
    labels = warning;
  };

  alerts.CephMonitorClockSkewTooHigh = {
    expr = ''
      abs(ceph_monitor_clock_skew_seconds) > 0.1
    '';
    for = "1m";
    labels = warning;
  };  

  alerts.CephOSDApplyLatencyTooHigh = {
    expr = ''
      ceph_osd_perf_apply_latency_seconds > 10
    '';
    for = "1m";
    labels = warning;
    annotations = {
      Latency = "{{ $value }}";
      Device = "{{ $labels.device }}";
    };
  };

  alerts.CephOSDDown = {
    expr = ''
      ceph_osd_up == 0
    '';
    for = "1m";
    labels = critical;
  };

  alerts.CephOSDReweighted = {
    expr = ''
      ceph_osd_weight < 1
    '';
    for = "1h";
    labels = warning;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephOSDUtilization = {
    expr = ''
      ceph_osd_utilization > 90
    '';
    for = "1m";
    labels = critical;
  };

  alerts.CephPgActivating = {
    expr = ''
      ceph_pg_activating > 0
    '';
    for = "5m";
    labels = critical;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephPgDown = {
    expr = ''
      ceph_pg_down > 0
    '';
    for = "3m";
    labels = critical;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephPgIncomplete = {
    expr = ''
      ceph_pg_incomplete > 0
    '';
    for = "2m";
    labels = critical;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephPgInconsistent = {
    expr = ''
      ceph_pg_inconsistent > 0
    '';
    for = "1m";
    labels = warning;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephPgBackfillTooFull = {
    expr = ''
      ceph_pg_backfill_toofull > 0
    '';
    for = "5m";
    labels = warning;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephPgUnavailable = {
    expr = ''
      ceph_pg_total - ceph_pg_active > 0
    '';
    for = "5m";
    labels = critical;
    annotations.Instance = "{{ $labels.cluster }}";
    annotations.Database = "{{ $value }}";
  };

  alerts.CephTargetDown = {
    expr = ''
      up{job="ceph"} == 0
    '';
    for = "10m";
    labels = critical;
  };

  alerts.CephWarnState = {
    expr = ''
      ceph_health_status == 1
    '';
    for = "30m";
    labels = warning;
  };
}