nixos/hosts/vm/victoriametrics/rules/postgresql.nix

{ critical, warning, ... }:

{
  alerts.PostgresqlDown = {
    expr = "pg_up == 0";
    for = "0m";
    labels = critical;
  };

  alerts.PostgresqlExporterError = {
    expr = "pg_exporter_last_scrape_error > 0";
    for = "0m";
    labels = critical;
  };

  alerts.PostgresqlReplicationLag = {
    expr = ''
      pg_replication_lag > 30
      and on(instance)
      pg_replication_is_replica == 1
    '';
    for = "0m";
    labels = critical;
    annotations = {
      Lag = "{{ $value }}";
      DB = "{{ $labels.datname }}";
    };
  };

  alerts.PostgresqlTableNotVacuumed = {
    expr = ''
      time() - pg_stat_user_tables_last_autovacuum
      > 60 * 60 * 24
    '';
    for = "0m";
    labels = warning;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlTableNotAnalyzed = {
    expr = ''
      time() - pg_stat_user_tables_last_autoanalyze
      > 60 * 60 * 24
    '';
    for = "0m";
    labels = warning;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlTooManyConnections = {
    expr = ''
      (
        sum by (datname)
          (pg_stat_activity_count{datname!~"template.*|postgres"})
      ) * 100
      > pg_settings_max_connections * 80
    '';
    for = "2m";
    labels = warning;
    annotations = {
      UsageLevel = "{{ $value | humanizePercentage }}";
      DB = "{{ $labels.datname }}";
    };
  };

  alerts.PostgresqlDeadLocks = {
    expr = ''
      increase(
        pg_stat_database_deadlocks{
          datname!~"template.*|postgres"
        }[1m]
      ) > 5
    '';
    for = "0m";
    labels = warning;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlSlowQueries = {
    expr = "pg_slow_queries > 0";
    for = "2m";
    labels = warning;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlHighRollbackRate = {
    expr = ''
      (
        rate(
          pg_stat_database_xact_rollback{datname!~"template.*"}[3m]
        ) / rate(
        pg_stat_database_xact_commit{datname!~"template.*"}[3m]
        )
      ) > 0.2
    '';
    for = "0m";
    labels = warning;
    annotations = {
      RollbackRate = "{{ $value | humanizePercentage }}";
      DB = "{{ $labels.datname }}";
    };
  };

  alerts.PostgresqlWalReplicationStopped = {
    expr = "rate(pg_xlog_position_bytes[1m]) == 0";
    for = "0m";
    labels = critical;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlHighRateStatementTimeout = {
    expr = ''
      rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3
    '';
    for = "0m";
    labels = critical;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlHighRateDeadlock = {
    expr = ''
      increase(
        postgresql_errors_total{type="deadlock_detected"}[1m]
      ) > 1
    '';
    for = "0m";
    labels = critical;
    annotations.DB = "{{ $labels.datname }}";
  };

  alert.PostgresqlTooManyDeadTuples = {
    expr = ''
      (
        (pg_stat_user_tables_n_dead_tup > 10000)
        / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)
      ) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)
    '';
    for = "2m";
    labels = warning;
    annotations.DB = "{{ $labels.datname }}";
  };

  alert.PostgresqlSplitBrain = {
    expr = ''
      count(pg_replication_is_replica == 0) != 1
    '';
    for = "0m";
    labels = critical;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlPromotedNode = {
    expr = ''
      pg_replication_is_replica
      and
      changes(pg_replication_is_replica[1m]) > 0
    '';
    for = "0m";
    labels = warning;
    annotations.DB = "{{ $labels.datname }}";
  };

  alerts.PostgresqlTooManyLocksAcquired = {
    expr = ''
      (
        (sum (pg_locks_count))
        / (
          pg_settings_max_locks_per_transaction
          * pg_settings_max_connections
        )
      ) * 100 > 20
    '';
    for = "2m";
    labels = critical;
    annotations = {
      DB = "{{ $labels.datname }}";
      Deadlocks = "{{ $value | humanizePercentage }}";
    };
  };
}