{ critical, warning, ... }: { alerts.PostgresqlDown = { expr = "pg_up == 0"; for = "0m"; labels = critical; }; alerts.PostgresqlExporterError = { expr = "pg_exporter_last_scrape_error > 0"; for = "0m"; labels = critical; }; alerts.PostgresqlReplicationLag = { expr = '' pg_replication_lag > 30 and on(instance) pg_replication_is_replica == 1 ''; for = "0m"; labels = critical; annotations = { Lag = "{{ $value }}"; DB = "{{ $labels.datname }}"; }; }; alerts.PostgresqlTableNotVacuumed = { expr = '' time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24 ''; for = "0m"; labels = warning; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlTableNotAnalyzed = { expr = '' time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24 ''; for = "0m"; labels = warning; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlTooManyConnections = { expr = '' ( sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) ) * 100 > pg_settings_max_connections * 80 ''; for = "2m"; labels = warning; annotations = { UsageLevel = "{{ $value | humanizePercentage }}"; DB = "{{ $labels.datname }}"; }; }; alerts.PostgresqlDeadLocks = { expr = '' increase( pg_stat_database_deadlocks{ datname!~"template.*|postgres" }[1m] ) > 5 ''; for = "0m"; labels = warning; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlSlowQueries = { expr = "pg_slow_queries > 0"; for = "2m"; labels = warning; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlHighRollbackRate = { expr = '' ( rate( pg_stat_database_xact_rollback{datname!~"template.*"}[3m] ) / rate( pg_stat_database_xact_commit{datname!~"template.*"}[3m] ) ) > 0.2 ''; for = "0m"; labels = warning; annotations = { RollbackRate = "{{ $value | humanizePercentage }}"; DB = "{{ $labels.datname }}"; }; }; alerts.PostgresqlWalReplicationStopped = { expr = "rate(pg_xlog_position_bytes[1m]) == 0"; for = "0m"; labels = critical; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlHighRateStatementTimeout = { expr = '' rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3 ''; for = "0m"; labels = critical; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlHighRateDeadlock = { expr = '' increase( postgresql_errors_total{type="deadlock_detected"}[1m] ) > 1 ''; for = "0m"; labels = critical; annotations.DB = "{{ $labels.datname }}"; }; alert.PostgresqlTooManyDeadTuples = { expr = '' ( (pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup) ) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1) ''; for = "2m"; labels = warning; annotations.DB = "{{ $labels.datname }}"; }; alert.PostgresqlSplitBrain = { expr = '' count(pg_replication_is_replica == 0) != 1 ''; for = "0m"; labels = critical; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlPromotedNode = { expr = '' pg_replication_is_replica and changes(pg_replication_is_replica[1m]) > 0 ''; for = "0m"; labels = warning; annotations.DB = "{{ $labels.datname }}"; }; alerts.PostgresqlTooManyLocksAcquired = { expr = '' ( (sum (pg_locks_count)) / ( pg_settings_max_locks_per_transaction * pg_settings_max_connections ) ) * 100 > 20 ''; for = "2m"; labels = critical; annotations = { DB = "{{ $labels.datname }}"; Deadlocks = "{{ $value | humanizePercentage }}"; }; }; }