Mixin: Switch to timeseries and improve stat panels.

mysqld-mixin/alerts/alerts.yaml

@@ -0,0 +1,127 @@
+groups:
+- name: MySQLAlerts
+  rules:
+  - alert: MySQLDown
+    expr: mysql_up != 1
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} is not up.'
+      summary: MySQL not up.
+  - alert: MySQLRestarted
+    expr: mysql_global_status_uptime < 60
+    labels:
+      severity: critical
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} restarted less than a minute ago.'
+      summary: MySQL restarted.
+  - alert: MySQLSlowQueries
+    expr: rate(mysql_global_status_slow_queries[5m]) * 60 > 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} is facing slow queries.'
+      summary: MySQL slow queries.
+  - alert: MySQLTuneThreadCacheSize
+    expr: mysql_global_status_threads_created / mysql_global_status_connections > 0.5
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} thread cache needs tuning.'
+      summary: MySQL thread cache needs tuning.
+  - alert: MySQLInsufficientTempTableSize
+    expr: mysql_global_status_created_tmp_disk_tables / mysql_global_status_created_tmp_tables > 0.2
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} has insufficient temp table size.'
+      summary: MySQL temp table size insufficient.
+  - alert: MySQLInsufficientBinLogCacheSize
+    expr: mysql_global_status_binlog_cache_disk_use / mysql_global_status_binlog_cache_use > 0.2
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} has insufficient binary log cache size.'
+      summary: MySQL binary log cache size insufficient.
+  - alert: MySQLInsufficientBinLogStmtCacheSize
+    expr: mysql_global_status_binlog_stmt_cache_disk_use / mysql_global_status_binlog_stmt_cache_use > 0.2
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: 'MySQL {{$labels.job}} on {{$labels.instance}} has insufficient binary log statement cache size.'
+      summary: MySQL binary log statement cache size insufficient.
+  - alert: MySQLGaleraNotReady
+    expr: mysql_global_status_wsrep_ready != 1
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: '{{$labels.job}} on {{$labels.instance}} is not ready.'
+      summary: Galera cluster node not ready.
+  - alert: MySQLGaleraOutOfSync
+    expr: (mysql_global_status_wsrep_local_state != 4 and mysql_global_variables_wsrep_desync
+      == 0)
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: '{{$labels.job}} on {{$labels.instance}} is not in sync ({{$value}}
+        != 4).'
+      summary: Galera cluster node out of sync.
+  - alert: MySQLGaleraDonorFallingBehind
+    expr: (mysql_global_status_wsrep_local_state == 2 and mysql_global_status_wsrep_local_recv_queue > 100)
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: '{{$labels.job}} on {{$labels.instance}} is a donor (hotbackup)
+        and is falling behind (queue size {{$value}}).'
+      summary: XtraDB cluster donor node falling behind.
+  - alert: MySQLReplicationNotRunning
+    expr: mysql_slave_status_slave_io_running == 0 or mysql_slave_status_slave_sql_running
+      == 0
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      description: "Replication on {{$labels.instance}} (IO or SQL) has been down for more than 2 minutes."
+      summary: Replication is not running.
+  - alert: MySQLReplicationLag
+    expr: (instance:mysql_slave_lag_seconds > 30) and on(instance) (predict_linear(instance:mysql_slave_lag_seconds[5m],
+      60 * 2) > 0)
+    for: 1m
+    labels:
+      severity: critical
+    annotations:
+      description: "Replication on {{$labels.instance}} has fallen behind and is not recovering."
+      summary: MySQL slave replication is lagging.
+  - alert: MySQLHeartbeatLag
+    expr: (instance:mysql_heartbeat_lag_seconds > 30) and on(instance) (predict_linear(instance:mysql_heartbeat_lag_seconds[5m],
+      60 * 2) > 0)
+    for: 1m
+    labels:
+      severity: critical
+    annotations:
+      description: "The heartbeat is lagging on {{$labels.instance}} and is not recovering."
+      summary: MySQL heartbeat is lagging.
+  - alert: MySQLInnoDBLogWaits
+    expr: rate(mysql_global_status_innodb_log_waits[15m]) > 10
+    labels:
+      severity: warning
+    annotations:
+      description: The innodb logs are waiting for disk at a rate of {{$value}} /
+        second
+      summary: MySQL innodb log writes stalling.
+  - alert: MySQLSlaveBehindMaster
+    expr: rate(mysql_slave_status_seconds_behind_master[5m]) > 30
+    labels:
+      severity: warning
+    annotations:
+      description: MySQL slave on {{$labels.instance}} is behind master by more than 30 seconds.
+      summary: MySQL innodb slave behind master.