The Four Golden Signals of Monitoring: Latency, Traffic, Errors & Saturation (2026)

# Prometheus: latency histogram (best practice)
# Use a histogram, not a gauge — histograms allow percentile calculation

http_request_duration_seconds = Histogram(
    "http_request_duration_seconds",
    "HTTP request latency",
    labelnames=["method", "path", "status"],
    buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
)

# PromQL: p99 latency by path
histogram_quantile(0.99,
  sum by (path, le) (
    rate(http_request_duration_seconds_bucket[5m])
  )
)

# Alert: p99 latency exceeds SLO
- alert: HighLatencyP99
  expr: |
    histogram_quantile(0.99,
      sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
    ) > 1.0
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "p99 latency {{ $value | humanizeDuration }} exceeds 1s SLO"

# Separate "fast" and "slow" latency:
# Track successful requests (2xx) separately from errors (5xx)
# A slow error is a different problem than a slow success

# Latency by percentile: dashboard PromQL
# p50:  histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))
# p95:  histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
# p99:  histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))
# p999: histogram_quantile(0.999, rate(http_request_duration_seconds_bucket[5m]))

# Traffic = demand on the system
# For APIs: requests per second
rate(http_requests_total[1m])

# For databases: queries per second
rate(mysql_global_status_queries[1m])

# For queues: messages per second
rate(rabbitmq_queue_messages_published_total[1m])

# Traffic anomaly alerts (sudden drop is often a problem):
- alert: TrafficDropDetected
  expr: |
    rate(http_requests_total[5m])
    < (rate(http_requests_total[1h] offset 1h) * 0.5)
  for: 10m
  annotations:
    summary: "Traffic dropped >50% vs same time last hour — possible outage"

# Traffic spike alert (may need to scale):
- alert: TrafficSpike
  expr: |
    rate(http_requests_total[5m])
    > (rate(http_requests_total[1h] offset 1h) * 3)
  for: 5m
  annotations:
    summary: "Traffic 3x higher than last hour — check saturation signals"

# Breakdown by endpoint for capacity planning:
# topk(10, rate(http_requests_total[5m])) by (path)

# Error rate: server errors vs total
sum(rate(http_requests_total{status=~"5.."}[5m]))
/ sum(rate(http_requests_total[5m]))

# Alert: server error rate > 1% for 5 minutes
- alert: HighErrorRate
  expr: |
    sum(rate(http_requests_total{status=~"5.."}[5m]))
    / sum(rate(http_requests_total[5m])) > 0.01
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: "Error rate {{ $value | humanizePercentage }} — exceeds 1% threshold"

# Track 4xx separately (client errors, not your fault but worth watching):
- alert: ClientErrorSpike
  expr: |
    rate(http_requests_total{status=~"4.."}[5m])
    > rate(http_requests_total{status=~"4.."}[5m] offset 1h) * 5
  for: 5m
  annotations:
    summary: "4xx errors spiked 5x vs 1h ago — possible API contract change"

# Break down errors by endpoint to find the culprit:
# topk(5,
#   rate(http_requests_total{status=~"5.."}[5m])
# ) by (path)

# "Slow errors" — errors that also take a long time
# Don't count error latency in your p99 SLO
# Separate SLO for error latency vs success latency

# CPU saturation alert
- alert: HighCpuSaturation
  expr: avg(rate(process_cpu_seconds_total[5m])) * 100 > 80
  for: 5m
  annotations:
    summary: "CPU at {{ $value }}% — approaching saturation"

# Memory saturation
- alert: HighMemorySaturation
  expr: |
    process_resident_memory_bytes
    / node_memory_MemTotal_bytes > 0.85
  for: 5m
  annotations:
    summary: "Memory at {{ $value | humanizePercentage }} of total"

# Database connection pool saturation
- alert: DbPoolSaturation
  expr: |
    db_pool_checked_out_connections
    / db_pool_max_connections > 0.80
  for: 2m
  annotations:
    summary: "DB pool {{ $value | humanizePercentage }} utilized — new connections will block"

# Thread pool saturation (Java/Tomcat example)
- alert: ThreadPoolSaturation
  expr: |
    tomcat_threads_busy_threads
    / tomcat_threads_config_max_threads > 0.90
  for: 2m
  annotations:
    summary: "Thread pool {{ $value | humanizePercentage }} — request queuing imminent"