| 1234567891011121314151617181920212223242526272829 |
- service: checkout-service
- runbook_version: "1.2"
- procedures:
- - pattern: "DB pool exhausted"
- severity: P1
- steps:
- - "Identify slow queries: run SHOW PROCESSLIST on the primary DB; look for queries >5s"
- - "Run EXPLAIN on the slowest query to identify full table scans or missing indexes"
- - "Temporary mitigation: increase pool_size from 10 to 20 via config hot-reload"
- - "Permanent fix: add the missing index. Example: CREATE INDEX idx_orders_user_id ON orders(user_id)"
- - "Validate index creation: run EXPLAIN again to confirm index is used"
- - "Restart pods once index is verified: kubectl rollout restart deployment/checkout-service"
- - "Monitor db_pool_active_connections and http_request_duration_p99 to confirm recovery"
- - pattern: "high_p99_latency"
- severity: P1
- steps:
- - "Check downstream dependencies: DB query time, Redis cache hit rate, external API latency"
- - "Review request_queue_depth metric — if >100, consider pod autoscaling"
- - "Enable slow query logging if not active: SET GLOBAL slow_query_log = 'ON'"
- - "Consider circuit breaker activation to shed load and protect downstream services"
- - "Check recent deployments: git log --since='2 hours ago' on checkout-service"
- - pattern: "connection timeout"
- severity: P2
- steps:
- - "Check network connectivity between checkout-service pods and DB"
- - "Verify DB security group rules allow connections from checkout-service subnet"
- - "Check for DB maintenance windows or failover events in RDS console"
|
