gabriel / musehub public
cloudwatch-alerts.sh bash
196 lines 9.5 KB
Raw
sha256:7afc59b2ee9c70d18b748d269602c588385d6e936cc22b056fd775ab9f2fd499 fix: real CloudWatch alerting on both envs, add environment… Sonnet 5 minor ⚠ breaking 22 hours ago
1 #!/usr/bin/env bash
2 # MuseHub — CloudWatch log group, metric filters, alarms, and SNS alerting.
3 #
4 # Run once per environment; safe to re-run (put-metric-alarm is idempotent,
5 # log groups/topics/subscriptions are created only if absent).
6 #
7 # Staging and production are separate AWS accounts with separate log groups
8 # (/musehub/staging, /musehub/production) — this script targets one
9 # environment per run, matching deploy/push.sh's per-environment AWS_PROFILE
10 # pattern. Run it once per environment, under that environment's own SSO
11 # session (aws sso login --profile musehub-nonproduction|musehub-production).
12 #
13 # Prerequisites:
14 # * AWS CLI configured with permissions for logs:*, cloudwatch:*, sns:*
15 # (the operator's own SSO session already has this via AdministratorAccess)
16 # * ALERT_EMAILS set in the environment, or accept the default (Gabriel + Aaron)
17 #
18 # Usage:
19 # bash deploy/cloudwatch-alerts.sh --env staging --profile musehub-nonproduction
20 # bash deploy/cloudwatch-alerts.sh --env production --profile musehub-production
21
22 set -euo pipefail
23
24 # ── Config ────────────────────────────────────────────────────────────────────
25
26 AWS_REGION="${AWS_REGION:-us-east-1}"
27 AWS_PROFILE_ARG="${AWS_PROFILE_ARG:-}"
28 ENV=""
29 LOG_RETENTION_DAYS=30 # Hot retention: 30 days in CloudWatch
30 # Route alerts to both maintainers by default — override with ALERT_EMAILS
31 # (comma-separated) if that ever needs to change.
32 ALERT_EMAILS="${ALERT_EMAILS:[email protected],[email protected]}"
33
34 # Alarm thresholds — aligned with the pre-launch checklist and #149's RPO/RTO decisions.
35 THRESHOLD_5XX_RATE="1" # percent: alarm when 5xx / total > 1%
36 THRESHOLD_P99_LATENCY_MS="2000" # milliseconds
37
38 # ── Helpers ───────────────────────────────────────────────────────────────────
39
40 log() { echo "[cloudwatch] $*"; }
41 die() { echo "[cloudwatch] ERROR: $*" >&2; exit 1; }
42
43 aws_cmd() {
44 if [[ -n "$AWS_PROFILE_ARG" ]]; then
45 aws --region "$AWS_REGION" --profile "$AWS_PROFILE_ARG" "$@"
46 else
47 aws --region "$AWS_REGION" "$@"
48 fi
49 }
50
51 # ── Parse args ────────────────────────────────────────────────────────────────
52
53 while [[ $# -gt 0 ]]; do
54 case "$1" in
55 --env) ENV="$2"; shift 2 ;;
56 --profile) AWS_PROFILE_ARG="$2"; shift 2 ;;
57 --region) AWS_REGION="$2"; shift 2 ;;
58 *) die "Unknown argument: $1" ;;
59 esac
60 done
61
62 case "$ENV" in
63 staging|production) ;;
64 *) die "Usage: bash deploy/cloudwatch-alerts.sh --env staging|production --profile <sso-profile>" ;;
65 esac
66
67 LOG_GROUP="/musehub/${ENV}"
68 SNS_TOPIC_NAME="musehub-${ENV}-alerts"
69
70 # ── Step 1: Log group + explicit hot retention ───────────────────────────────
71 # The log group should already exist (created by the app's awslogs driver on
72 # first deploy) — this just ensures retention is set, since CloudWatch Logs
73 # default to unlimited retention otherwise (a cost/compliance gap, not an
74 # observability one).
75
76 log "[1/5] Setting ${LOG_RETENTION_DAYS}-day retention on $LOG_GROUP..."
77 aws_cmd logs create-log-group --log-group-name "$LOG_GROUP" 2>/dev/null || true # idempotent
78 aws_cmd logs put-retention-policy \
79 --log-group-name "$LOG_GROUP" \
80 --retention-in-days "$LOG_RETENTION_DAYS"
81
82 # ── Step 2: SNS topic + subscriptions ────────────────────────────────────────
83
84 log "[2/5] Creating SNS topic $SNS_TOPIC_NAME..."
85
86 SNS_ARN=$(aws_cmd sns create-topic \
87 --name "$SNS_TOPIC_NAME" \
88 --query TopicArn --output text)
89
90 log "SNS topic ARN: $SNS_ARN"
91
92 IFS=',' read -ra EMAILS <<< "$ALERT_EMAILS"
93 for email in "${EMAILS[@]}"; do
94 email="$(echo "$email" | xargs)" # trim whitespace
95 [[ -z "$email" ]] && continue
96 aws_cmd sns subscribe \
97 --topic-arn "$SNS_ARN" \
98 --protocol email \
99 --notification-endpoint "$email" \
100 --query SubscriptionArn --output text > /dev/null
101 log " Email subscription created for $email (must confirm via the email AWS sends)"
102 done
103
104 # ── Step 3: Metric filters (extract from structured JSON logs) ─────────────────
105
106 log "[3/5] Creating metric filters on $LOG_GROUP..."
107
108 aws_cmd logs put-metric-filter \
109 --log-group-name "$LOG_GROUP" \
110 --filter-name "musehub-5xx-count" \
111 --filter-pattern '{ $.status >= 500 }' \
112 --metric-transformations \
113 metricName="5xxCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0"
114
115 aws_cmd logs put-metric-filter \
116 --log-group-name "$LOG_GROUP" \
117 --filter-name "musehub-request-count" \
118 --filter-pattern '{ $.status >= 100 }' \
119 --metric-transformations \
120 metricName="RequestCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0"
121
122 aws_cmd logs put-metric-filter \
123 --log-group-name "$LOG_GROUP" \
124 --filter-name "musehub-duration-ms" \
125 --filter-pattern '{ $.duration_ms > 0 }' \
126 --metric-transformations \
127 metricName="RequestDurationMs-${ENV}",metricNamespace="MuseHub",metricValue='$.duration_ms',defaultValue="0"
128
129 log "Metric filters created: 5xxCount-${ENV}, RequestCount-${ENV}, RequestDurationMs-${ENV}"
130
131 # ── Step 4: CloudWatch Alarms ─────────────────────────────────────────────────
132
133 log "[4/5] Creating CloudWatch alarms (5xx>${THRESHOLD_5XX_RATE}%, p99>${THRESHOLD_P99_LATENCY_MS}ms)..."
134
135 # ─ 4a: 5xx error rate ────────────────────────────────────────────────────────
136 aws_cmd cloudwatch put-metric-alarm \
137 --alarm-name "musehub-${ENV}-5xx-rate-high" \
138 --alarm-description "[$ENV] 5xx error rate exceeded ${THRESHOLD_5XX_RATE}% — investigate immediately" \
139 --alarm-actions "$SNS_ARN" \
140 --ok-actions "$SNS_ARN" \
141 --metrics \
142 "[{\"Id\":\"e1\",\"Expression\":\"(m1/m2)*100\",\"Label\":\"5xxRate\",\"ReturnData\":true},
143 {\"Id\":\"m1\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"5xxCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false},
144 {\"Id\":\"m2\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"RequestCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false}]" \
145 --comparison-operator GreaterThanThreshold \
146 --threshold "$THRESHOLD_5XX_RATE" \
147 --evaluation-periods 2 \
148 --datapoints-to-alarm 2 \
149 --treat-missing-data notBreaching
150
151 # ─ 4b: p99 request latency ───────────────────────────────────────────────────
152 aws_cmd cloudwatch put-metric-alarm \
153 --alarm-name "musehub-${ENV}-p99-latency-high" \
154 --alarm-description "[$ENV] p99 request latency exceeded ${THRESHOLD_P99_LATENCY_MS}ms" \
155 --alarm-actions "$SNS_ARN" \
156 --ok-actions "$SNS_ARN" \
157 --namespace MuseHub \
158 --metric-name "RequestDurationMs-${ENV}" \
159 --period 60 \
160 --evaluation-periods 3 \
161 --datapoints-to-alarm 2 \
162 --threshold "$THRESHOLD_P99_LATENCY_MS" \
163 --comparison-operator GreaterThanThreshold \
164 --treat-missing-data notBreaching \
165 --extended-statistic "p99"
166
167 log "Alarms created: musehub-${ENV}-5xx-rate-high, musehub-${ENV}-p99-latency-high"
168
169 # ── Not done here — needs its own follow-up, not a "quick win" ──────────────
170 #
171 # Disk/memory/DB-connection alarms need the CloudWatch Agent installed and
172 # configured on the instance (it is not installed today), publishing to the
173 # CWAgent namespace. The previous version of this script had an alarm
174 # pointing at AWS/RDS DatabaseConnections — this project uses self-hosted
175 # Postgres in a Docker container on EC2, not RDS; that alarm could never
176 # fire and has been removed rather than left silently broken. A real
177 # self-hosted-Postgres connection alarm needs the CloudWatch Agent's
178 # procstat plugin or a custom metric pushed from inside the container.
179 #
180 # CloudWatch Agent install (Ubuntu — the previous version of this script
181 # incorrectly said `yum`, which doesn't exist on these instances):
182 # curl -O https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
183 # sudo dpkg -i amazon-cloudwatch-agent.deb
184 # sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
185 # -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
186
187 # ── Step 5: Summary ───────────────────────────────────────────────────────────
188
189 log "[5/5] Done."
190 log ""
191 log " Environment: $ENV"
192 log " Log group : $LOG_GROUP (${LOG_RETENTION_DAYS}d retention)"
193 log " SNS topic : $SNS_ARN (subscribers: $ALERT_EMAILS)"
194 log " Alarms : 5xx rate > ${THRESHOLD_5XX_RATE}% | p99 > ${THRESHOLD_P99_LATENCY_MS}ms"
195 log ""
196 log " Verify with: aws cloudwatch describe-alarms --alarm-name-prefix musehub-${ENV} --region $AWS_REGION"
File History 1 commit
sha256:7afc59b2ee9c70d18b748d269602c588385d6e936cc22b056fd775ab9f2fd499 fix: real CloudWatch alerting on both envs, add environment… Sonnet 5 minor ⚠ 22 hours ago