deploy/cloudwatch-alerts.sh · gabriel/musehub

cloudwatch-alerts.sh bash

196 lines 9.5 KB

sha256:23efc08a3fcec5132abb7a4827626dd99f59bf69f64eeeaefad3c1d72a08fe36 Merge 'fix/cloudwatch-alerts-and-log-fields' into 'dev' — p… Human 18 hours ago

1	#!/usr/bin/env bash
2	# MuseHub — CloudWatch log group, metric filters, alarms, and SNS alerting.
3	#
4	# Run once per environment; safe to re-run (put-metric-alarm is idempotent,
5	# log groups/topics/subscriptions are created only if absent).
6	#
7	# Staging and production are separate AWS accounts with separate log groups
8	# (/musehub/staging, /musehub/production) — this script targets one
9	# environment per run, matching deploy/push.sh's per-environment AWS_PROFILE
10	# pattern. Run it once per environment, under that environment's own SSO
11	# session (aws sso login --profile musehub-nonproduction\|musehub-production).
12	#
13	# Prerequisites:
14	# * AWS CLI configured with permissions for logs:, cloudwatch:, sns:*
15	# (the operator's own SSO session already has this via AdministratorAccess)
16	# * ALERT_EMAILS set in the environment, or accept the default (Gabriel + Aaron)
17	#
18	# Usage:
19	# bash deploy/cloudwatch-alerts.sh --env staging --profile musehub-nonproduction
20	# bash deploy/cloudwatch-alerts.sh --env production --profile musehub-production
21
22	set -euo pipefail
23
24	# ── Config ────────────────────────────────────────────────────────────────────
25
26	AWS_REGION="${AWS_REGION:-us-east-1}"
27	AWS_PROFILE_ARG="${AWS_PROFILE_ARG:-}"
28	ENV=""
29	LOG_RETENTION_DAYS=30 # Hot retention: 30 days in CloudWatch
30	# Route alerts to both maintainers by default — override with ALERT_EMAILS
31	# (comma-separated) if that ever needs to change.
32	ALERT_EMAILS="${ALERT_EMAILS:[email protected],[email protected]}"
33
34	# Alarm thresholds — aligned with the pre-launch checklist and #149's RPO/RTO decisions.
35	THRESHOLD_5XX_RATE="1" # percent: alarm when 5xx / total > 1%
36	THRESHOLD_P99_LATENCY_MS="2000" # milliseconds
37
38	# ── Helpers ───────────────────────────────────────────────────────────────────
39
40	log() { echo "[cloudwatch] $*"; }
41	die() { echo "[cloudwatch] ERROR: $*" >&2; exit 1; }
42
43	aws_cmd() {
44	if [[ -n "$AWS_PROFILE_ARG" ]]; then
45	aws --region "$AWS_REGION" --profile "$AWS_PROFILE_ARG" "$@"
46	else
47	aws --region "$AWS_REGION" "$@"
48	fi
49	}
50
51	# ── Parse args ────────────────────────────────────────────────────────────────
52
53	while [[ $# -gt 0 ]]; do
54	case "$1" in
55	--env) ENV="$2"; shift 2 ;;
56	--profile) AWS_PROFILE_ARG="$2"; shift 2 ;;
57	--region) AWS_REGION="$2"; shift 2 ;;
58	*) die "Unknown argument: $1" ;;
59	esac
60	done
61
62	case "$ENV" in
63	staging\|production) ;;
64	*) die "Usage: bash deploy/cloudwatch-alerts.sh --env staging\|production --profile <sso-profile>" ;;
65	esac
66
67	LOG_GROUP="/musehub/${ENV}"
68	SNS_TOPIC_NAME="musehub-${ENV}-alerts"
69
70	# ── Step 1: Log group + explicit hot retention ───────────────────────────────
71	# The log group should already exist (created by the app's awslogs driver on
72	# first deploy) — this just ensures retention is set, since CloudWatch Logs
73	# default to unlimited retention otherwise (a cost/compliance gap, not an
74	# observability one).
75
76	log "[1/5] Setting ${LOG_RETENTION_DAYS}-day retention on $LOG_GROUP..."
77	aws_cmd logs create-log-group --log-group-name "$LOG_GROUP" 2>/dev/null \|\| true # idempotent
78	aws_cmd logs put-retention-policy \
79	--log-group-name "$LOG_GROUP" \
80	--retention-in-days "$LOG_RETENTION_DAYS"
81
82	# ── Step 2: SNS topic + subscriptions ────────────────────────────────────────
83
84	log "[2/5] Creating SNS topic $SNS_TOPIC_NAME..."
85
86	SNS_ARN=$(aws_cmd sns create-topic \
87	--name "$SNS_TOPIC_NAME" \
88	--query TopicArn --output text)
89
90	log "SNS topic ARN: $SNS_ARN"
91
92	IFS=',' read -ra EMAILS <<< "$ALERT_EMAILS"
93	for email in "${EMAILS[@]}"; do
94	email="$(echo "$email" \| xargs)" # trim whitespace
95	[[ -z "$email" ]] && continue
96	aws_cmd sns subscribe \
97	--topic-arn "$SNS_ARN" \
98	--protocol email \
99	--notification-endpoint "$email" \
100	--query SubscriptionArn --output text > /dev/null
101	log " Email subscription created for $email (must confirm via the email AWS sends)"
102	done
103
104	# ── Step 3: Metric filters (extract from structured JSON logs) ─────────────────
105
106	log "[3/5] Creating metric filters on $LOG_GROUP..."
107
108	aws_cmd logs put-metric-filter \
109	--log-group-name "$LOG_GROUP" \
110	--filter-name "musehub-5xx-count" \
111	--filter-pattern '{ $.status >= 500 }' \
112	--metric-transformations \
113	metricName="5xxCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0"
114
115	aws_cmd logs put-metric-filter \
116	--log-group-name "$LOG_GROUP" \
117	--filter-name "musehub-request-count" \
118	--filter-pattern '{ $.status >= 100 }' \
119	--metric-transformations \
120	metricName="RequestCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0"
121
122	aws_cmd logs put-metric-filter \
123	--log-group-name "$LOG_GROUP" \
124	--filter-name "musehub-duration-ms" \
125	--filter-pattern '{ $.duration_ms > 0 }' \
126	--metric-transformations \
127	metricName="RequestDurationMs-${ENV}",metricNamespace="MuseHub",metricValue='$.duration_ms',defaultValue="0"
128
129	log "Metric filters created: 5xxCount-${ENV}, RequestCount-${ENV}, RequestDurationMs-${ENV}"
130
131	# ── Step 4: CloudWatch Alarms ─────────────────────────────────────────────────
132
133	log "[4/5] Creating CloudWatch alarms (5xx>${THRESHOLD_5XX_RATE}%, p99>${THRESHOLD_P99_LATENCY_MS}ms)..."
134
135	# ─ 4a: 5xx error rate ────────────────────────────────────────────────────────
136	aws_cmd cloudwatch put-metric-alarm \
137	--alarm-name "musehub-${ENV}-5xx-rate-high" \
138	--alarm-description "[$ENV] 5xx error rate exceeded ${THRESHOLD_5XX_RATE}% — investigate immediately" \
139	--alarm-actions "$SNS_ARN" \
140	--ok-actions "$SNS_ARN" \
141	--metrics \
142	"[{\"Id\":\"e1\",\"Expression\":\"(m1/m2)*100\",\"Label\":\"5xxRate\",\"ReturnData\":true},
143	{\"Id\":\"m1\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"5xxCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false},
144	{\"Id\":\"m2\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"RequestCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false}]" \
145	--comparison-operator GreaterThanThreshold \
146	--threshold "$THRESHOLD_5XX_RATE" \
147	--evaluation-periods 2 \
148	--datapoints-to-alarm 2 \
149	--treat-missing-data notBreaching
150
151	# ─ 4b: p99 request latency ───────────────────────────────────────────────────
152	aws_cmd cloudwatch put-metric-alarm \
153	--alarm-name "musehub-${ENV}-p99-latency-high" \
154	--alarm-description "[$ENV] p99 request latency exceeded ${THRESHOLD_P99_LATENCY_MS}ms" \
155	--alarm-actions "$SNS_ARN" \
156	--ok-actions "$SNS_ARN" \
157	--namespace MuseHub \
158	--metric-name "RequestDurationMs-${ENV}" \
159	--period 60 \
160	--evaluation-periods 3 \
161	--datapoints-to-alarm 2 \
162	--threshold "$THRESHOLD_P99_LATENCY_MS" \
163	--comparison-operator GreaterThanThreshold \
164	--treat-missing-data notBreaching \
165	--extended-statistic "p99"
166
167	log "Alarms created: musehub-${ENV}-5xx-rate-high, musehub-${ENV}-p99-latency-high"
168
169	# ── Not done here — needs its own follow-up, not a "quick win" ──────────────
170	#
171	# Disk/memory/DB-connection alarms need the CloudWatch Agent installed and
172	# configured on the instance (it is not installed today), publishing to the
173	# CWAgent namespace. The previous version of this script had an alarm
174	# pointing at AWS/RDS DatabaseConnections — this project uses self-hosted
175	# Postgres in a Docker container on EC2, not RDS; that alarm could never
176	# fire and has been removed rather than left silently broken. A real
177	# self-hosted-Postgres connection alarm needs the CloudWatch Agent's
178	# procstat plugin or a custom metric pushed from inside the container.
179	#
180	# CloudWatch Agent install (Ubuntu — the previous version of this script
181	# incorrectly said `yum`, which doesn't exist on these instances):
182	# curl -O https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
183	# sudo dpkg -i amazon-cloudwatch-agent.deb
184	# sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
185	# -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
186
187	# ── Step 5: Summary ───────────────────────────────────────────────────────────
188
189	log "[5/5] Done."
190	log ""
191	log " Environment: $ENV"
192	log " Log group : $LOG_GROUP (${LOG_RETENTION_DAYS}d retention)"
193	log " SNS topic : $SNS_ARN (subscribers: $ALERT_EMAILS)"
194	log " Alarms : 5xx rate > ${THRESHOLD_5XX_RATE}% \| p99 > ${THRESHOLD_P99_LATENCY_MS}ms"
195	log ""
196	log " Verify with: aws cloudwatch describe-alarms --alarm-name-prefix musehub-${ENV} --region $AWS_REGION"

File History 15 commits

sha256:23efc08a3fcec5132abb7a4827626dd99f59bf69f64eeeaefad3c1d72a08fe36 Merge 'fix/cloudwatch-alerts-and-log-fields' into 'dev' — p… Human 18 hours ago

sha256:9c64dbfd65ef4e8a85f500e5909c06c2e6c65255a69e84188ee73b63f111cecd Merge 'docs/status-banners-closed-tickets' into 'dev' — pro… Human 19 hours ago

sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226 Merge branch 'fix/two-column-scroll-layout' into dev Human 63 days ago

sha256:408916fc5973ba59c6e4eebaa80ebdcc801c0a63205651e25009d11548f79454 chore: bump version to 0.2.0.dev2 — nightly.2, matching muse Sonnet 4.6 patch 66 days ago

sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2 chore: bump version to 0.2.0rc15 for musehub#113 fix release Sonnet 4.6 patch 69 days ago

sha256:0032d6cfa33bc3c8367436ad768e7dd0e339b4332153160247da8266cb5fa352 Merge branch 'task/version-tags-phase3-server' into dev Human 71 days ago

sha256:4669620efda9ff41c55bdefd1f7bfe1c239d468428744c84ead9957e5a003a53 merge: rescue snapshot-recovery hardening (c00aa21d) into d… Opus 4.8 minor ⚠ 84 days ago

sha256:a59da49c4611b970fc4b6ae48678ce4943261c213a07ddbd73ce9201df869b4a fix: remove false-positive proposal_comments index drop fro… Sonnet 4.6 patch 88 days ago

sha256:0a240d6dbff234f07d98a28a4a9a68db702f3f9ff9260196f24219bdb1c0b6f3 feat: render markdown mists as HTML with heading anchor links Sonnet 4.6 patch 89 days ago

sha256:24a7d47486ebc4ebd1832830580e177ec6f877b48dced8c000e198cdec4ce9d6 Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump … Human 90 days ago

sha256:b9ff931d147e0114a1f17060f415b89ed551c170a91ff226c70437aa5c85f9ee Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump … Human 90 days ago

sha256:d1122d21e73471879b460037b22c0b50fded7c423444a176f248428f75dac39c Merge 'task/fix-issue-pagination-cursor' into 'dev' — propo… Human 90 days ago

sha256:01e18975e73d2b3cd5b6db7929c895bef9aa6e0d4391dc5b2adfc548b41318dd Merge 'feat/adding-debug-logs-to-staging' into 'dev' — prop… Human 90 days ago

sha256:6b1949fc2797ca4c1936a637a4cbfec828ef56cf52398a2e74ca3c4f494e728f fix: use wire_bytes not mpack_bytes_raw in compute_object_b… Sonnet 4.6 patch 102 days ago

sha256:b99f2455dc346966d040133f5203297e6e3ef5803a93728a2c30568d0a0f7583 rename: delta_add → delta_upsert across wire format, models… Sonnet 4.6 patch 104 days ago