gabriel / musehub public
deploy.sh bash
328 lines 13.7 KB
Raw
sha256:8a1389ae62d0a688d5e027763249bd8faedfc39ab00857d65da6620d1ab8a689 Merge 'feat/9a-4-f7-overseer-provenance' into 'dev' — propo… Human 19 hours ago
1 #!/usr/bin/env bash
2 # Zero-downtime blue-green deploy for MuseHub.
3 #
4 # Strategy:
5 # Two slots — blue (port 1337) and green (port 1338).
6 # The active slot serves traffic via nginx. The inactive slot is stopped.
7 # Deploy:
8 # 1. Pull the new image from ECR (old slot keeps serving).
9 # 2. Run migrations against the live DB (before swap — forward-compatible).
10 # 3. Start the inactive slot with the new image.
11 # 4. Health-check the new slot.
12 # 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime).
13 # 6. Stop the old slot.
14 #
15 # Called by deploy/push.sh via SSM — do not run directly in production.
16 # For manual use on the instance (ECR_IMAGE must match the account this
17 # instance lives in — Nonproduction for staging, Production for prod):
18 # ECR_IMAGE=<account-id>.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \
19 # IMAGE_TAG=<tag> bash deploy/deploy.sh
20 #
21 # First-time setup:
22 # bash deploy/deploy.sh --init
23 # (Initialises .active-slot and /etc/nginx/musehub-active-port if missing)
24
25 set -euo pipefail
26
27 APP_DIR="/opt/musehub"
28 DEPLOY_LOG="/tmp/musehub-deploy.log"
29
30 # Tee all output to a log file so push.sh can stream it live via a second SSM call.
31 exec > >(tee -a "$DEPLOY_LOG") 2>&1
32 echo "" >> "$DEPLOY_LOG"
33 echo "=== deploy started at $(date -u '+%Y-%m-%dT%H:%M:%SZ') ===" >> "$DEPLOY_LOG"
34 SLOT_FILE="$APP_DIR/.active-slot"
35 NGINX_PORT_FILE="/etc/nginx/musehub-active-port"
36 ECR_REGISTRY="992382692655.dkr.ecr.us-east-1.amazonaws.com"
37 ECR_IMAGE="${ECR_IMAGE:-${ECR_REGISTRY}/musehub/musehub}"
38 IMAGE_TAG="${IMAGE_TAG:-latest}"
39 MUSEHUB_ENV="${MUSEHUB_ENV:-staging}"
40 # Ceiling for the app/worker containers' memory cgroup. Unpacking a pushed
41 # mpack currently loads the whole payload + all decoded objects into memory
42 # at once (no streaming) — repos with large mpacks (~300MB+) can OOM at the
43 # default 2g. 3g leaves real headroom on this instance's 3.7G total without
44 # starving postgres. This is a stopgap, not a fix for the underlying
45 # non-streaming unpack path (see production-readiness follow-up).
46 APP_MEMORY_LIMIT="${APP_MEMORY_LIMIT:-3g}"
47 FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"
48 REGION="us-east-1"
49 HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz"
50 HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz"
51 HEALTH_RETRIES=30 # × 2s = 60s max wait
52
53 cd "$APP_DIR"
54
55 # ── Helpers ───────────────────────────────────────────────────────────────────
56
57 log() { echo "[deploy] $*"; }
58 die() { echo "[deploy] ERROR: $*" >&2; exit 1; }
59
60 health_check() {
61 local url="$1"
62 local slot="$2"
63 log "Health-checking $slot at $url ..."
64 for i in $(seq 1 "$HEALTH_RETRIES"); do
65 if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then
66 log "$slot is healthy (attempt $i)"
67 return 0
68 fi
69 sleep 2
70 done
71 die "$slot failed health check after $((HEALTH_RETRIES * 2))s"
72 }
73
74 nginx_point_to() {
75 local slot="$1"
76 sudo musehub-set-slot "$slot"
77 log "nginx now pointing to $slot"
78 }
79
80 # Repair the active-port file if it contains a bare port number instead of
81 # a full nginx upstream directive. Called once at startup so a botched
82 # manual intervention cannot be the root cause of a new deploy failing.
83 sanitize_nginx_port_file() {
84 [ -f "$NGINX_PORT_FILE" ] || return 0
85 local content
86 content=$(cat "$NGINX_PORT_FILE")
87 # Already correct — nothing to do
88 if echo "$content" | grep -qE '^server 127\.0\.0\.1:[0-9]+;$'; then
89 return 0
90 fi
91 # Derive correct slot from .active-slot file, or fall back to blue
92 local slot
93 slot=$(cat "$SLOT_FILE" 2>/dev/null || echo "blue")
94 if [ "$slot" != "blue" ] && [ "$slot" != "green" ]; then
95 slot="blue"
96 fi
97 log "WARNING: $NGINX_PORT_FILE has unexpected content — correcting via musehub-set-slot $slot"
98 sudo musehub-set-slot "$slot"
99 log "Sanitized active-port file; nginx reloaded."
100 }
101
102 # ── Init mode ─────────────────────────────────────────────────────────────────
103
104 if [ "${1:-}" = "--init" ]; then
105 log "Init: installing musehub-set-slot and pointing nginx to blue"
106 sudo cp "$APP_DIR/deploy/set-active-slot.sh" /usr/local/bin/musehub-set-slot
107 sudo chmod +x /usr/local/bin/musehub-set-slot
108 sudo musehub-set-slot blue
109 log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy."
110 exit 0
111 fi
112
113 # ── Validate required env vars ────────────────────────────────────────────────
114
115 [ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set."
116 [ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set."
117
118 # ── Read active slot ──────────────────────────────────────────────────────────
119
120 if [ ! -f "$SLOT_FILE" ]; then
121 die ".active-slot not found. Run: bash deploy/deploy.sh --init"
122 fi
123
124 ACTIVE_SLOT=$(cat "$SLOT_FILE")
125 if [ "$ACTIVE_SLOT" = "blue" ]; then
126 NEW_SLOT="green"
127 NEW_PORT=1338
128 OLD_CONTAINER="musehub-blue"
129 NEW_CONTAINER="musehub-green"
130 HEALTH_URL="$HEALTH_URL_GREEN"
131 else
132 NEW_SLOT="blue"
133 NEW_PORT=1337
134 OLD_CONTAINER="musehub-green"
135 NEW_CONTAINER="musehub-blue"
136 HEALTH_URL="$HEALTH_URL_BLUE"
137 fi
138
139 log "Image: $FULL_IMAGE"
140 log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)"
141
142 # Guard: ensure the nginx upstream file is well-formed before we touch anything.
143 sanitize_nginx_port_file
144
145 # ── Step 0: Apply nginx config if updated ────────────────────────────────────
146 # Determine the domain from the current installed config, re-substitute, and
147 # reload nginx if the content changed. Safe to run on every deploy.
148
149 NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf"
150 NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging"
151 NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub"
152
153 if [ -f "$NGINX_CONF_SRC" ]; then
154 # Detect which installed config exists (staging vs prod)
155 if [ -f "$NGINX_CONF_DEST" ]; then
156 NGINX_CONF_INSTALLED="$NGINX_CONF_DEST"
157 elif [ -f "$NGINX_CONF_DEST_PROD" ]; then
158 NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD"
159 else
160 NGINX_CONF_INSTALLED=""
161 fi
162
163 if [ -n "$NGINX_CONF_INSTALLED" ]; then
164 # Extract domain from the installed config (first server_name line)
165 DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';')
166 if [ -n "$DOMAIN" ]; then
167 NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC")
168 CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED")
169 if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then
170 log "[0/6] nginx config changed — applying update for $DOMAIN..."
171 echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null
172 if sudo nginx -t 2>&1; then
173 sudo nginx -s reload
174 log "nginx config updated and reloaded."
175 else
176 log "WARNING: new nginx config failed validation — reverting."
177 echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null
178 fi
179 else
180 log "[0/6] nginx config unchanged — skipping reload."
181 fi
182 fi
183 fi
184 fi
185
186 # ── Step 1: Login to ECR and pull new image ───────────────────────────────────
187 # Retries the full login+pull cycle (not just the pull) since a stale/expired
188 # token is the failure mode seen in practice ("Your authorization token has
189 # expired" immediately after a successful `docker login`) -- re-fetching a
190 # fresh token from scratch on each attempt is the fix, not just retrying the
191 # pull with the same (possibly bad) token.
192
193 log "[1/6] Pulling image from ECR..."
194 PULL_ATTEMPTS=3
195 for attempt in $(seq 1 "$PULL_ATTEMPTS"); do
196 # `docker logout` before each attempt: observed in practice that retrying
197 # login+pull within the *same* invocation can keep hitting the same stale
198 # cached credential state, while a `docker logout` first (clearing
199 # ~/.docker/config.json's entry for this registry) reliably unblocks it --
200 # equivalent to what a completely separate, later invocation was doing by
201 # accident.
202 sudo docker logout "$ECR_REGISTRY" >/dev/null 2>&1 || true
203 if aws ecr get-login-password --region "$REGION" | \
204 sudo docker login --username AWS --password-stdin "$ECR_REGISTRY" \
205 && sudo docker pull "$FULL_IMAGE"; then
206 break
207 fi
208 if [ "$attempt" -eq "$PULL_ATTEMPTS" ]; then
209 die "ECR login/pull failed after $PULL_ATTEMPTS attempts."
210 fi
211 log "ECR login/pull failed (attempt $attempt/$PULL_ATTEMPTS) — retrying in 5s..."
212 sleep 5
213 done
214 log "Pull complete."
215
216 # ── Step 2: Run migrations against the live DB ────────────────────────────────
217
218 log "[2/6] Running migrations..."
219
220 _alembic() {
221 sudo docker run --rm \
222 --network musehub_musehub-internal \
223 --env-file "$APP_DIR/.env" \
224 -e SKIP_MIGRATIONS=0 \
225 "$FULL_IMAGE" "$@"
226 }
227
228 # If upgrade head fails (e.g. stale revision ID from a migration history reset),
229 # stamp to the current head to re-anchor Alembic's tracking, then retry.
230 # The retry is a no-op when the schema already matches head.
231 if ! _alembic alembic upgrade head; then
232 log "upgrade head failed — re-anchoring Alembic revision to head and retrying..."
233 _alembic alembic stamp --purge head
234 _alembic alembic upgrade head
235 fi
236 log "Migrations complete."
237
238 # Schema parity gate — hard fail. Uses the same benign-diff filter as the S2
239 # test (alembic_version table, semantically-equivalent server_default variants,
240 # column comments) so spurious false positives never block a deploy.
241 _alembic python -m musehub.db.schema_gate \
242 || die "Schema gate failed — ORM drift detected. Write a migration (alembic revision --autogenerate) before deploying."
243
244 # ── Step 3: Start the new slot ────────────────────────────────────────────────
245
246 log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..."
247
248 # Remove if a failed previous deploy left it around
249 sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true
250
251 sudo docker run -d \
252 --name "$NEW_CONTAINER" \
253 --network musehub_musehub-internal \
254 --env-file "$APP_DIR/.env" \
255 -e SKIP_MIGRATIONS=1 \
256 -e RELEASE_VERSION="${IMAGE_TAG}" \
257 -v musehub_data:/data \
258 -p "127.0.0.1:${NEW_PORT}:1337" \
259 --restart unless-stopped \
260 --memory "$APP_MEMORY_LIMIT" \
261 --log-driver awslogs \
262 --log-opt awslogs-region=us-east-1 \
263 --log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \
264 --log-opt awslogs-stream="$NEW_CONTAINER" \
265 --log-opt awslogs-create-group=true \
266 "$FULL_IMAGE"
267
268 # ── Step 4: Health-check the new slot ────────────────────────────────────────
269
270 health_check "$HEALTH_URL" "$NEW_SLOT"
271
272 # ── Step 5: Flip nginx to the new slot (instant, zero downtime) ───────────────
273
274 log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..."
275 nginx_point_to "$NEW_SLOT"
276
277 # ── Step 6: Stop the old slot ────────────────────────────────────────────────
278
279 log "[6/6] Stopping old slot ($ACTIVE_SLOT)..."
280 # `docker stop` sends SIGTERM and waits (--time) before SIGKILL, giving the
281 # app's lifespan shutdown handler (closes the DB pool, stops the Playwright
282 # browser) a chance to actually run, and letting any in-flight requests that
283 # were accepted just before the nginx flip finish rather than being dropped.
284 # `docker rm -f` (the previous behavior) sends SIGKILL immediately and skips
285 # all of that — the graceful-shutdown code existed but was never triggered.
286 sudo docker stop --time 15 "$OLD_CONTAINER" 2>/dev/null || true
287 sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true
288
289 # ── Step 7: Restart the background worker ────────────────────────────────────
290
291 log "[7/7] Restarting background worker..."
292 sudo docker stop --time 15 musehub-worker 2>/dev/null || true
293 sudo docker rm -f musehub-worker 2>/dev/null || true
294 sudo docker run -d \
295 --name musehub-worker \
296 --network musehub_musehub-internal \
297 --env-file "$APP_DIR/.env" \
298 -e SKIP_MIGRATIONS=1 \
299 -e RELEASE_VERSION="${IMAGE_TAG}" \
300 -v musehub_data:/data \
301 --restart unless-stopped \
302 --no-healthcheck \
303 --memory "$APP_MEMORY_LIMIT" \
304 --log-driver awslogs \
305 --log-opt awslogs-region=us-east-1 \
306 --log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \
307 --log-opt awslogs-stream=musehub-worker \
308 --log-opt awslogs-create-group=true \
309 "$FULL_IMAGE" python -m musehub.worker
310 log "Worker started."
311
312 # ── Step 8: Prune old images (keep last 3) ───────────────────────────────────
313
314 log "[8/8] Pruning old images (keeping last 3)..."
315 KEEP_IMAGES=3
316 OLD_IDS=$(sudo docker images "$ECR_IMAGE" --format "{{.ID}}" \
317 | awk '!seen[$0]++' \
318 | tail -n +$((KEEP_IMAGES + 1)))
319 if [ -n "$OLD_IDS" ]; then
320 echo "$OLD_IDS" | xargs sudo docker rmi -f 2>/dev/null || true
321 log "Image prune complete."
322 else
323 log "No old images to prune."
324 fi
325
326 log ""
327 log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)"
328 log "Image: $FULL_IMAGE"
File History 19 commits
sha256:8a1389ae62d0a688d5e027763249bd8faedfc39ab00857d65da6620d1ab8a689 Merge 'feat/9a-4-f7-overseer-provenance' into 'dev' — propo… Human 19 hours ago
sha256:bee12c5cbde2334f98421c6c209d768fa6b8004d6705c9ea798ce6c1651bc11f Merge 'infra/database-phase3-4-cleanup' into 'dev' — propos… Human 1 day ago
sha256:23efc08a3fcec5132abb7a4827626dd99f59bf69f64eeeaefad3c1d72a08fe36 Merge 'fix/cloudwatch-alerts-and-log-fields' into 'dev' — p… Human 1 day ago
sha256:9c64dbfd65ef4e8a85f500e5909c06c2e6c65255a69e84188ee73b63f111cecd Merge 'docs/status-banners-closed-tickets' into 'dev' — pro… Human 1 day ago
sha256:5528fe0d7ff3bfcea26d42b3c6e2a7f72127d57444f44cbb23761a869d0961f0 Merge 'docs/multi-remote-workflow' into 'dev' — proposal: d… Human 12 days ago
sha256:20ae2cb8b3425921e96f2131287a153721794504a8d0688f67cc2e3cd158e3ef Merge 'fix/production-backups-r2-bucket' into 'dev' — propo… Human 12 days ago
sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226 Merge branch 'fix/two-column-scroll-layout' into dev Human 64 days ago
sha256:408916fc5973ba59c6e4eebaa80ebdcc801c0a63205651e25009d11548f79454 chore: bump version to 0.2.0.dev2 — nightly.2, matching muse Sonnet 4.6 patch 67 days ago
sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2 chore: bump version to 0.2.0rc15 for musehub#113 fix release Sonnet 4.6 patch 70 days ago
sha256:0032d6cfa33bc3c8367436ad768e7dd0e339b4332153160247da8266cb5fa352 Merge branch 'task/version-tags-phase3-server' into dev Human 72 days ago
sha256:4669620efda9ff41c55bdefd1f7bfe1c239d468428744c84ead9957e5a003a53 merge: rescue snapshot-recovery hardening (c00aa21d) into d… Opus 4.8 minor ⚠ 85 days ago
sha256:a59da49c4611b970fc4b6ae48678ce4943261c213a07ddbd73ce9201df869b4a fix: remove false-positive proposal_comments index drop fro… Sonnet 4.6 patch 89 days ago
sha256:0a240d6dbff234f07d98a28a4a9a68db702f3f9ff9260196f24219bdb1c0b6f3 feat: render markdown mists as HTML with heading anchor links Sonnet 4.6 patch 90 days ago
sha256:24a7d47486ebc4ebd1832830580e177ec6f877b48dced8c000e198cdec4ce9d6 Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump … Human 91 days ago
sha256:b9ff931d147e0114a1f17060f415b89ed551c170a91ff226c70437aa5c85f9ee Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump … Human 91 days ago
sha256:d1122d21e73471879b460037b22c0b50fded7c423444a176f248428f75dac39c Merge 'task/fix-issue-pagination-cursor' into 'dev' — propo… Human 91 days ago
sha256:01e18975e73d2b3cd5b6db7929c895bef9aa6e0d4391dc5b2adfc548b41318dd Merge 'feat/adding-debug-logs-to-staging' into 'dev' — prop… Human 91 days ago
sha256:6b1949fc2797ca4c1936a637a4cbfec828ef56cf52398a2e74ca3c4f494e728f fix: use wire_bytes not mpack_bytes_raw in compute_object_b… Sonnet 4.6 patch 103 days ago
sha256:b99f2455dc346966d040133f5203297e6e3ef5803a93728a2c30568d0a0f7583 rename: delta_add → delta_upsert across wire format, models… Sonnet 4.6 patch 105 days ago