deploy.sh
bash
sha256:1221bc046bda400826d150dab864512fec7e57281420893a59522c2f335fc85b
Merge 'fix/production-deploy-target' into 'dev' — proposal:…
Human
3 days ago
| 1 | #!/usr/bin/env bash |
| 2 | # Zero-downtime blue-green deploy for MuseHub. |
| 3 | # |
| 4 | # Strategy: |
| 5 | # Two slots — blue (port 1337) and green (port 1338). |
| 6 | # The active slot serves traffic via nginx. The inactive slot is stopped. |
| 7 | # Deploy: |
| 8 | # 1. Pull the new image from ECR (old slot keeps serving). |
| 9 | # 2. Run migrations against the live DB (before swap — forward-compatible). |
| 10 | # 3. Start the inactive slot with the new image. |
| 11 | # 4. Health-check the new slot. |
| 12 | # 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime). |
| 13 | # 6. Stop the old slot. |
| 14 | # |
| 15 | # Called by deploy/push.sh via SSM — do not run directly in production. |
| 16 | # For manual use on the instance (ECR_IMAGE must match the account this |
| 17 | # instance lives in — Nonproduction for staging, Production for prod): |
| 18 | # ECR_IMAGE=<account-id>.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \ |
| 19 | # IMAGE_TAG=<tag> bash deploy/deploy.sh |
| 20 | # |
| 21 | # First-time setup: |
| 22 | # bash deploy/deploy.sh --init |
| 23 | # (Initialises .active-slot and /etc/nginx/musehub-active-port if missing) |
| 24 | |
| 25 | set -euo pipefail |
| 26 | |
| 27 | APP_DIR="/opt/musehub" |
| 28 | DEPLOY_LOG="/tmp/musehub-deploy.log" |
| 29 | |
| 30 | # Tee all output to a log file so push.sh can stream it live via a second SSM call. |
| 31 | exec > >(tee -a "$DEPLOY_LOG") 2>&1 |
| 32 | echo "" >> "$DEPLOY_LOG" |
| 33 | echo "=== deploy started at $(date -u '+%Y-%m-%dT%H:%M:%SZ') ===" >> "$DEPLOY_LOG" |
| 34 | SLOT_FILE="$APP_DIR/.active-slot" |
| 35 | NGINX_PORT_FILE="/etc/nginx/musehub-active-port" |
| 36 | ECR_REGISTRY="992382692655.dkr.ecr.us-east-1.amazonaws.com" |
| 37 | ECR_IMAGE="${ECR_IMAGE:-${ECR_REGISTRY}/musehub/musehub}" |
| 38 | IMAGE_TAG="${IMAGE_TAG:-latest}" |
| 39 | FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}" |
| 40 | REGION="us-east-1" |
| 41 | HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz" |
| 42 | HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz" |
| 43 | HEALTH_RETRIES=30 # × 2s = 60s max wait |
| 44 | |
| 45 | cd "$APP_DIR" |
| 46 | |
| 47 | # ── Helpers ─────────────────────────────────────────────────────────────────── |
| 48 | |
| 49 | log() { echo "[deploy] $*"; } |
| 50 | die() { echo "[deploy] ERROR: $*" >&2; exit 1; } |
| 51 | |
| 52 | health_check() { |
| 53 | local url="$1" |
| 54 | local slot="$2" |
| 55 | log "Health-checking $slot at $url ..." |
| 56 | for i in $(seq 1 "$HEALTH_RETRIES"); do |
| 57 | if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then |
| 58 | log "$slot is healthy (attempt $i)" |
| 59 | return 0 |
| 60 | fi |
| 61 | sleep 2 |
| 62 | done |
| 63 | die "$slot failed health check after $((HEALTH_RETRIES * 2))s" |
| 64 | } |
| 65 | |
| 66 | nginx_point_to() { |
| 67 | local slot="$1" |
| 68 | sudo musehub-set-slot "$slot" |
| 69 | log "nginx now pointing to $slot" |
| 70 | } |
| 71 | |
| 72 | # Repair the active-port file if it contains a bare port number instead of |
| 73 | # a full nginx upstream directive. Called once at startup so a botched |
| 74 | # manual intervention cannot be the root cause of a new deploy failing. |
| 75 | sanitize_nginx_port_file() { |
| 76 | [ -f "$NGINX_PORT_FILE" ] || return 0 |
| 77 | local content |
| 78 | content=$(cat "$NGINX_PORT_FILE") |
| 79 | # Already correct — nothing to do |
| 80 | if echo "$content" | grep -qE '^server 127\.0\.0\.1:[0-9]+;$'; then |
| 81 | return 0 |
| 82 | fi |
| 83 | # Derive correct slot from .active-slot file, or fall back to blue |
| 84 | local slot |
| 85 | slot=$(cat "$SLOT_FILE" 2>/dev/null || echo "blue") |
| 86 | if [ "$slot" != "blue" ] && [ "$slot" != "green" ]; then |
| 87 | slot="blue" |
| 88 | fi |
| 89 | log "WARNING: $NGINX_PORT_FILE has unexpected content — correcting via musehub-set-slot $slot" |
| 90 | sudo musehub-set-slot "$slot" |
| 91 | log "Sanitized active-port file; nginx reloaded." |
| 92 | } |
| 93 | |
| 94 | # ── Init mode ───────────────────────────────────────────────────────────────── |
| 95 | |
| 96 | if [ "${1:-}" = "--init" ]; then |
| 97 | log "Init: installing musehub-set-slot and pointing nginx to blue" |
| 98 | sudo cp "$APP_DIR/deploy/set-active-slot.sh" /usr/local/bin/musehub-set-slot |
| 99 | sudo chmod +x /usr/local/bin/musehub-set-slot |
| 100 | sudo musehub-set-slot blue |
| 101 | log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy." |
| 102 | exit 0 |
| 103 | fi |
| 104 | |
| 105 | # ── Validate required env vars ──────────────────────────────────────────────── |
| 106 | |
| 107 | [ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set." |
| 108 | [ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set." |
| 109 | |
| 110 | # ── Read active slot ────────────────────────────────────────────────────────── |
| 111 | |
| 112 | if [ ! -f "$SLOT_FILE" ]; then |
| 113 | die ".active-slot not found. Run: bash deploy/deploy.sh --init" |
| 114 | fi |
| 115 | |
| 116 | ACTIVE_SLOT=$(cat "$SLOT_FILE") |
| 117 | if [ "$ACTIVE_SLOT" = "blue" ]; then |
| 118 | NEW_SLOT="green" |
| 119 | NEW_PORT=1338 |
| 120 | OLD_CONTAINER="musehub-blue" |
| 121 | NEW_CONTAINER="musehub-green" |
| 122 | HEALTH_URL="$HEALTH_URL_GREEN" |
| 123 | else |
| 124 | NEW_SLOT="blue" |
| 125 | NEW_PORT=1337 |
| 126 | OLD_CONTAINER="musehub-green" |
| 127 | NEW_CONTAINER="musehub-blue" |
| 128 | HEALTH_URL="$HEALTH_URL_BLUE" |
| 129 | fi |
| 130 | |
| 131 | log "Image: $FULL_IMAGE" |
| 132 | log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)" |
| 133 | |
| 134 | # Guard: ensure the nginx upstream file is well-formed before we touch anything. |
| 135 | sanitize_nginx_port_file |
| 136 | |
| 137 | # ── Step 0: Apply nginx config if updated ──────────────────────────────────── |
| 138 | # Determine the domain from the current installed config, re-substitute, and |
| 139 | # reload nginx if the content changed. Safe to run on every deploy. |
| 140 | |
| 141 | NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf" |
| 142 | NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging" |
| 143 | NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub" |
| 144 | |
| 145 | if [ -f "$NGINX_CONF_SRC" ]; then |
| 146 | # Detect which installed config exists (staging vs prod) |
| 147 | if [ -f "$NGINX_CONF_DEST" ]; then |
| 148 | NGINX_CONF_INSTALLED="$NGINX_CONF_DEST" |
| 149 | elif [ -f "$NGINX_CONF_DEST_PROD" ]; then |
| 150 | NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD" |
| 151 | else |
| 152 | NGINX_CONF_INSTALLED="" |
| 153 | fi |
| 154 | |
| 155 | if [ -n "$NGINX_CONF_INSTALLED" ]; then |
| 156 | # Extract domain from the installed config (first server_name line) |
| 157 | DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';') |
| 158 | if [ -n "$DOMAIN" ]; then |
| 159 | NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC") |
| 160 | CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED") |
| 161 | if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then |
| 162 | log "[0/6] nginx config changed — applying update for $DOMAIN..." |
| 163 | echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null |
| 164 | if sudo nginx -t 2>&1; then |
| 165 | sudo nginx -s reload |
| 166 | log "nginx config updated and reloaded." |
| 167 | else |
| 168 | log "WARNING: new nginx config failed validation — reverting." |
| 169 | echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null |
| 170 | fi |
| 171 | else |
| 172 | log "[0/6] nginx config unchanged — skipping reload." |
| 173 | fi |
| 174 | fi |
| 175 | fi |
| 176 | fi |
| 177 | |
| 178 | # ── Step 1: Login to ECR and pull new image ─────────────────────────────────── |
| 179 | |
| 180 | log "[1/6] Pulling image from ECR..." |
| 181 | aws ecr get-login-password --region "$REGION" | \ |
| 182 | sudo docker login --username AWS --password-stdin "$ECR_REGISTRY" |
| 183 | sudo docker pull "$FULL_IMAGE" |
| 184 | log "Pull complete." |
| 185 | |
| 186 | # ── Step 2: Run migrations against the live DB ──────────────────────────────── |
| 187 | |
| 188 | log "[2/6] Running migrations..." |
| 189 | |
| 190 | _alembic() { |
| 191 | sudo docker run --rm \ |
| 192 | --network musehub_musehub-internal \ |
| 193 | --env-file "$APP_DIR/.env" \ |
| 194 | -e SKIP_MIGRATIONS=0 \ |
| 195 | "$FULL_IMAGE" "$@" |
| 196 | } |
| 197 | |
| 198 | # If upgrade head fails (e.g. stale revision ID from a migration history reset), |
| 199 | # stamp to the current head to re-anchor Alembic's tracking, then retry. |
| 200 | # The retry is a no-op when the schema already matches head. |
| 201 | if ! _alembic alembic upgrade head; then |
| 202 | log "upgrade head failed — re-anchoring Alembic revision to head and retrying..." |
| 203 | _alembic alembic stamp --purge head |
| 204 | _alembic alembic upgrade head |
| 205 | fi |
| 206 | log "Migrations complete." |
| 207 | |
| 208 | # Schema parity gate — hard fail. Uses the same benign-diff filter as the S2 |
| 209 | # test (alembic_version table, semantically-equivalent server_default variants, |
| 210 | # column comments) so spurious false positives never block a deploy. |
| 211 | _alembic python -m musehub.db.schema_gate \ |
| 212 | || die "Schema gate failed — ORM drift detected. Write a migration (alembic revision --autogenerate) before deploying." |
| 213 | |
| 214 | # ── Step 3: Start the new slot ──────────────────────────────────────────────── |
| 215 | |
| 216 | log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..." |
| 217 | |
| 218 | # Remove if a failed previous deploy left it around |
| 219 | sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true |
| 220 | |
| 221 | sudo docker run -d \ |
| 222 | --name "$NEW_CONTAINER" \ |
| 223 | --network musehub_musehub-internal \ |
| 224 | --env-file "$APP_DIR/.env" \ |
| 225 | -e SKIP_MIGRATIONS=1 \ |
| 226 | -v musehub_data:/data \ |
| 227 | -p "127.0.0.1:${NEW_PORT}:1337" \ |
| 228 | --restart unless-stopped \ |
| 229 | --log-driver awslogs \ |
| 230 | --log-opt awslogs-region=us-east-1 \ |
| 231 | --log-opt awslogs-group=/musehub/staging \ |
| 232 | --log-opt awslogs-stream="$NEW_CONTAINER" \ |
| 233 | --log-opt awslogs-create-group=true \ |
| 234 | "$FULL_IMAGE" |
| 235 | |
| 236 | # ── Step 4: Health-check the new slot ──────────────────────────────────────── |
| 237 | |
| 238 | health_check "$HEALTH_URL" "$NEW_SLOT" |
| 239 | |
| 240 | # ── Step 5: Flip nginx to the new slot (instant, zero downtime) ─────────────── |
| 241 | |
| 242 | log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..." |
| 243 | nginx_point_to "$NEW_SLOT" |
| 244 | |
| 245 | # ── Step 6: Stop the old slot ──────────────────────────────────────────────── |
| 246 | |
| 247 | log "[6/6] Stopping old slot ($ACTIVE_SLOT)..." |
| 248 | sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true |
| 249 | |
| 250 | # ── Step 7: Restart the background worker ──────────────────────────────────── |
| 251 | |
| 252 | log "[7/7] Restarting background worker..." |
| 253 | sudo docker rm -f musehub-worker 2>/dev/null || true |
| 254 | sudo docker run -d \ |
| 255 | --name musehub-worker \ |
| 256 | --network musehub_musehub-internal \ |
| 257 | --env-file "$APP_DIR/.env" \ |
| 258 | -e SKIP_MIGRATIONS=1 \ |
| 259 | -v musehub_data:/data \ |
| 260 | --restart unless-stopped \ |
| 261 | --no-healthcheck \ |
| 262 | --log-driver awslogs \ |
| 263 | --log-opt awslogs-region=us-east-1 \ |
| 264 | --log-opt awslogs-group=/musehub/staging \ |
| 265 | --log-opt awslogs-stream=musehub-worker \ |
| 266 | --log-opt awslogs-create-group=true \ |
| 267 | "$FULL_IMAGE" python -m musehub.worker |
| 268 | log "Worker started." |
| 269 | |
| 270 | # ── Step 8: Prune old images (keep last 3) ─────────────────────────────────── |
| 271 | |
| 272 | log "[8/8] Pruning old images (keeping last 3)..." |
| 273 | KEEP_IMAGES=3 |
| 274 | OLD_IDS=$(sudo docker images "$ECR_IMAGE" --format "{{.ID}}" \ |
| 275 | | awk '!seen[$0]++' \ |
| 276 | | tail -n +$((KEEP_IMAGES + 1))) |
| 277 | if [ -n "$OLD_IDS" ]; then |
| 278 | echo "$OLD_IDS" | xargs sudo docker rmi -f 2>/dev/null || true |
| 279 | log "Image prune complete." |
| 280 | else |
| 281 | log "No old images to prune." |
| 282 | fi |
| 283 | |
| 284 | log "" |
| 285 | log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)" |
| 286 | log "Image: $FULL_IMAGE" |
File History
15 commits
sha256:1221bc046bda400826d150dab864512fec7e57281420893a59522c2f335fc85b
Merge 'fix/production-deploy-target' into 'dev' — proposal:…
Human
3 days ago
sha256:20ae2cb8b3425921e96f2131287a153721794504a8d0688f67cc2e3cd158e3ef
Merge 'fix/production-backups-r2-bucket' into 'dev' — propo…
Human
3 days ago
sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226
Merge branch 'fix/two-column-scroll-layout' into dev
Human
55 days ago
sha256:408916fc5973ba59c6e4eebaa80ebdcc801c0a63205651e25009d11548f79454
chore: bump version to 0.2.0.dev2 — nightly.2, matching muse
Sonnet 4.6
patch
58 days ago
sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2
chore: bump version to 0.2.0rc15 for musehub#113 fix release
Sonnet 4.6
patch
61 days ago
sha256:0032d6cfa33bc3c8367436ad768e7dd0e339b4332153160247da8266cb5fa352
Merge branch 'task/version-tags-phase3-server' into dev
Human
63 days ago
sha256:4669620efda9ff41c55bdefd1f7bfe1c239d468428744c84ead9957e5a003a53
merge: rescue snapshot-recovery hardening (c00aa21d) into d…
Opus 4.8
minor
⚠
76 days ago
sha256:a59da49c4611b970fc4b6ae48678ce4943261c213a07ddbd73ce9201df869b4a
fix: remove false-positive proposal_comments index drop fro…
Sonnet 4.6
patch
80 days ago
sha256:0a240d6dbff234f07d98a28a4a9a68db702f3f9ff9260196f24219bdb1c0b6f3
feat: render markdown mists as HTML with heading anchor links
Sonnet 4.6
patch
81 days ago
sha256:24a7d47486ebc4ebd1832830580e177ec6f877b48dced8c000e198cdec4ce9d6
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
82 days ago
sha256:b9ff931d147e0114a1f17060f415b89ed551c170a91ff226c70437aa5c85f9ee
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
82 days ago
sha256:d1122d21e73471879b460037b22c0b50fded7c423444a176f248428f75dac39c
Merge 'task/fix-issue-pagination-cursor' into 'dev' — propo…
Human
82 days ago
sha256:01e18975e73d2b3cd5b6db7929c895bef9aa6e0d4391dc5b2adfc548b41318dd
Merge 'feat/adding-debug-logs-to-staging' into 'dev' — prop…
Human
82 days ago
sha256:6b1949fc2797ca4c1936a637a4cbfec828ef56cf52398a2e74ca3c4f494e728f
fix: use wire_bytes not mpack_bytes_raw in compute_object_b…
Sonnet 4.6
patch
94 days ago
sha256:b99f2455dc346966d040133f5203297e6e3ef5803a93728a2c30568d0a0f7583
rename: delta_add → delta_upsert across wire format, models…
Sonnet 4.6
patch
96 days ago