deploy/deploy.sh · gabriel/musehub

1

#!/usr/bin/env bash

2

# Zero-downtime blue-green deploy for MuseHub.

3

#

4

# Strategy:

5

# Two slots — blue (port 1337) and green (port 1338).

6

# The active slot serves traffic via nginx. The inactive slot is stopped.

7

# Deploy:

8

# 1. Pull the new image from ECR (old slot keeps serving).

9

# 2. Run migrations against the live DB (before swap — forward-compatible).

10

# 3. Start the inactive slot with the new image.

11

# 4. Health-check the new slot.

12

# 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime).

13

# 6. Stop the old slot.

14

#

15

# Called by deploy/push.sh via SSM — do not run directly in production.

16

# For manual use on the instance (ECR_IMAGE must match the account this

17

# instance lives in — Nonproduction for staging, Production for prod):

18

# ECR_IMAGE=<account-id>.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \

19

# IMAGE_TAG=<tag> bash deploy/deploy.sh

20

#

21

# First-time setup:

22

# bash deploy/deploy.sh --init

23

# (Initialises .active-slot and /etc/nginx/musehub-active-port if missing)

set -euo pipefail

APP_DIR="/opt/musehub"

28

DEPLOY_LOG="/tmp/musehub-deploy.log"

29

30

# Tee all output to a log file so push.sh can stream it live via a second SSM call.

31

exec > >(tee -a "$DEPLOY_LOG") 2>&1

32

echo "" >> "$DEPLOY_LOG"

33

echo "=== deploy started at $(date -u '+%Y-%m-%dT%H:%M:%SZ') ===" >> "$DEPLOY_LOG"

34

SLOT_FILE="$APP_DIR/.active-slot"

35

NGINX_PORT_FILE="/etc/nginx/musehub-active-port"

36

# Default to staging's registry for manual on-instance invocation with no

37

# ECR_IMAGE set. push.sh always passes a fully-qualified, per-environment

38

# ECR_IMAGE (see its ECR_REGISTRY associative array) — this default only

39

# matters when someone runs this script by hand.

40

ECR_IMAGE="${ECR_IMAGE:-992382692655.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub}"

41

IMAGE_TAG="${IMAGE_TAG:-latest}"

42

# Derived from ECR_IMAGE, never independently hardcoded — docker login must

43

# always authenticate against the same registry docker pull is about to use.

44

# musehub#194: a separately hardcoded ECR_REGISTRY here (always Nonproduction)

45

# silently drifted from ECR_IMAGE's actual registry once "prod" deploys

46

# started passing a real Production image, so a "prod" deploy authenticated

47

# to Nonproduction while trying to pull from Production.

48

ECR_REGISTRY="${ECR_IMAGE%%/*}"

49

MUSEHUB_ENV="${MUSEHUB_ENV:-staging}"

50

# Ceiling for the app/worker containers' memory cgroup. Unpacking a pushed

51

# mpack currently loads the whole payload + all decoded objects into memory

52

# at once (no streaming) — repos with large mpacks (~300MB+) can OOM at the

53

# default 2g. 3g leaves real headroom on this instance's 3.7G total without

54

# starving postgres. This is a stopgap, not a fix for the underlying

55

# non-streaming unpack path (see production-readiness follow-up).

56

APP_MEMORY_LIMIT="${APP_MEMORY_LIMIT:-3g}"

57

FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"

58

REGION="us-east-1"

59

HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz"

60

HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz"

61

HEALTH_RETRIES=30 # × 2s = 60s max wait

cd "$APP_DIR"

# ── Helpers ───────────────────────────────────────────────────────────────────

66

67

log() { echo "[deploy] $*"; }

68

die() { echo "[deploy] ERROR: $*" >&2; exit 1; }

health_check() {

local url="$1"

local slot="$2"

log "Health-checking $slot at $url ..."

74

for i in $(seq 1 "$HEALTH_RETRIES"); do

75

if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then

76

log "$slot is healthy (attempt $i)"

return 0

fi

sleep 2

done

die "$slot failed health check after $((HEALTH_RETRIES * 2))s"

}

nginx_point_to() {

local slot="$1"

sudo musehub-set-slot "$slot"

87

log "nginx now pointing to $slot"

88

}

89

90

# Repair the active-port file if it contains a bare port number instead of

91

# a full nginx upstream directive. Called once at startup so a botched

92

# manual intervention cannot be the root cause of a new deploy failing.

93

sanitize_nginx_port_file() {

94

[ -f "$NGINX_PORT_FILE" ] || return 0

95

local content

96

content=$(cat "$NGINX_PORT_FILE")

97

# Already correct — nothing to do

98

if echo "$content" | grep -qE '^server 127\.0\.0\.1:[0-9]+;$'; then

99

return 0

100

fi

101

# Derive correct slot from .active-slot file, or fall back to blue

102

local slot

103

slot=$(cat "$SLOT_FILE" 2>/dev/null || echo "blue")

104

if [ "$slot" != "blue" ] && [ "$slot" != "green" ]; then

105

slot="blue"

106

fi

107

log "WARNING: $NGINX_PORT_FILE has unexpected content — correcting via musehub-set-slot $slot"

108

sudo musehub-set-slot "$slot"

109

log "Sanitized active-port file; nginx reloaded."

110

}

111

112

# ── Init mode ─────────────────────────────────────────────────────────────────

113

114

if [ "${1:-}" = "--init" ]; then

115

log "Init: installing musehub-set-slot and pointing nginx to blue"

116

sudo cp "$APP_DIR/deploy/set-active-slot.sh" /usr/local/bin/musehub-set-slot

117

sudo chmod +x /usr/local/bin/musehub-set-slot

118

sudo musehub-set-slot blue

119

log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy."

exit 0

fi

# ── Validate required env vars ────────────────────────────────────────────────

124

125

[ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set."

126

[ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set."

127

128

# ── Read active slot ──────────────────────────────────────────────────────────

129

130

if [ ! -f "$SLOT_FILE" ]; then

131

die ".active-slot not found. Run: bash deploy/deploy.sh --init"

132

fi

133

134

ACTIVE_SLOT=$(cat "$SLOT_FILE")

135

if [ "$ACTIVE_SLOT" = "blue" ]; then

136

NEW_SLOT="green"

137

NEW_PORT=1338

138

OLD_CONTAINER="musehub-blue"

139

NEW_CONTAINER="musehub-green"

140

HEALTH_URL="$HEALTH_URL_GREEN"

else

NEW_SLOT="blue"

NEW_PORT=1337

OLD_CONTAINER="musehub-green"

145

NEW_CONTAINER="musehub-blue"

146

HEALTH_URL="$HEALTH_URL_BLUE"

147

fi

148

149

log "Image: $FULL_IMAGE"

150

log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)"

151

152

# Guard: ensure the nginx upstream file is well-formed before we touch anything.

153

sanitize_nginx_port_file

154

155

# ── Step 0: Apply nginx config if updated ────────────────────────────────────

156

# Determine the domain from the current installed config, re-substitute, and

157

# reload nginx if the content changed. Safe to run on every deploy.

158

159

NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf"

160

NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging"

161

NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub"

162

163

if [ -f "$NGINX_CONF_SRC" ]; then

164

# Detect which installed config exists (staging vs prod)

165

if [ -f "$NGINX_CONF_DEST" ]; then

166

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST"

167

elif [ -f "$NGINX_CONF_DEST_PROD" ]; then

168

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD"

169

else

170

NGINX_CONF_INSTALLED=""

171

fi

172

173

if [ -n "$NGINX_CONF_INSTALLED" ]; then

174

# Extract domain from the installed config (first server_name line)

175

DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';')

176

if [ -n "$DOMAIN" ]; then

177

NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC")

178

CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED")

179

if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then

180

log "[0/6] nginx config changed — applying update for $DOMAIN..."

181

echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

182

if sudo nginx -t 2>&1; then

183

sudo nginx -s reload

184

log "nginx config updated and reloaded."

185

else

186

log "WARNING: new nginx config failed validation — reverting."

187

echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

188

fi

189

else

190

log "[0/6] nginx config unchanged — skipping reload."

fi

fi

fi

fi

# ── Step 1: Login to ECR and pull new image ───────────────────────────────────

197

# Retries the full login+pull cycle (not just the pull) since a stale/expired

198

# token is the failure mode seen in practice ("Your authorization token has

199

# expired" immediately after a successful `docker login`) -- re-fetching a

200

# fresh token from scratch on each attempt is the fix, not just retrying the

201

# pull with the same (possibly bad) token.

202

203

log "[1/6] Pulling image from ECR..."

204

PULL_ATTEMPTS=3

205

for attempt in $(seq 1 "$PULL_ATTEMPTS"); do

206

# `docker logout` before each attempt: observed in practice that retrying

207

# login+pull within the *same* invocation can keep hitting the same stale

208

# cached credential state, while a `docker logout` first (clearing

209

# ~/.docker/config.json's entry for this registry) reliably unblocks it --

210

# equivalent to what a completely separate, later invocation was doing by

211

# accident.

212

sudo docker logout "$ECR_REGISTRY" >/dev/null 2>&1 || true

213

if aws ecr get-login-password --region "$REGION" | \

214

sudo docker login --username AWS --password-stdin "$ECR_REGISTRY" \

215

&& sudo docker pull "$FULL_IMAGE"; then

216

break

217

fi

218

if [ "$attempt" -eq "$PULL_ATTEMPTS" ]; then

219

die "ECR login/pull failed after $PULL_ATTEMPTS attempts."

220

fi

221

log "ECR login/pull failed (attempt $attempt/$PULL_ATTEMPTS) — retrying in 5s..."

sleep 5

done

log "Pull complete."

# ── Step 2: Run migrations against the live DB ────────────────────────────────

227

228

log "[2/6] Running migrations..."

229

230

_alembic() {

231

sudo docker run --rm \

232

--network musehub_musehub-internal \

233

--env-file "$APP_DIR/.env" \

234

-e SKIP_MIGRATIONS=0 \

"$FULL_IMAGE" "$@"

}

# If upgrade head fails (e.g. stale revision ID from a migration history reset),

239

# stamp to the current head to re-anchor Alembic's tracking, then retry.

240

# The retry is a no-op when the schema already matches head.

241

if ! _alembic alembic upgrade head; then

242

log "upgrade head failed — re-anchoring Alembic revision to head and retrying..."

243

_alembic alembic stamp --purge head

244

_alembic alembic upgrade head

245

fi

246

log "Migrations complete."

247

248

# Schema parity gate — hard fail. Uses the same benign-diff filter as the S2

249

# test (alembic_version table, semantically-equivalent server_default variants,

250

# column comments) so spurious false positives never block a deploy.

251

_alembic python -m musehub.db.schema_gate \

252

|| die "Schema gate failed — ORM drift detected. Write a migration (alembic revision --autogenerate) before deploying."

253

254

# ── Step 3: Start the new slot ────────────────────────────────────────────────

255

256

log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..."

257

258

# Remove if a failed previous deploy left it around

259

sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true

260

261

sudo docker run -d \

262

--name "$NEW_CONTAINER" \

263

--network musehub_musehub-internal \

264

--env-file "$APP_DIR/.env" \

265

-e SKIP_MIGRATIONS=1 \

266

-e RELEASE_VERSION="${IMAGE_TAG}" \

267

-v musehub_data:/data \

268

-p "127.0.0.1:${NEW_PORT}:1337" \

269

--restart unless-stopped \

270

--memory "$APP_MEMORY_LIMIT" \

271

--log-driver awslogs \

272

--log-opt awslogs-region=us-east-1 \

273

--log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \

274

--log-opt awslogs-stream="$NEW_CONTAINER" \

275

--log-opt awslogs-create-group=true \

276

"$FULL_IMAGE"

277

278

# ── Step 4: Health-check the new slot ────────────────────────────────────────

279

280

health_check "$HEALTH_URL" "$NEW_SLOT"

281

282

# ── Step 5: Flip nginx to the new slot (instant, zero downtime) ───────────────

283

284

log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..."

285

nginx_point_to "$NEW_SLOT"

286

287

# ── Step 6: Stop the old slot ────────────────────────────────────────────────

288

289

log "[6/6] Stopping old slot ($ACTIVE_SLOT)..."

290

# `docker stop` sends SIGTERM and waits (--time) before SIGKILL, giving the

291

# app's lifespan shutdown handler (closes the DB pool, stops the Playwright

292

# browser) a chance to actually run, and letting any in-flight requests that

293

# were accepted just before the nginx flip finish rather than being dropped.

294

# `docker rm -f` (the previous behavior) sends SIGKILL immediately and skips

295

# all of that — the graceful-shutdown code existed but was never triggered.

296

sudo docker stop --time 15 "$OLD_CONTAINER" 2>/dev/null || true

297

sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true

298

299

# ── Step 7: Restart the background worker ────────────────────────────────────

300

301

log "[7/7] Restarting background worker..."

302

sudo docker stop --time 15 musehub-worker 2>/dev/null || true

303

sudo docker rm -f musehub-worker 2>/dev/null || true

304

sudo docker run -d \

305

--name musehub-worker \

306

--network musehub_musehub-internal \

307

--env-file "$APP_DIR/.env" \

308

-e SKIP_MIGRATIONS=1 \

309

-e RELEASE_VERSION="${IMAGE_TAG}" \

310

-v musehub_data:/data \

311

--restart unless-stopped \

312

--no-healthcheck \

313

--memory "$APP_MEMORY_LIMIT" \

314

--log-driver awslogs \

315

--log-opt awslogs-region=us-east-1 \

316

--log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \

317

--log-opt awslogs-stream=musehub-worker \

318

--log-opt awslogs-create-group=true \

319

"$FULL_IMAGE" python -m musehub.worker

320

log "Worker started."

321

322

# ── Step 8: Prune old images (keep last 3) ───────────────────────────────────

323

324

log "[8/8] Pruning old images (keeping last 3)..."

325

KEEP_IMAGES=3

326

OLD_IDS=$(sudo docker images "$ECR_IMAGE" --format "{{.ID}}" \

327

| awk '!seen[$0]++' \

328

| tail -n +$((KEEP_IMAGES + 1)))

329

if [ -n "$OLD_IDS" ]; then

330

echo "$OLD_IDS" | xargs sudo docker rmi -f 2>/dev/null || true

331

log "Image prune complete."

332

else

333

log "No old images to prune."

fi

log ""

log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)"

338

log "Image: $FULL_IMAGE"