deploy/deploy.sh · gabriel/musehub

1

#!/usr/bin/env bash

2

# Zero-downtime blue-green deploy for MuseHub.

3

#

4

# Strategy:

5

# Two slots — blue (port 1337) and green (port 1338).

6

# The active slot serves traffic via nginx. The inactive slot is stopped.

7

# Deploy:

8

# 1. Pull the new image from ECR (old slot keeps serving).

9

# 2. Run migrations against the live DB (before swap — forward-compatible).

10

# 3. Start the inactive slot with the new image.

11

# 4. Health-check the new slot.

12

# 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime).

13

# 6. Stop the old slot.

14

#

15

# Called by deploy/push.sh via SSM — do not run directly in production.

16

# For manual use on the instance (ECR_IMAGE must match the account this

17

# instance lives in — Nonproduction for staging, Production for prod):

18

# ECR_IMAGE=<account-id>.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \

19

# IMAGE_TAG=<tag> bash deploy/deploy.sh

20

#

21

# First-time setup:

22

# bash deploy/deploy.sh --init

23

# (Initialises .active-slot and /etc/nginx/musehub-active-port if missing)

set -euo pipefail

APP_DIR="/opt/musehub"

28

DEPLOY_LOG="/tmp/musehub-deploy.log"

29

30

# Tee all output to a log file so push.sh can stream it live via a second SSM call.

31

exec > >(tee -a "$DEPLOY_LOG") 2>&1

32

echo "" >> "$DEPLOY_LOG"

33

echo "=== deploy started at $(date -u '+%Y-%m-%dT%H:%M:%SZ') ===" >> "$DEPLOY_LOG"

34

SLOT_FILE="$APP_DIR/.active-slot"

35

NGINX_PORT_FILE="/etc/nginx/musehub-active-port"

36

ECR_REGISTRY="992382692655.dkr.ecr.us-east-1.amazonaws.com"

37

ECR_IMAGE="${ECR_IMAGE:-${ECR_REGISTRY}/musehub/musehub}"

38

IMAGE_TAG="${IMAGE_TAG:-latest}"

39

MUSEHUB_ENV="${MUSEHUB_ENV:-staging}"

40

# Ceiling for the app/worker containers' memory cgroup. Unpacking a pushed

41

# mpack currently loads the whole payload + all decoded objects into memory

42

# at once (no streaming) — repos with large mpacks (~300MB+) can OOM at the

43

# default 2g. 3g leaves real headroom on this instance's 3.7G total without

44

# starving postgres. This is a stopgap, not a fix for the underlying

45

# non-streaming unpack path (see production-readiness follow-up).

46

APP_MEMORY_LIMIT="${APP_MEMORY_LIMIT:-3g}"

47

FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"

48

REGION="us-east-1"

49

HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz"

50

HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz"

51

HEALTH_RETRIES=30 # × 2s = 60s max wait

cd "$APP_DIR"

# ── Helpers ───────────────────────────────────────────────────────────────────

56

57

log() { echo "[deploy] $*"; }

58

die() { echo "[deploy] ERROR: $*" >&2; exit 1; }

health_check() {

local url="$1"

local slot="$2"

log "Health-checking $slot at $url ..."

64

for i in $(seq 1 "$HEALTH_RETRIES"); do

65

if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then

66

log "$slot is healthy (attempt $i)"

return 0

fi

sleep 2

done

die "$slot failed health check after $((HEALTH_RETRIES * 2))s"

}

nginx_point_to() {

local slot="$1"

sudo musehub-set-slot "$slot"

77

log "nginx now pointing to $slot"

78

}

79

80

# Repair the active-port file if it contains a bare port number instead of

81

# a full nginx upstream directive. Called once at startup so a botched

82

# manual intervention cannot be the root cause of a new deploy failing.

83

sanitize_nginx_port_file() {

84

[ -f "$NGINX_PORT_FILE" ] || return 0

85

local content

86

content=$(cat "$NGINX_PORT_FILE")

87

# Already correct — nothing to do

88

if echo "$content" | grep -qE '^server 127\.0\.0\.1:[0-9]+;$'; then

89

return 0

90

fi

91

# Derive correct slot from .active-slot file, or fall back to blue

92

local slot

93

slot=$(cat "$SLOT_FILE" 2>/dev/null || echo "blue")

94

if [ "$slot" != "blue" ] && [ "$slot" != "green" ]; then

95

slot="blue"

96

fi

97

log "WARNING: $NGINX_PORT_FILE has unexpected content — correcting via musehub-set-slot $slot"

98

sudo musehub-set-slot "$slot"

99

log "Sanitized active-port file; nginx reloaded."

100

}

101

102

# ── Init mode ─────────────────────────────────────────────────────────────────

103

104

if [ "${1:-}" = "--init" ]; then

105

log "Init: installing musehub-set-slot and pointing nginx to blue"

106

sudo cp "$APP_DIR/deploy/set-active-slot.sh" /usr/local/bin/musehub-set-slot

107

sudo chmod +x /usr/local/bin/musehub-set-slot

108

sudo musehub-set-slot blue

109

log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy."

exit 0

fi

# ── Validate required env vars ────────────────────────────────────────────────

114

115

[ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set."

116

[ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set."

117

118

# ── Read active slot ──────────────────────────────────────────────────────────

119

120

if [ ! -f "$SLOT_FILE" ]; then

121

die ".active-slot not found. Run: bash deploy/deploy.sh --init"

122

fi

123

124

ACTIVE_SLOT=$(cat "$SLOT_FILE")

125

if [ "$ACTIVE_SLOT" = "blue" ]; then

126

NEW_SLOT="green"

127

NEW_PORT=1338

128

OLD_CONTAINER="musehub-blue"

129

NEW_CONTAINER="musehub-green"

130

HEALTH_URL="$HEALTH_URL_GREEN"

else

NEW_SLOT="blue"

NEW_PORT=1337

OLD_CONTAINER="musehub-green"

135

NEW_CONTAINER="musehub-blue"

136

HEALTH_URL="$HEALTH_URL_BLUE"

137

fi

138

139

log "Image: $FULL_IMAGE"

140

log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)"

141

142

# Guard: ensure the nginx upstream file is well-formed before we touch anything.

143

sanitize_nginx_port_file

144

145

# ── Step 0: Apply nginx config if updated ────────────────────────────────────

146

# Determine the domain from the current installed config, re-substitute, and

147

# reload nginx if the content changed. Safe to run on every deploy.

148

149

NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf"

150

NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging"

151

NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub"

152

153

if [ -f "$NGINX_CONF_SRC" ]; then

154

# Detect which installed config exists (staging vs prod)

155

if [ -f "$NGINX_CONF_DEST" ]; then

156

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST"

157

elif [ -f "$NGINX_CONF_DEST_PROD" ]; then

158

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD"

159

else

160

NGINX_CONF_INSTALLED=""

161

fi

162

163

if [ -n "$NGINX_CONF_INSTALLED" ]; then

164

# Extract domain from the installed config (first server_name line)

165

DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';')

166

if [ -n "$DOMAIN" ]; then

167

NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC")

168

CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED")

169

if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then

170

log "[0/6] nginx config changed — applying update for $DOMAIN..."

171

echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

172

if sudo nginx -t 2>&1; then

173

sudo nginx -s reload

174

log "nginx config updated and reloaded."

175

else

176

log "WARNING: new nginx config failed validation — reverting."

177

echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

178

fi

179

else

180

log "[0/6] nginx config unchanged — skipping reload."

fi

fi

fi

fi

# ── Step 1: Login to ECR and pull new image ───────────────────────────────────

187

188

log "[1/6] Pulling image from ECR..."

189

aws ecr get-login-password --region "$REGION" | \

190

sudo docker login --username AWS --password-stdin "$ECR_REGISTRY"

191

sudo docker pull "$FULL_IMAGE"

192

log "Pull complete."

193

194

# ── Step 2: Run migrations against the live DB ────────────────────────────────

195

196

log "[2/6] Running migrations..."

197

198

_alembic() {

199

sudo docker run --rm \

200

--network musehub_musehub-internal \

201

--env-file "$APP_DIR/.env" \

202

-e SKIP_MIGRATIONS=0 \

"$FULL_IMAGE" "$@"

}

# If upgrade head fails (e.g. stale revision ID from a migration history reset),

207

# stamp to the current head to re-anchor Alembic's tracking, then retry.

208

# The retry is a no-op when the schema already matches head.

209

if ! _alembic alembic upgrade head; then

210

log "upgrade head failed — re-anchoring Alembic revision to head and retrying..."

211

_alembic alembic stamp --purge head

212

_alembic alembic upgrade head

213

fi

214

log "Migrations complete."

215

216

# Schema parity gate — hard fail. Uses the same benign-diff filter as the S2

217

# test (alembic_version table, semantically-equivalent server_default variants,

218

# column comments) so spurious false positives never block a deploy.

219

_alembic python -m musehub.db.schema_gate \

220

|| die "Schema gate failed — ORM drift detected. Write a migration (alembic revision --autogenerate) before deploying."

221

222

# ── Step 3: Start the new slot ────────────────────────────────────────────────

223

224

log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..."

225

226

# Remove if a failed previous deploy left it around

227

sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true

228

229

sudo docker run -d \

230

--name "$NEW_CONTAINER" \

231

--network musehub_musehub-internal \

232

--env-file "$APP_DIR/.env" \

233

-e SKIP_MIGRATIONS=1 \

234

-v musehub_data:/data \

235

-p "127.0.0.1:${NEW_PORT}:1337" \

236

--restart unless-stopped \

237

--memory "$APP_MEMORY_LIMIT" \

238

--log-driver awslogs \

239

--log-opt awslogs-region=us-east-1 \

240

--log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \

241

--log-opt awslogs-stream="$NEW_CONTAINER" \

242

--log-opt awslogs-create-group=true \

243

"$FULL_IMAGE"

244

245

# ── Step 4: Health-check the new slot ────────────────────────────────────────

246

247

health_check "$HEALTH_URL" "$NEW_SLOT"

248

249

# ── Step 5: Flip nginx to the new slot (instant, zero downtime) ───────────────

250

251

log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..."

252

nginx_point_to "$NEW_SLOT"

253

254

# ── Step 6: Stop the old slot ────────────────────────────────────────────────

255

256

log "[6/6] Stopping old slot ($ACTIVE_SLOT)..."

257

sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true

258

259

# ── Step 7: Restart the background worker ────────────────────────────────────

260

261

log "[7/7] Restarting background worker..."

262

sudo docker rm -f musehub-worker 2>/dev/null || true

263

sudo docker run -d \

264

--name musehub-worker \

265

--network musehub_musehub-internal \

266

--env-file "$APP_DIR/.env" \

267

-e SKIP_MIGRATIONS=1 \

268

-v musehub_data:/data \

269

--restart unless-stopped \

270

--no-healthcheck \

271

--memory "$APP_MEMORY_LIMIT" \

272

--log-driver awslogs \

273

--log-opt awslogs-region=us-east-1 \

274

--log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \

275

--log-opt awslogs-stream=musehub-worker \

276

--log-opt awslogs-create-group=true \

277

"$FULL_IMAGE" python -m musehub.worker

278

log "Worker started."

279

280

# ── Step 8: Prune old images (keep last 3) ───────────────────────────────────

281

282

log "[8/8] Pruning old images (keeping last 3)..."

283

KEEP_IMAGES=3

284

OLD_IDS=$(sudo docker images "$ECR_IMAGE" --format "{{.ID}}" \

285

| awk '!seen[$0]++' \

286

| tail -n +$((KEEP_IMAGES + 1)))

287

if [ -n "$OLD_IDS" ]; then

288

echo "$OLD_IDS" | xargs sudo docker rmi -f 2>/dev/null || true

289

log "Image prune complete."

290

else

291

log "No old images to prune."

fi

log ""

log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)"

296

log "Image: $FULL_IMAGE"