deploy/deploy.sh · gabriel/musehub

1

#!/usr/bin/env bash

2

# Zero-downtime blue-green deploy for MuseHub.

3

#

4

# Strategy:

5

# Two slots — blue (port 1337) and green (port 1338).

6

# The active slot serves traffic via nginx. The inactive slot is stopped.

7

# Deploy:

8

# 1. Pull the new image from ECR (old slot keeps serving).

9

# 2. Run migrations against the live DB (before swap — forward-compatible).

10

# 3. Start the inactive slot with the new image.

11

# 4. Health-check the new slot.

12

# 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime).

13

# 6. Stop the old slot.

14

#

15

# Called by deploy/push.sh via SSM — do not run directly in production.

16

# For manual use on the instance (ECR_IMAGE must match the account this

17

# instance lives in — Nonproduction for staging, Production for prod):

18

# ECR_IMAGE=<account-id>.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \

19

# IMAGE_TAG=<tag> bash deploy/deploy.sh

20

#

21

# First-time setup:

22

# bash deploy/deploy.sh --init

23

# (Initialises .active-slot and /etc/nginx/musehub-active-port if missing)

set -euo pipefail

APP_DIR="/opt/musehub"

28

DEPLOY_LOG="/tmp/musehub-deploy.log"

29

30

# Tee all output to a log file so push.sh can stream it live via a second SSM call.

31

exec > >(tee -a "$DEPLOY_LOG") 2>&1

32

echo "" >> "$DEPLOY_LOG"

33

echo "=== deploy started at $(date -u '+%Y-%m-%dT%H:%M:%SZ') ===" >> "$DEPLOY_LOG"

34

SLOT_FILE="$APP_DIR/.active-slot"

35

NGINX_PORT_FILE="/etc/nginx/musehub-active-port"

36

ECR_REGISTRY="992382692655.dkr.ecr.us-east-1.amazonaws.com"

37

ECR_IMAGE="${ECR_IMAGE:-${ECR_REGISTRY}/musehub/musehub}"

38

IMAGE_TAG="${IMAGE_TAG:-latest}"

39

FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"

40

REGION="us-east-1"

41

HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz"

42

HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz"

43

HEALTH_RETRIES=30 # × 2s = 60s max wait

cd "$APP_DIR"

# ── Helpers ───────────────────────────────────────────────────────────────────

48

49

log() { echo "[deploy] $*"; }

50

die() { echo "[deploy] ERROR: $*" >&2; exit 1; }

health_check() {

local url="$1"

local slot="$2"

log "Health-checking $slot at $url ..."

56

for i in $(seq 1 "$HEALTH_RETRIES"); do

57

if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then

58

log "$slot is healthy (attempt $i)"

return 0

fi

sleep 2

done

die "$slot failed health check after $((HEALTH_RETRIES * 2))s"

}

nginx_point_to() {

local slot="$1"

sudo musehub-set-slot "$slot"

69

log "nginx now pointing to $slot"

70

}

71

72

# Repair the active-port file if it contains a bare port number instead of

73

# a full nginx upstream directive. Called once at startup so a botched

74

# manual intervention cannot be the root cause of a new deploy failing.

75

sanitize_nginx_port_file() {

76

[ -f "$NGINX_PORT_FILE" ] || return 0

77

local content

78

content=$(cat "$NGINX_PORT_FILE")

79

# Already correct — nothing to do

80

if echo "$content" | grep -qE '^server 127\.0\.0\.1:[0-9]+;$'; then

81

return 0

82

fi

83

# Derive correct slot from .active-slot file, or fall back to blue

84

local slot

85

slot=$(cat "$SLOT_FILE" 2>/dev/null || echo "blue")

86

if [ "$slot" != "blue" ] && [ "$slot" != "green" ]; then

87

slot="blue"

88

fi

89

log "WARNING: $NGINX_PORT_FILE has unexpected content — correcting via musehub-set-slot $slot"

90

sudo musehub-set-slot "$slot"

91

log "Sanitized active-port file; nginx reloaded."

92

}

93

94

# ── Init mode ─────────────────────────────────────────────────────────────────

95

96

if [ "${1:-}" = "--init" ]; then

97

log "Init: installing musehub-set-slot and pointing nginx to blue"

98

sudo cp "$APP_DIR/deploy/set-active-slot.sh" /usr/local/bin/musehub-set-slot

99

sudo chmod +x /usr/local/bin/musehub-set-slot

100

sudo musehub-set-slot blue

101

log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy."

exit 0

fi

# ── Validate required env vars ────────────────────────────────────────────────

106

107

[ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set."

108

[ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set."

109

110

# ── Read active slot ──────────────────────────────────────────────────────────

111

112

if [ ! -f "$SLOT_FILE" ]; then

113

die ".active-slot not found. Run: bash deploy/deploy.sh --init"

114

fi

115

116

ACTIVE_SLOT=$(cat "$SLOT_FILE")

117

if [ "$ACTIVE_SLOT" = "blue" ]; then

118

NEW_SLOT="green"

119

NEW_PORT=1338

120

OLD_CONTAINER="musehub-blue"

121

NEW_CONTAINER="musehub-green"

122

HEALTH_URL="$HEALTH_URL_GREEN"

else

NEW_SLOT="blue"

NEW_PORT=1337

OLD_CONTAINER="musehub-green"

127

NEW_CONTAINER="musehub-blue"

128

HEALTH_URL="$HEALTH_URL_BLUE"

129

fi

130

131

log "Image: $FULL_IMAGE"

132

log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)"

133

134

# Guard: ensure the nginx upstream file is well-formed before we touch anything.

135

sanitize_nginx_port_file

136

137

# ── Step 0: Apply nginx config if updated ────────────────────────────────────

138

# Determine the domain from the current installed config, re-substitute, and

139

# reload nginx if the content changed. Safe to run on every deploy.

140

141

NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf"

142

NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging"

143

NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub"

144

145

if [ -f "$NGINX_CONF_SRC" ]; then

146

# Detect which installed config exists (staging vs prod)

147

if [ -f "$NGINX_CONF_DEST" ]; then

148

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST"

149

elif [ -f "$NGINX_CONF_DEST_PROD" ]; then

150

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD"

151

else

152

NGINX_CONF_INSTALLED=""

153

fi

154

155

if [ -n "$NGINX_CONF_INSTALLED" ]; then

156

# Extract domain from the installed config (first server_name line)

157

DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';')

158

if [ -n "$DOMAIN" ]; then

159

NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC")

160

CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED")

161

if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then

162

log "[0/6] nginx config changed — applying update for $DOMAIN..."

163

echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

164

if sudo nginx -t 2>&1; then

165

sudo nginx -s reload

166

log "nginx config updated and reloaded."

167

else

168

log "WARNING: new nginx config failed validation — reverting."

169

echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

170

fi

171

else

172

log "[0/6] nginx config unchanged — skipping reload."

fi

fi

fi

fi

# ── Step 1: Login to ECR and pull new image ───────────────────────────────────

179

180

log "[1/6] Pulling image from ECR..."

181

aws ecr get-login-password --region "$REGION" | \

182

sudo docker login --username AWS --password-stdin "$ECR_REGISTRY"

183

sudo docker pull "$FULL_IMAGE"

184

log "Pull complete."

185

186

# ── Step 2: Run migrations against the live DB ────────────────────────────────

187

188

log "[2/6] Running migrations..."

189

190

_alembic() {

191

sudo docker run --rm \

192

--network musehub_musehub-internal \

193

--env-file "$APP_DIR/.env" \

194

-e SKIP_MIGRATIONS=0 \

"$FULL_IMAGE" "$@"

}

# If upgrade head fails (e.g. stale revision ID from a migration history reset),

199

# stamp to the current head to re-anchor Alembic's tracking, then retry.

200

# The retry is a no-op when the schema already matches head.

201

if ! _alembic alembic upgrade head; then

202

log "upgrade head failed — re-anchoring Alembic revision to head and retrying..."

203

_alembic alembic stamp --purge head

204

_alembic alembic upgrade head

205

fi

206

log "Migrations complete."

207

208

# Schema parity gate — hard fail. Uses the same benign-diff filter as the S2

209

# test (alembic_version table, semantically-equivalent server_default variants,

210

# column comments) so spurious false positives never block a deploy.

211

_alembic python -m musehub.db.schema_gate \

212

|| die "Schema gate failed — ORM drift detected. Write a migration (alembic revision --autogenerate) before deploying."

213

214

# ── Step 3: Start the new slot ────────────────────────────────────────────────

215

216

log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..."

217

218

# Remove if a failed previous deploy left it around

219

sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true

220

221

sudo docker run -d \

222

--name "$NEW_CONTAINER" \

223

--network musehub_musehub-internal \

224

--env-file "$APP_DIR/.env" \

225

-e SKIP_MIGRATIONS=1 \

226

-v musehub_data:/data \

227

-p "127.0.0.1:${NEW_PORT}:1337" \

228

--restart unless-stopped \

229

--log-driver awslogs \

230

--log-opt awslogs-region=us-east-1 \

231

--log-opt awslogs-group=/musehub/staging \

232

--log-opt awslogs-stream="$NEW_CONTAINER" \

233

--log-opt awslogs-create-group=true \

234

"$FULL_IMAGE"

235

236

# ── Step 4: Health-check the new slot ────────────────────────────────────────

237

238

health_check "$HEALTH_URL" "$NEW_SLOT"

239

240

# ── Step 5: Flip nginx to the new slot (instant, zero downtime) ───────────────

241

242

log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..."

243

nginx_point_to "$NEW_SLOT"

244

245

# ── Step 6: Stop the old slot ────────────────────────────────────────────────

246

247

log "[6/6] Stopping old slot ($ACTIVE_SLOT)..."

248

sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true

249

250

# ── Step 7: Restart the background worker ────────────────────────────────────

251

252

log "[7/7] Restarting background worker..."

253

sudo docker rm -f musehub-worker 2>/dev/null || true

254

sudo docker run -d \

255

--name musehub-worker \

256

--network musehub_musehub-internal \

257

--env-file "$APP_DIR/.env" \

258

-e SKIP_MIGRATIONS=1 \

259

-v musehub_data:/data \

260

--restart unless-stopped \

261

--no-healthcheck \

262

--log-driver awslogs \

263

--log-opt awslogs-region=us-east-1 \

264

--log-opt awslogs-group=/musehub/staging \

265

--log-opt awslogs-stream=musehub-worker \

266

--log-opt awslogs-create-group=true \

267

"$FULL_IMAGE" python -m musehub.worker

268

log "Worker started."

269

270

# ── Step 8: Prune old images (keep last 3) ───────────────────────────────────

271

272

log "[8/8] Pruning old images (keeping last 3)..."

273

KEEP_IMAGES=3

274

OLD_IDS=$(sudo docker images "$ECR_IMAGE" --format "{{.ID}}" \

275

| awk '!seen[$0]++' \

276

| tail -n +$((KEEP_IMAGES + 1)))

277

if [ -n "$OLD_IDS" ]; then

278

echo "$OLD_IDS" | xargs sudo docker rmi -f 2>/dev/null || true

279

log "Image prune complete."

280

else

281

log "No old images to prune."

fi

log ""

log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)"

286

log "Image: $FULL_IMAGE"