push.sh
bash
sha256:47c92210e23600ae975e8943d129f6c752fb4d5835fd19b8d74fbc8489ec83f8
fix: deploy build silently served a stale Docker layer for …
Sonnet 5
minor
⚠ breaking
22 hours ago
| 1 | #!/usr/bin/env bash |
| 2 | # MuseHub deploy orchestrator — build, push to ECR, trigger blue-green via SSM. |
| 3 | # |
| 4 | # Usage: |
| 5 | # bash deploy/push.sh staging # deploy to staging only |
| 6 | # bash deploy/push.sh prod # deploy to prod only |
| 7 | # bash deploy/push.sh staging prod # staging first, then prod |
| 8 | # |
| 9 | # What it does: |
| 10 | # 1. Builds a linux/amd64 Docker image from the local repo. |
| 11 | # 2. Tags it with <commit-hash>-<timestamp> for traceability. |
| 12 | # 3. Pushes the image to each target's own ECR (staging and prod are |
| 13 | # separate AWS accounts with separate, non-shared ECR repos). |
| 14 | # 4. Sends an SSM command to each target instance to run deploy.sh, |
| 15 | # which pulls the image and performs a zero-downtime blue-green swap. |
| 16 | # 5. Polls SSM until the deploy completes or fails, streaming the output. |
| 17 | # |
| 18 | # Staging and production are different AWS accounts, each authenticated via |
| 19 | # the operator's own IAM Identity Center SSO session — run `aws sso login |
| 20 | # --profile musehub-nonproduction` (staging) or `--profile musehub-production` |
| 21 | # (prod) first. No shared IAM user is used by either target; the legacy |
| 22 | # `musehub-infra` static credential is retired from this path (kept around |
| 23 | # for now for any scripts that still reference it directly, but no longer |
| 24 | # used by push.sh). |
| 25 | # |
| 26 | # Prerequisites (one-time, already done): |
| 27 | # - `musehub-nonproduction` and `musehub-production` SSO profiles configured |
| 28 | # in ~/.aws/config |
| 29 | # - Docker Desktop running |
| 30 | # - musehub-ec2-ssm / musehub-production-ec2-ssm roles have ecr pull permissions |
| 31 | # - AWS CLI installed on instances (run deploy/bootstrap-instance.sh once) |
| 32 | # |
| 33 | # Rollback to a previous image: |
| 34 | # IMAGE_TAG=<previous-tag> bash deploy/push.sh staging |
| 35 | # (skips build+push, triggers SSM with the specified tag directly) |
| 36 | |
| 37 | set -euo pipefail |
| 38 | |
| 39 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 40 | REPO_DIR="$(dirname "$SCRIPT_DIR")" |
| 41 | ECOSYSTEM_DIR="$(dirname "$REPO_DIR")" |
| 42 | |
| 43 | ECR_REPO="musehub/musehub" |
| 44 | REGION="us-east-1" |
| 45 | |
| 46 | # Per-environment config — staging and prod are separate AWS accounts. |
| 47 | # |
| 48 | # Deliberately `case` dispatch rather than `declare -A`: associative arrays |
| 49 | # require bash 4+, but macOS ships bash 3.2 as /bin/bash with no version |
| 50 | # guard elsewhere in this script, so `declare -A` fails outright for anyone |
| 51 | # running under the system bash rather than a Homebrew-installed bash 4/5. |
| 52 | instance_id_for() { |
| 53 | case "$1" in |
| 54 | staging) echo "i-07547cd20bee2dea5" ;; |
| 55 | prod) echo "i-043aaed71bef11903" ;; |
| 56 | esac |
| 57 | } |
| 58 | ecr_registry_for() { |
| 59 | case "$1" in |
| 60 | staging) echo "992382692655.dkr.ecr.us-east-1.amazonaws.com" ;; |
| 61 | prod) echo "672469410277.dkr.ecr.us-east-1.amazonaws.com" ;; |
| 62 | esac |
| 63 | } |
| 64 | aws_profile_for() { |
| 65 | case "$1" in |
| 66 | staging) echo "musehub-nonproduction" ;; # operator's own SSO session |
| 67 | prod) echo "musehub-production" ;; # operator's own SSO session |
| 68 | esac |
| 69 | } |
| 70 | |
| 71 | # ── Parse targets ───────────────────────────────────────────────────────────── |
| 72 | |
| 73 | if [ $# -eq 0 ]; then |
| 74 | echo "Usage: bash deploy/push.sh [staging] [prod]" |
| 75 | echo " staging deploy to staging.musehub.ai" |
| 76 | echo " prod deploy to musehub.ai" |
| 77 | echo " staging prod staging first, then prod" |
| 78 | echo "" |
| 79 | echo "Rollback (skips build+push, redeploys a previous tag):" |
| 80 | echo " IMAGE_TAG=<tag> bash deploy/push.sh staging" |
| 81 | exit 1 |
| 82 | fi |
| 83 | |
| 84 | TARGETS=() |
| 85 | for arg in "$@"; do |
| 86 | case "$arg" in |
| 87 | staging|prod) TARGETS+=("$arg") ;; |
| 88 | *) echo "Unknown target: $arg (must be staging or prod)" >&2; exit 1 ;; |
| 89 | esac |
| 90 | done |
| 91 | |
| 92 | # ── Helpers ─────────────────────────────────────────────────────────────────── |
| 93 | |
| 94 | log() { echo "[push] $*"; } |
| 95 | die() { echo "[push] ERROR: $*" >&2; exit 1; } |
| 96 | |
| 97 | profile_args() { |
| 98 | local env="$1" |
| 99 | local profile |
| 100 | profile="$(aws_profile_for "$env")" |
| 101 | if [ -n "$profile" ]; then |
| 102 | echo "--profile $profile" |
| 103 | fi |
| 104 | } |
| 105 | |
| 106 | check_auth() { |
| 107 | local env="$1" |
| 108 | local profile |
| 109 | profile="$(aws_profile_for "$env")" |
| 110 | if [ -n "$profile" ]; then |
| 111 | if ! aws sts get-caller-identity --profile "$profile" > /dev/null 2>&1; then |
| 112 | die "No valid SSO session for profile '$profile'. Run: aws sso login --profile $profile" |
| 113 | fi |
| 114 | fi |
| 115 | } |
| 116 | |
| 117 | # ── Image tag ───────────────────────────────────────────────────────────────── |
| 118 | |
| 119 | # If IMAGE_TAG is already set (rollback mode), skip build+push. |
| 120 | if [ -n "${IMAGE_TAG:-}" ]; then |
| 121 | log "Rollback mode — using existing tag: $IMAGE_TAG" |
| 122 | SKIP_BUILD=true |
| 123 | else |
| 124 | COMMIT_HASH=$(muse -C "$REPO_DIR" rev-parse HEAD --json 2>/dev/null \ |
| 125 | | python3 -c "import sys,json; cid=json.load(sys.stdin)['commit_id']; print(cid.removeprefix('sha256:')[:8])" \ |
| 126 | 2>/dev/null || echo "local") |
| 127 | IMAGE_TAG="${COMMIT_HASH}-$(date +%Y%m%d%H%M%S)" |
| 128 | SKIP_BUILD=false |
| 129 | fi |
| 130 | |
| 131 | log "Image tag: $IMAGE_TAG" |
| 132 | |
| 133 | for target in "${TARGETS[@]}"; do |
| 134 | check_auth "$target" |
| 135 | done |
| 136 | |
| 137 | # ── Build ───────────────────────────────────────────────────────────────────── |
| 138 | |
| 139 | LOCAL_TAG="musehub-build:${IMAGE_TAG}" |
| 140 | CRANE_TAR="/tmp/musehub-${IMAGE_TAG}.tar" |
| 141 | |
| 142 | if [ "$SKIP_BUILD" = false ]; then |
| 143 | log "[1/3] Building image for linux/amd64..." |
| 144 | # --no-cache-filter runtime: a stale BuildKit cache hit was observed live |
| 145 | # (2026-10-21) on the runtime stage's COPY of musehub/pyproject.toml -- |
| 146 | # the deployed container kept serving an old version string (0.2.1rc4) |
| 147 | # three commits after a version bump to 0.2.1rc5 that touched nothing |
| 148 | # but that one file, even though every .py file change in between had |
| 149 | # correctly busted the cache. A clean no-cache rebuild immediately |
| 150 | # produced the correct content, confirming a cache-key bug rather than a |
| 151 | # build-context problem. The expensive builder stage (wheel builds, |
| 152 | # cryptography compiled from source) keeps its cache; only the cheap |
| 153 | # runtime stage (plain file COPYs) is forced fresh every deploy. |
| 154 | docker build \ |
| 155 | --platform linux/amd64 \ |
| 156 | --no-cache-filter runtime \ |
| 157 | --tag "$LOCAL_TAG" \ |
| 158 | -f "$REPO_DIR/Dockerfile" \ |
| 159 | "$ECOSYSTEM_DIR" |
| 160 | log "Build complete." |
| 161 | docker save "$LOCAL_TAG" -o "$CRANE_TAR" |
| 162 | |
| 163 | # ── Push to each target's ECR via crane ─────────────────────────────────── |
| 164 | # crane bypasses Docker Desktop's VPNKit proxy, which drops connections |
| 165 | # mid-upload on large layer pushes. Never use `docker push` to ECR. |
| 166 | # Staging and prod are separate AWS accounts, so the same tarball is |
| 167 | # pushed to each target's own registry under its own credentials. |
| 168 | |
| 169 | log "[2/3] Pushing to ECR for: ${TARGETS[*]}" |
| 170 | for target in "${TARGETS[@]}"; do |
| 171 | registry="$(ecr_registry_for "$target")" |
| 172 | image="${registry}/${ECR_REPO}" |
| 173 | eval "aws ecr get-login-password --region \"$REGION\" $(profile_args "$target")" | \ |
| 174 | crane auth login "$registry" --username AWS --password-stdin |
| 175 | crane push "$CRANE_TAR" "${image}:${IMAGE_TAG}" |
| 176 | crane push "$CRANE_TAR" "${image}:latest" |
| 177 | log " Pushed to $target ($registry)" |
| 178 | done |
| 179 | rm -f "$CRANE_TAR" |
| 180 | log "Push complete. Tag: $IMAGE_TAG" |
| 181 | else |
| 182 | log "[1/3] Skipping build (rollback mode)." |
| 183 | log "[2/3] Skipping push (rollback mode) — assumes the tag already exists in each target registry." |
| 184 | fi |
| 185 | |
| 186 | # ── Trigger deploy via SSM ──────────────────────────────────────────────────── |
| 187 | |
| 188 | log "[3/3] Triggering deploy on: ${TARGETS[*]}" |
| 189 | |
| 190 | deploy_to() { |
| 191 | local env="$1" |
| 192 | local instance_id |
| 193 | instance_id="$(instance_id_for "$env")" |
| 194 | local ecr_image |
| 195 | ecr_image="$(ecr_registry_for "$env")/${ECR_REPO}" |
| 196 | local pargs |
| 197 | pargs=$(profile_args "$env") |
| 198 | |
| 199 | # deploy.sh's MUSEHUB_ENV drives its CloudWatch log group — "prod" (the |
| 200 | # CLI target name) must map to "production" (the actual env/log-group name). |
| 201 | local musehub_env |
| 202 | if [ "$env" = "prod" ]; then musehub_env="production"; else musehub_env="staging"; fi |
| 203 | |
| 204 | log "" |
| 205 | log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |
| 206 | log "→ Deploying to $env ($instance_id)" |
| 207 | log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |
| 208 | |
| 209 | local deploy_sh_b64 |
| 210 | deploy_sh_b64=$(base64 -i "$SCRIPT_DIR/deploy.sh" | tr -d '\n') |
| 211 | |
| 212 | local nginx_conf_b64 |
| 213 | nginx_conf_b64=$(base64 -i "$SCRIPT_DIR/nginx-cf.conf" | tr -d '\n') |
| 214 | |
| 215 | local set_slot_b64 |
| 216 | set_slot_b64=$(base64 -i "$SCRIPT_DIR/set-active-slot.sh" | tr -d '\n') |
| 217 | |
| 218 | # ── Fire the deploy command ─────────────────────────────────────────────── |
| 219 | local deploy_cmd_id |
| 220 | deploy_cmd_id=$(eval "aws ssm send-command \ |
| 221 | --region \"$REGION\" \ |
| 222 | $pargs \ |
| 223 | --instance-ids \"$instance_id\" \ |
| 224 | --document-name \"AWS-RunShellScript\" \ |
| 225 | --parameters \"commands=[ |
| 226 | \\\"echo '${deploy_sh_b64}' | base64 -d > /opt/musehub/deploy/deploy.sh && chmod +x /opt/musehub/deploy/deploy.sh\\\", |
| 227 | \\\"echo '${nginx_conf_b64}' | base64 -d > /opt/musehub/deploy/nginx-cf.conf\\\", |
| 228 | \\\"echo '${set_slot_b64}' | base64 -d > /usr/local/bin/musehub-set-slot && chmod +x /usr/local/bin/musehub-set-slot\\\", |
| 229 | \\\"export ECR_IMAGE=${ecr_image}\\\", |
| 230 | \\\"export IMAGE_TAG=${IMAGE_TAG}\\\", |
| 231 | \\\"export MUSEHUB_ENV=${musehub_env}\\\", |
| 232 | \\\"bash /opt/musehub/deploy/deploy.sh\\\" |
| 233 | ]\" \ |
| 234 | --comment \"musehub ${IMAGE_TAG} → ${env}\" \ |
| 235 | --timeout-seconds 600 \ |
| 236 | --query \"Command.CommandId\" \ |
| 237 | --output text") |
| 238 | |
| 239 | log " SSM command: $deploy_cmd_id" |
| 240 | |
| 241 | # ── Stream deploy output live ───────────────────────────────────────────── |
| 242 | # SSM updates StandardOutputContent as the command runs. We poll every 5s, |
| 243 | # diff against what we've already printed, and show new lines immediately. |
| 244 | log " Live output:" |
| 245 | log "" |
| 246 | |
| 247 | local lines_seen=0 |
| 248 | local elapsed=0 |
| 249 | local final_status="" |
| 250 | |
| 251 | while true; do |
| 252 | sleep 5 |
| 253 | elapsed=$((elapsed + 5)) |
| 254 | |
| 255 | local invocation |
| 256 | invocation=$(eval "aws ssm get-command-invocation \ |
| 257 | --region \"$REGION\" \ |
| 258 | $pargs \ |
| 259 | --command-id \"$deploy_cmd_id\" \ |
| 260 | --instance-id \"$instance_id\" \ |
| 261 | --output json" 2>/dev/null || echo '{"Status":"Pending","StandardOutputContent":"","StandardErrorContent":""}') |
| 262 | |
| 263 | local cmd_status all_stdout all_stderr |
| 264 | cmd_status=$(echo "$invocation" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('Status','Pending'))" 2>/dev/null || echo "Pending") |
| 265 | all_stdout=$(echo "$invocation" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('StandardOutputContent',''),end='')" 2>/dev/null || echo "") |
| 266 | all_stderr=$(echo "$invocation" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('StandardErrorContent',''),end='')" 2>/dev/null || echo "") |
| 267 | |
| 268 | # Print any new stdout lines |
| 269 | local total_lines |
| 270 | total_lines=$(echo "$all_stdout" | wc -l) |
| 271 | if [ "$total_lines" -gt "$lines_seen" ]; then |
| 272 | echo "$all_stdout" | tail -n +"$((lines_seen + 1))" |
| 273 | lines_seen=$total_lines |
| 274 | fi |
| 275 | |
| 276 | case "$cmd_status" in |
| 277 | Success) |
| 278 | final_status="success" |
| 279 | break |
| 280 | ;; |
| 281 | Failed|Cancelled|TimedOut|Cancelling) |
| 282 | final_status="failed" |
| 283 | [ -n "$all_stderr" ] && echo "STDERR: $all_stderr" |
| 284 | break |
| 285 | ;; |
| 286 | esac |
| 287 | |
| 288 | if [ "$elapsed" -ge 600 ]; then |
| 289 | die "Deploy timed out after 10 min. SSM command: $deploy_cmd_id" |
| 290 | fi |
| 291 | done |
| 292 | |
| 293 | echo "" |
| 294 | if [ "$final_status" = "success" ]; then |
| 295 | log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |
| 296 | log "✅ $env deploy succeeded." |
| 297 | log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |
| 298 | return 0 |
| 299 | else |
| 300 | log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |
| 301 | log "❌ $env deploy FAILED. Full output:" |
| 302 | log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |
| 303 | eval "aws ssm get-command-invocation \ |
| 304 | --region \"$REGION\" \ |
| 305 | $pargs \ |
| 306 | --command-id \"$deploy_cmd_id\" \ |
| 307 | --instance-id \"$instance_id\" \ |
| 308 | --query \"[StandardOutputContent,StandardErrorContent]\" \ |
| 309 | --output text" 2>/dev/null || true |
| 310 | return 1 |
| 311 | fi |
| 312 | } |
| 313 | |
| 314 | for target in "${TARGETS[@]}"; do |
| 315 | deploy_to "$target" |
| 316 | done |
| 317 | |
| 318 | log "" |
| 319 | log "All done." |
| 320 | log " Tag: $IMAGE_TAG" |
File History
1 commit
sha256:47c92210e23600ae975e8943d129f6c752fb4d5835fd19b8d74fbc8489ec83f8
fix: deploy build silently served a stale Docker layer for …
Sonnet 5
minor
⚠
22 hours ago