"""Section 7.3 — Deployment readiness tests. Covers: Zero-downtime : deploy.sh is blue-green (two slots, nginx flip); health check URLs point to /healthz not a UI page. /healthz : returns 200 when DB + storage healthy; 503 with JSON body when either is down; exempt from auth; fast. Non-root user : Dockerfile uses USER instruction (non-root); container runs as the 'musehub' system user. Read-only FS : docker-compose.yml sets read_only: true on musehub service; /tmp is mounted as tmpfs; /data is a named volume. Resource limits: CPU and memory limits set on musehub, postgres, runner. """ from __future__ import annotations import re import subprocess from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest from httpx import AsyncClient _ROOT = Path(__file__).resolve().parents[1] _DOCKERFILE = _ROOT / "Dockerfile" _COMPOSE = _ROOT / "docker-compose.yml" _DEPLOY_SH = _ROOT / "deploy" / "deploy.sh" # ═══════════════════════════════════════════════════════════════════════════════ # Zero-downtime deploy # ═══════════════════════════════════════════════════════════════════════════════ class TestZeroDowntimeDeploy: _src = _DEPLOY_SH.read_text() def test_two_slots_defined(self) -> None: """deploy.sh must define both blue and green slots.""" assert "blue" in self._src and "green" in self._src def test_nginx_flip_present(self) -> None: """deploy.sh must reload nginx after health check passes (atomic flip).""" assert "nginx -s reload" in self._src or "nginx_point_to" in self._src def test_health_check_before_nginx_flip(self) -> None: """Health check must happen before the nginx flip — never flip a sick slot.""" src = self._src health_pos = src.find("health_check") nginx_pos = src.find("nginx_point_to") assert health_pos != -1, "health_check function not found in deploy.sh" assert nginx_pos != -1, "nginx_point_to not found in deploy.sh" assert health_pos < nginx_pos, ( "nginx flip happens before health check — would route to an unhealthy slot" ) def test_health_urls_point_to_healthz(self) -> None: """deploy.sh must use /healthz not a UI page as the readiness signal.""" assert "/healthz" in self._src, ( "deploy.sh does not use /healthz — health check may pass even when " "DB or storage is down (UI pages don't probe dependencies)" ) assert "/explore" not in self._src, ( "deploy.sh still references /explore as a health URL — update to /healthz" ) def test_old_slot_stopped_after_flip(self) -> None: """deploy.sh must stop the old slot after the nginx flip to free resources.""" src = self._src nginx_pos = src.find("nginx_point_to") stop_pos = src.find("docker rm -f", nginx_pos) assert stop_pos != -1, ( "deploy.sh does not stop the old slot after the nginx flip" ) def test_dockerfile_healthcheck_uses_healthz(self) -> None: """Dockerfile HEALTHCHECK must probe /healthz.""" src = _DOCKERFILE.read_text() hc_lines = [l for l in src.splitlines() if "HEALTHCHECK" in l or l.strip().startswith("CMD")] combined = " ".join(hc_lines) assert "/healthz" in combined, ( "Dockerfile HEALTHCHECK does not probe /healthz — " "docker will report 'healthy' even when DB is down" ) # ═══════════════════════════════════════════════════════════════════════════════ # /healthz endpoint # ═══════════════════════════════════════════════════════════════════════════════ class TestHealthzEndpoint: async def test_healthz_returns_200_when_healthy(self, client: AsyncClient) -> None: """GET /healthz must return 200 when DB and storage are reachable.""" resp = await client.get("/healthz") assert resp.status_code == 200 body = resp.json() assert body["status"] == "ok" assert body["db"] is True assert body["storage"] is True async def test_healthz_returns_json(self, client: AsyncClient) -> None: resp = await client.get("/healthz") assert resp.headers["content-type"].startswith("application/json") async def test_healthz_no_auth_required(self, client: AsyncClient) -> None: """Healthz must be reachable without any Authorization header.""" resp = await client.get("/healthz") # Must not be 401 or 403 assert resp.status_code not in (401, 403), ( f"/healthz returned {resp.status_code} — health check must be unauthenticated" ) async def test_healthz_503_when_db_down(self, client: AsyncClient) -> None: """GET /healthz must return 503 when the DB probe fails.""" from sqlalchemy.exc import OperationalError # Patch the DB execute to simulate a broken connection with patch( "musehub.main.AsyncSession.execute", new_callable=AsyncMock, side_effect=OperationalError("connection refused", None, None), ): resp = await client.get("/healthz") assert resp.status_code == 503 body = resp.json() assert body["status"] == "unhealthy" assert body["db"] is False async def test_healthz_503_when_storage_down(self, client: AsyncClient) -> None: """GET /healthz must return 503 when the storage probe fails.""" from musehub.storage.backends import BlobBackend # Backend pointing at an unreachable endpoint — head_bucket will fail. bad_backend = BlobBackend( bucket="muse-objects", endpoint_url="http://127.0.0.1:19999", # nothing listening here access_key_id="x", secret_access_key="x", region="us-east-1", ) with patch("musehub.storage.backends.get_backend", return_value=bad_backend): resp = await client.get("/healthz") assert resp.status_code == 503 body = resp.json() assert body["status"] == "unhealthy" assert body["storage"] is False async def test_healthz_body_has_db_and_storage_keys(self, client: AsyncClient) -> None: """Response body must expose both db and storage status for monitoring.""" resp = await client.get("/healthz") body = resp.json() assert "db" in body, "healthz response missing 'db' key" assert "storage" in body, "healthz response missing 'storage' key" async def test_healthz_fast(self, client: AsyncClient) -> None: """Healthz must respond in under 2 s (load balancer timeout is typically 5 s).""" import time start = time.monotonic() await client.get("/healthz") elapsed = time.monotonic() - start assert elapsed < 2.0, f"/healthz took {elapsed:.2f}s — too slow for a probe" def test_healthz_route_registered(self) -> None: """The /healthz route must be registered in the FastAPI app.""" from musehub.main import app paths = [route.path for route in app.routes] assert "/healthz" in paths, "/healthz route not registered in app" # ═══════════════════════════════════════════════════════════════════════════════ # Non-root container user # ═══════════════════════════════════════════════════════════════════════════════ class TestNonRootUser: _src = _DOCKERFILE.read_text() def test_dockerfile_has_user_instruction(self) -> None: """Dockerfile must have a USER instruction.""" user_lines = [l.strip() for l in self._src.splitlines() if l.strip().upper().startswith("USER ")] assert user_lines, "Dockerfile has no USER instruction — container runs as root" def test_dockerfile_user_is_not_root(self) -> None: """Dockerfile USER must not be root or UID 0.""" user_lines = [l.strip() for l in self._src.splitlines() if l.strip().upper().startswith("USER ")] for line in user_lines: user = line.split()[1].lower() assert user not in ("root", "0"), ( f"Dockerfile sets USER to {user!r} — container must run as non-root" ) def test_dockerfile_creates_system_user(self) -> None: """Dockerfile must create a dedicated system user (groupadd + useradd).""" assert "groupadd" in self._src and "useradd" in self._src, ( "Dockerfile does not create a dedicated system user" ) def test_dockerfile_user_applied_after_installs(self) -> None: """USER instruction must come after RUN pip install (installs need root).""" lines = self._src.splitlines() user_idx = next( (i for i, l in enumerate(lines) if l.strip().upper().startswith("USER ")), None ) pip_idx = max( (i for i, l in enumerate(lines) if "pip install" in l), default=None ) assert user_idx is not None and pip_idx is not None assert user_idx > pip_idx, ( "USER instruction appears before pip install — " "package installation would fail without root" ) # ═══════════════════════════════════════════════════════════════════════════════ # Read-only filesystem # ═══════════════════════════════════════════════════════════════════════════════ class TestReadOnlyFilesystem: def _parse_yaml(self) -> None: import yaml return yaml.safe_load(_COMPOSE.read_text()) def test_musehub_service_read_only(self) -> None: """musehub service must have read_only: true.""" src = _COMPOSE.read_text() # Structural check: read_only appears in the musehub service block # Find musehub service block (between 'musehub:' and the next top-level key) in_musehub = False for line in src.splitlines(): if re.match(r'^ musehub:', line): in_musehub = True elif re.match(r'^ \w', line) and in_musehub: in_musehub = False if in_musehub and "read_only: true" in line: return pytest.fail( "musehub service in docker-compose.yml does not have read_only: true" ) def test_tmp_is_tmpfs_or_volume(self) -> None: """/tmp must be writable (tmpfs or volume) so uvicorn can write temp files.""" src = _COMPOSE.read_text() assert "tmpfs" in src or "/tmp" in src, ( "No tmpfs mount for /tmp — uvicorn and Python will fail to write temp files " "when the root filesystem is read-only" ) def test_data_volume_is_explicit(self) -> None: """/data object store must be an explicit named volume (not read-only).""" src = _COMPOSE.read_text() assert "musehub_data:/data" in src, ( "/data is not mounted as an explicit volume — objects cannot be written " "when the root filesystem is read-only" ) # ═══════════════════════════════════════════════════════════════════════════════ # Resource limits # ═══════════════════════════════════════════════════════════════════════════════ class TestResourceLimits: _src = _COMPOSE.read_text() def _service_limits(self, service_name: str) -> str: """Extract the text block for a given service.""" lines = self._src.splitlines() in_service = False block_lines = [] for line in lines: if re.match(rf'^ {re.escape(service_name)}:', line): in_service = True elif re.match(r'^ \w', line) and in_service: break if in_service: block_lines.append(line) return "\n".join(block_lines) def test_musehub_has_cpu_limit(self) -> None: block = self._service_limits("musehub") assert "cpus:" in block, "musehub service has no CPU limit" def test_musehub_has_memory_limit(self) -> None: block = self._service_limits("musehub") assert "memory:" in block, "musehub service has no memory limit" def test_musehub_memory_limit_sane(self) -> None: """musehub memory limit must be ≥ 256 MiB (app needs headroom).""" block = self._service_limits("musehub") m = re.search(r'memory:\s*(\d+)([MmGg])', block) if m: amount = int(m.group(1)) unit = m.group(2).upper() mb = amount * 1024 if unit == "G" else amount assert mb >= 256, f"musehub memory limit {mb}M is below 256M minimum" def test_postgres_has_cpu_limit(self) -> None: block = self._service_limits("postgres") assert "cpus:" in block, "postgres service has no CPU limit" def test_postgres_has_memory_limit(self) -> None: block = self._service_limits("postgres") assert "memory:" in block, "postgres service has no memory limit" def test_runner_has_cpu_limit(self) -> None: block = self._service_limits("musehub-runner") assert "cpus:" in block, "musehub-runner service has no CPU limit" def test_runner_has_memory_limit(self) -> None: block = self._service_limits("musehub-runner") assert "memory:" in block, "musehub-runner service has no memory limit" def test_all_services_have_deploy_block(self) -> None: """All three services must have a deploy: block (where limits live).""" for svc in ("musehub", "postgres", "musehub-runner"): block = self._service_limits(svc) assert "deploy:" in block, f"{svc} service has no deploy: block" # ═══════════════════════════════════════════════════════════════════════════════ # musehub#194 — ECR_REGISTRY must derive from ECR_IMAGE, never be independently # hardcoded. A "prod" deploy passes ECR_IMAGE pointing at Production's registry # (see push.sh's per-environment ECR_REGISTRY map) — deploy.sh's docker # login/logout must authenticate against that SAME registry, not a hardcoded # Nonproduction one, or `docker pull` fails with an auth mismatch. # ═══════════════════════════════════════════════════════════════════════════════ class TestEcrRegistryDerivedFromImage: """Executes the real variable-derivation lines from deploy.sh in a subprocess (not just a text/regex check) — this is a bash variable scoping bug, so the test proves actual runtime behavior.""" _STAGING_IMAGE = "992382692655.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub" _PROD_IMAGE = "672469410277.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub" @staticmethod def _extract_var_block() -> str: """Return the exact source lines from ECR_IMAGE's definition through FULL_IMAGE's definition — the minimal slice needed to resolve ECR_REGISTRY/ECR_IMAGE/FULL_IMAGE, without deploy.sh's logging setup, `cd`, or anything requiring a real /opt/musehub.""" src = _DEPLOY_SH.read_text() start_marker = 'ECR_IMAGE="${ECR_IMAGE:-' end_marker = 'FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"' start = src.index(start_marker) # Back up to the start of that line (the marker may not be at col 0). line_start = src.rfind("\n", 0, start) + 1 end = src.index(end_marker) + len(end_marker) return src[line_start:end] def _resolve(self, ecr_image: str | None) -> dict[str, str]: block = self._extract_var_block() script = ( f"{block}\n" 'echo "ECR_REGISTRY=$ECR_REGISTRY"\n' 'echo "ECR_IMAGE=$ECR_IMAGE"\n' 'echo "FULL_IMAGE=$FULL_IMAGE"\n' ) env: dict[str, str] = {"PATH": "/usr/bin:/bin", "IMAGE_TAG": "abc123"} if ecr_image is not None: env["ECR_IMAGE"] = ecr_image result = subprocess.run( ["bash", "-c", script], env=env, capture_output=True, text=True, timeout=5, ) assert result.returncode == 0, result.stderr out: dict[str, str] = {} for line in result.stdout.splitlines(): key, _, value = line.partition("=") out[key] = value return out def test_staging_registry_derived_from_ecr_image(self) -> None: resolved = self._resolve(self._STAGING_IMAGE) assert resolved["ECR_REGISTRY"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com" def test_production_registry_derived_from_ecr_image(self) -> None: """musehub#194: the exact regression — a prod ECR_IMAGE must resolve to Production's registry, never the hardcoded staging one.""" resolved = self._resolve(self._PROD_IMAGE) assert resolved["ECR_REGISTRY"] == "672469410277.dkr.ecr.us-east-1.amazonaws.com" assert resolved["ECR_REGISTRY"] != "992382692655.dkr.ecr.us-east-1.amazonaws.com" def test_full_image_matches_passed_ecr_image_and_tag(self) -> None: resolved = self._resolve(self._PROD_IMAGE) assert resolved["FULL_IMAGE"] == f"{self._PROD_IMAGE}:abc123" def test_default_ecr_image_falls_back_to_staging_registry(self) -> None: """No ECR_IMAGE set (manual on-instance invocation) defaults to staging — matches the script's documented manual-use fallback.""" resolved = self._resolve(None) assert resolved["ECR_REGISTRY"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com" def test_no_independently_hardcoded_production_or_staging_registry_constant(self) -> None: """Guards against reintroducing a second, independent ECR_REGISTRY source of truth — the account ID must only ever appear as part of ECR_IMAGE's default value, never as a bare ECR_REGISTRY="..." literal.""" src = _DEPLOY_SH.read_text() assert not re.search(r'^ECR_REGISTRY="\d', src, re.MULTILINE), ( "ECR_REGISTRY is hardcoded to a literal account ID again — it must " "be derived from ECR_IMAGE so docker login and docker pull always " "target the same registry." )