"""Section 7.3 — Deployment readiness tests. Covers: Zero-downtime : deploy.sh is blue-green (two slots, nginx flip); health check URLs point to /healthz not a UI page. /healthz : returns 200 when DB + storage healthy; 503 with JSON body when either is down; exempt from auth; fast. Non-root user : Dockerfile uses USER instruction (non-root); container runs as the 'musehub' system user. Read-only FS : docker-compose.yml sets read_only: true on musehub service; /tmp is mounted as tmpfs; /data is a named volume. Resource limits: CPU and memory limits set on musehub, postgres, runner. """ from __future__ import annotations import re from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest from httpx import AsyncClient _ROOT = Path(__file__).resolve().parents[1] _DOCKERFILE = _ROOT / "Dockerfile" _COMPOSE = _ROOT / "docker-compose.yml" _DEPLOY_SH = _ROOT / "deploy" / "deploy.sh" # ═══════════════════════════════════════════════════════════════════════════════ # Zero-downtime deploy # ═══════════════════════════════════════════════════════════════════════════════ class TestZeroDowntimeDeploy: _src = _DEPLOY_SH.read_text() def test_two_slots_defined(self) -> None: """deploy.sh must define both blue and green slots.""" assert "blue" in self._src and "green" in self._src def test_nginx_flip_present(self) -> None: """deploy.sh must reload nginx after health check passes (atomic flip).""" assert "nginx -s reload" in self._src or "nginx_point_to" in self._src def test_health_check_before_nginx_flip(self) -> None: """Health check must happen before the nginx flip — never flip a sick slot.""" src = self._src health_pos = src.find("health_check") nginx_pos = src.find("nginx_point_to") assert health_pos != -1, "health_check function not found in deploy.sh" assert nginx_pos != -1, "nginx_point_to not found in deploy.sh" assert health_pos < nginx_pos, ( "nginx flip happens before health check — would route to an unhealthy slot" ) def test_health_urls_point_to_healthz(self) -> None: """deploy.sh must use /healthz not a UI page as the readiness signal.""" assert "/healthz" in self._src, ( "deploy.sh does not use /healthz — health check may pass even when " "DB or storage is down (UI pages don't probe dependencies)" ) assert "/explore" not in self._src, ( "deploy.sh still references /explore as a health URL — update to /healthz" ) def test_old_slot_stopped_after_flip(self) -> None: """deploy.sh must stop the old slot after the nginx flip to free resources.""" src = self._src nginx_pos = src.find("nginx_point_to") stop_pos = src.find("docker rm -f", nginx_pos) assert stop_pos != -1, ( "deploy.sh does not stop the old slot after the nginx flip" ) def test_dockerfile_healthcheck_uses_healthz(self) -> None: """Dockerfile HEALTHCHECK must probe /healthz.""" src = _DOCKERFILE.read_text() hc_lines = [l for l in src.splitlines() if "HEALTHCHECK" in l or l.strip().startswith("CMD")] combined = " ".join(hc_lines) assert "/healthz" in combined, ( "Dockerfile HEALTHCHECK does not probe /healthz — " "docker will report 'healthy' even when DB is down" ) # ═══════════════════════════════════════════════════════════════════════════════ # /healthz endpoint # ═══════════════════════════════════════════════════════════════════════════════ class TestHealthzEndpoint: async def test_healthz_returns_200_when_healthy(self, client: AsyncClient) -> None: """GET /healthz must return 200 when DB and storage are reachable.""" resp = await client.get("/healthz") assert resp.status_code == 200 body = resp.json() assert body["status"] == "ok" assert body["db"] is True assert body["storage"] is True async def test_healthz_returns_json(self, client: AsyncClient) -> None: resp = await client.get("/healthz") assert resp.headers["content-type"].startswith("application/json") async def test_healthz_no_auth_required(self, client: AsyncClient) -> None: """Healthz must be reachable without any Authorization header.""" resp = await client.get("/healthz") # Must not be 401 or 403 assert resp.status_code not in (401, 403), ( f"/healthz returned {resp.status_code} — health check must be unauthenticated" ) async def test_healthz_503_when_db_down(self, client: AsyncClient) -> None: """GET /healthz must return 503 when the DB probe fails.""" from sqlalchemy.exc import OperationalError # Patch the DB execute to simulate a broken connection with patch( "musehub.main.AsyncSession.execute", new_callable=AsyncMock, side_effect=OperationalError("connection refused", None, None), ): resp = await client.get("/healthz") assert resp.status_code == 503 body = resp.json() assert body["status"] == "unhealthy" assert body["db"] is False async def test_healthz_503_when_storage_down(self, client: AsyncClient) -> None: """GET /healthz must return 503 when the storage probe fails.""" from musehub.storage.backends import BlobBackend # Backend pointing at an unreachable endpoint — head_bucket will fail. bad_backend = BlobBackend( bucket="muse-objects", endpoint_url="http://127.0.0.1:19999", # nothing listening here access_key_id="x", secret_access_key="x", region="us-east-1", ) with patch("musehub.storage.backends.get_backend", return_value=bad_backend): resp = await client.get("/healthz") assert resp.status_code == 503 body = resp.json() assert body["status"] == "unhealthy" assert body["storage"] is False async def test_healthz_body_has_db_and_storage_keys(self, client: AsyncClient) -> None: """Response body must expose both db and storage status for monitoring.""" resp = await client.get("/healthz") body = resp.json() assert "db" in body, "healthz response missing 'db' key" assert "storage" in body, "healthz response missing 'storage' key" async def test_healthz_fast(self, client: AsyncClient) -> None: """Healthz must respond in under 2 s (load balancer timeout is typically 5 s).""" import time start = time.monotonic() await client.get("/healthz") elapsed = time.monotonic() - start assert elapsed < 2.0, f"/healthz took {elapsed:.2f}s — too slow for a probe" def test_healthz_route_registered(self) -> None: """The /healthz route must be registered in the FastAPI app.""" from musehub.main import app paths = [route.path for route in app.routes] assert "/healthz" in paths, "/healthz route not registered in app" # ═══════════════════════════════════════════════════════════════════════════════ # Non-root container user # ═══════════════════════════════════════════════════════════════════════════════ class TestNonRootUser: _src = _DOCKERFILE.read_text() def test_dockerfile_has_user_instruction(self) -> None: """Dockerfile must have a USER instruction.""" user_lines = [l.strip() for l in self._src.splitlines() if l.strip().upper().startswith("USER ")] assert user_lines, "Dockerfile has no USER instruction — container runs as root" def test_dockerfile_user_is_not_root(self) -> None: """Dockerfile USER must not be root or UID 0.""" user_lines = [l.strip() for l in self._src.splitlines() if l.strip().upper().startswith("USER ")] for line in user_lines: user = line.split()[1].lower() assert user not in ("root", "0"), ( f"Dockerfile sets USER to {user!r} — container must run as non-root" ) def test_dockerfile_creates_system_user(self) -> None: """Dockerfile must create a dedicated system user (groupadd + useradd).""" assert "groupadd" in self._src and "useradd" in self._src, ( "Dockerfile does not create a dedicated system user" ) def test_dockerfile_user_applied_after_installs(self) -> None: """USER instruction must come after RUN pip install (installs need root).""" lines = self._src.splitlines() user_idx = next( (i for i, l in enumerate(lines) if l.strip().upper().startswith("USER ")), None ) pip_idx = max( (i for i, l in enumerate(lines) if "pip install" in l), default=None ) assert user_idx is not None and pip_idx is not None assert user_idx > pip_idx, ( "USER instruction appears before pip install — " "package installation would fail without root" ) # ═══════════════════════════════════════════════════════════════════════════════ # Read-only filesystem # ═══════════════════════════════════════════════════════════════════════════════ class TestReadOnlyFilesystem: def _parse_yaml(self) -> None: import yaml return yaml.safe_load(_COMPOSE.read_text()) def test_musehub_service_read_only(self) -> None: """musehub service must have read_only: true.""" src = _COMPOSE.read_text() # Structural check: read_only appears in the musehub service block # Find musehub service block (between 'musehub:' and the next top-level key) in_musehub = False for line in src.splitlines(): if re.match(r'^ musehub:', line): in_musehub = True elif re.match(r'^ \w', line) and in_musehub: in_musehub = False if in_musehub and "read_only: true" in line: return pytest.fail( "musehub service in docker-compose.yml does not have read_only: true" ) def test_tmp_is_tmpfs_or_volume(self) -> None: """/tmp must be writable (tmpfs or volume) so uvicorn can write temp files.""" src = _COMPOSE.read_text() assert "tmpfs" in src or "/tmp" in src, ( "No tmpfs mount for /tmp — uvicorn and Python will fail to write temp files " "when the root filesystem is read-only" ) def test_data_volume_is_explicit(self) -> None: """/data object store must be an explicit named volume (not read-only).""" src = _COMPOSE.read_text() assert "musehub_data:/data" in src, ( "/data is not mounted as an explicit volume — objects cannot be written " "when the root filesystem is read-only" ) # ═══════════════════════════════════════════════════════════════════════════════ # Resource limits # ═══════════════════════════════════════════════════════════════════════════════ class TestResourceLimits: _src = _COMPOSE.read_text() def _service_limits(self, service_name: str) -> str: """Extract the text block for a given service.""" lines = self._src.splitlines() in_service = False block_lines = [] for line in lines: if re.match(rf'^ {re.escape(service_name)}:', line): in_service = True elif re.match(r'^ \w', line) and in_service: break if in_service: block_lines.append(line) return "\n".join(block_lines) def test_musehub_has_cpu_limit(self) -> None: block = self._service_limits("musehub") assert "cpus:" in block, "musehub service has no CPU limit" def test_musehub_has_memory_limit(self) -> None: block = self._service_limits("musehub") assert "memory:" in block, "musehub service has no memory limit" def test_musehub_memory_limit_sane(self) -> None: """musehub memory limit must be ≥ 256 MiB (app needs headroom).""" block = self._service_limits("musehub") m = re.search(r'memory:\s*(\d+)([MmGg])', block) if m: amount = int(m.group(1)) unit = m.group(2).upper() mb = amount * 1024 if unit == "G" else amount assert mb >= 256, f"musehub memory limit {mb}M is below 256M minimum" def test_postgres_has_cpu_limit(self) -> None: block = self._service_limits("postgres") assert "cpus:" in block, "postgres service has no CPU limit" def test_postgres_has_memory_limit(self) -> None: block = self._service_limits("postgres") assert "memory:" in block, "postgres service has no memory limit" def test_runner_has_cpu_limit(self) -> None: block = self._service_limits("musehub-runner") assert "cpus:" in block, "musehub-runner service has no CPU limit" def test_runner_has_memory_limit(self) -> None: block = self._service_limits("musehub-runner") assert "memory:" in block, "musehub-runner service has no memory limit" def test_all_services_have_deploy_block(self) -> None: """All three services must have a deploy: block (where limits live).""" for svc in ("musehub", "postgres", "musehub-runner"): block = self._service_limits(svc) assert "deploy:" in block, f"{svc} service has no deploy: block"