gabriel / musehub public

test_deployment.py file-level

at sha256:8 · View file ↗ · Intel ↗

History
1 files
1 commits
0 hotspots
0 🧊 dead
0 πŸ’₯ blast risk
sha256:4 fix(publish_issue): hard-block publishing a body that leaks a staging U… · gabriel · Sep 11, 2026
1 """Section 7.3 β€” Deployment readiness tests.
2
3 Covers:
4 Zero-downtime : deploy.sh is blue-green (two slots, nginx flip); health
5 check URLs point to /healthz not a UI page.
6 /healthz : returns 200 when DB + storage healthy; 503 with JSON body
7 when either is down; exempt from auth; fast.
8 Non-root user : Dockerfile uses USER instruction (non-root); container
9 runs as the 'musehub' system user.
10 Read-only FS : docker-compose.yml sets read_only: true on musehub service;
11 /tmp is mounted as tmpfs; /data is a named volume.
12 Resource limits: CPU and memory limits set on musehub, postgres, runner.
13 """
14 from __future__ import annotations
15
16 import re
17 import subprocess
18 from pathlib import Path
19 from unittest.mock import AsyncMock, MagicMock, patch
20
21 import pytest
22 from httpx import AsyncClient
23
24 _ROOT = Path(__file__).resolve().parents[1]
25 _DOCKERFILE = _ROOT / "Dockerfile"
26 _COMPOSE = _ROOT / "docker-compose.yml"
27 _DEPLOY_SH = _ROOT / "deploy" / "deploy.sh"
28 _PUSH_SH = _ROOT / "deploy" / "push.sh"
29
30
31 # ═══════════════════════════════════════════════════════════════════════════════
32 # Zero-downtime deploy
33 # ═══════════════════════════════════════════════════════════════════════════════
34
35 class TestZeroDowntimeDeploy:
36 _src = _DEPLOY_SH.read_text()
37
38 def test_two_slots_defined(self) -> None:
39 """deploy.sh must define both blue and green slots."""
40 assert "blue" in self._src and "green" in self._src
41
42 def test_nginx_flip_present(self) -> None:
43 """deploy.sh must reload nginx after health check passes (atomic flip)."""
44 assert "nginx -s reload" in self._src or "nginx_point_to" in self._src
45
46 def test_health_check_before_nginx_flip(self) -> None:
47 """Health check must happen before the nginx flip β€” never flip a sick slot."""
48 src = self._src
49 health_pos = src.find("health_check")
50 nginx_pos = src.find("nginx_point_to")
51 assert health_pos != -1, "health_check function not found in deploy.sh"
52 assert nginx_pos != -1, "nginx_point_to not found in deploy.sh"
53 assert health_pos < nginx_pos, (
54 "nginx flip happens before health check β€” would route to an unhealthy slot"
55 )
56
57 def test_health_urls_point_to_healthz(self) -> None:
58 """deploy.sh must use /healthz not a UI page as the readiness signal."""
59 assert "/healthz" in self._src, (
60 "deploy.sh does not use /healthz β€” health check may pass even when "
61 "DB or storage is down (UI pages don't probe dependencies)"
62 )
63 assert "/explore" not in self._src, (
64 "deploy.sh still references /explore as a health URL β€” update to /healthz"
65 )
66
67 def test_old_slot_stopped_after_flip(self) -> None:
68 """deploy.sh must stop the old slot after the nginx flip to free resources."""
69 src = self._src
70 nginx_pos = src.find("nginx_point_to")
71 stop_pos = src.find("docker rm -f", nginx_pos)
72 assert stop_pos != -1, (
73 "deploy.sh does not stop the old slot after the nginx flip"
74 )
75
76 def test_dockerfile_healthcheck_uses_healthz(self) -> None:
77 """Dockerfile HEALTHCHECK must probe /healthz."""
78 src = _DOCKERFILE.read_text()
79 hc_lines = [l for l in src.splitlines() if "HEALTHCHECK" in l or l.strip().startswith("CMD")]
80 combined = " ".join(hc_lines)
81 assert "/healthz" in combined, (
82 "Dockerfile HEALTHCHECK does not probe /healthz β€” "
83 "docker will report 'healthy' even when DB is down"
84 )
85
86
87 # ═══════════════════════════════════════════════════════════════════════════════
88 # /healthz endpoint
89 # ═══════════════════════════════════════════════════════════════════════════════
90
91 class TestHealthzEndpoint:
92 async def test_healthz_returns_200_when_healthy(self, client: AsyncClient) -> None:
93 """GET /healthz must return 200 when DB and storage are reachable."""
94 resp = await client.get("/healthz")
95 assert resp.status_code == 200
96 body = resp.json()
97 assert body["status"] == "ok"
98 assert body["db"] is True
99 assert body["storage"] is True
100
101 async def test_healthz_returns_json(self, client: AsyncClient) -> None:
102 resp = await client.get("/healthz")
103 assert resp.headers["content-type"].startswith("application/json")
104
105 async def test_healthz_no_auth_required(self, client: AsyncClient) -> None:
106 """Healthz must be reachable without any Authorization header."""
107 resp = await client.get("/healthz")
108 # Must not be 401 or 403
109 assert resp.status_code not in (401, 403), (
110 f"/healthz returned {resp.status_code} β€” health check must be unauthenticated"
111 )
112
113 async def test_healthz_503_when_db_down(self, client: AsyncClient) -> None:
114 """GET /healthz must return 503 when the DB probe fails."""
115 from sqlalchemy.exc import OperationalError
116
117 # Patch the DB execute to simulate a broken connection
118 with patch(
119 "musehub.main.AsyncSession.execute",
120 new_callable=AsyncMock,
121 side_effect=OperationalError("connection refused", None, None),
122 ):
123 resp = await client.get("/healthz")
124
125 assert resp.status_code == 503
126 body = resp.json()
127 assert body["status"] == "unhealthy"
128 assert body["db"] is False
129
130 async def test_healthz_503_when_storage_down(self, client: AsyncClient) -> None:
131 """GET /healthz must return 503 when the storage probe fails."""
132 from musehub.storage.backends import BlobBackend
133
134 # Backend pointing at an unreachable endpoint β€” head_bucket will fail.
135 bad_backend = BlobBackend(
136 bucket="muse-objects",
137 endpoint_url="http://127.0.0.1:19999", # nothing listening here
138 access_key_id="x",
139 secret_access_key="x",
140 region="us-east-1",
141 )
142
143 with patch("musehub.storage.backends.get_backend", return_value=bad_backend):
144 resp = await client.get("/healthz")
145
146 assert resp.status_code == 503
147 body = resp.json()
148 assert body["status"] == "unhealthy"
149 assert body["storage"] is False
150
151 async def test_healthz_body_has_db_and_storage_keys(self, client: AsyncClient) -> None:
152 """Response body must expose both db and storage status for monitoring."""
153 resp = await client.get("/healthz")
154 body = resp.json()
155 assert "db" in body, "healthz response missing 'db' key"
156 assert "storage" in body, "healthz response missing 'storage' key"
157
158 async def test_healthz_fast(self, client: AsyncClient) -> None:
159 """Healthz must respond in under 2 s (load balancer timeout is typically 5 s)."""
160 import time
161 start = time.monotonic()
162 await client.get("/healthz")
163 elapsed = time.monotonic() - start
164 assert elapsed < 2.0, f"/healthz took {elapsed:.2f}s β€” too slow for a probe"
165
166 def test_healthz_route_registered(self) -> None:
167 """The /healthz route must be registered in the FastAPI app."""
168 from musehub.main import app
169 paths = [route.path for route in app.routes]
170 assert "/healthz" in paths, "/healthz route not registered in app"
171
172
173 # ═══════════════════════════════════════════════════════════════════════════════
174 # Non-root container user
175 # ═══════════════════════════════════════════════════════════════════════════════
176
177 class TestNonRootUser:
178 _src = _DOCKERFILE.read_text()
179
180 def test_dockerfile_has_user_instruction(self) -> None:
181 """Dockerfile must have a USER instruction."""
182 user_lines = [l.strip() for l in self._src.splitlines()
183 if l.strip().upper().startswith("USER ")]
184 assert user_lines, "Dockerfile has no USER instruction β€” container runs as root"
185
186 def test_dockerfile_user_is_not_root(self) -> None:
187 """Dockerfile USER must not be root or UID 0."""
188 user_lines = [l.strip() for l in self._src.splitlines()
189 if l.strip().upper().startswith("USER ")]
190 for line in user_lines:
191 user = line.split()[1].lower()
192 assert user not in ("root", "0"), (
193 f"Dockerfile sets USER to {user!r} β€” container must run as non-root"
194 )
195
196 def test_dockerfile_creates_system_user(self) -> None:
197 """Dockerfile must create a dedicated system user (groupadd + useradd)."""
198 assert "groupadd" in self._src and "useradd" in self._src, (
199 "Dockerfile does not create a dedicated system user"
200 )
201
202 def test_dockerfile_user_applied_after_installs(self) -> None:
203 """USER instruction must come after RUN pip install (installs need root)."""
204 lines = self._src.splitlines()
205 user_idx = next(
206 (i for i, l in enumerate(lines) if l.strip().upper().startswith("USER ")), None
207 )
208 pip_idx = max(
209 (i for i, l in enumerate(lines) if "pip install" in l), default=None
210 )
211 assert user_idx is not None and pip_idx is not None
212 assert user_idx > pip_idx, (
213 "USER instruction appears before pip install β€” "
214 "package installation would fail without root"
215 )
216
217
218 # ═══════════════════════════════════════════════════════════════════════════════
219 # Read-only filesystem
220 # ═══════════════════════════════════════════════════════════════════════════════
221
222 class TestReadOnlyFilesystem:
223 def _parse_yaml(self) -> None:
224 import yaml
225 return yaml.safe_load(_COMPOSE.read_text())
226
227 def test_musehub_service_read_only(self) -> None:
228 """musehub service must have read_only: true."""
229 src = _COMPOSE.read_text()
230 # Structural check: read_only appears in the musehub service block
231 # Find musehub service block (between 'musehub:' and the next top-level key)
232 in_musehub = False
233 for line in src.splitlines():
234 if re.match(r'^ musehub:', line):
235 in_musehub = True
236 elif re.match(r'^ \w', line) and in_musehub:
237 in_musehub = False
238 if in_musehub and "read_only: true" in line:
239 return
240 pytest.fail(
241 "musehub service in docker-compose.yml does not have read_only: true"
242 )
243
244 def test_tmp_is_tmpfs_or_volume(self) -> None:
245 """/tmp must be writable (tmpfs or volume) so uvicorn can write temp files."""
246 src = _COMPOSE.read_text()
247 assert "tmpfs" in src or "/tmp" in src, (
248 "No tmpfs mount for /tmp β€” uvicorn and Python will fail to write temp files "
249 "when the root filesystem is read-only"
250 )
251
252 def test_data_volume_is_explicit(self) -> None:
253 """/data object store must be an explicit named volume (not read-only)."""
254 src = _COMPOSE.read_text()
255 assert "musehub_data:/data" in src, (
256 "/data is not mounted as an explicit volume β€” objects cannot be written "
257 "when the root filesystem is read-only"
258 )
259
260
261 # ═══════════════════════════════════════════════════════════════════════════════
262 # Resource limits
263 # ═══════════════════════════════════════════════════════════════════════════════
264
265 class TestResourceLimits:
266 _src = _COMPOSE.read_text()
267
268 def _service_limits(self, service_name: str) -> str:
269 """Extract the text block for a given service."""
270 lines = self._src.splitlines()
271 in_service = False
272 block_lines = []
273 for line in lines:
274 if re.match(rf'^ {re.escape(service_name)}:', line):
275 in_service = True
276 elif re.match(r'^ \w', line) and in_service:
277 break
278 if in_service:
279 block_lines.append(line)
280 return "\n".join(block_lines)
281
282 def test_musehub_has_cpu_limit(self) -> None:
283 block = self._service_limits("musehub")
284 assert "cpus:" in block, "musehub service has no CPU limit"
285
286 def test_musehub_has_memory_limit(self) -> None:
287 block = self._service_limits("musehub")
288 assert "memory:" in block, "musehub service has no memory limit"
289
290 def test_musehub_memory_limit_sane(self) -> None:
291 """musehub memory limit must be β‰₯ 256 MiB (app needs headroom)."""
292 block = self._service_limits("musehub")
293 m = re.search(r'memory:\s*(\d+)([MmGg])', block)
294 if m:
295 amount = int(m.group(1))
296 unit = m.group(2).upper()
297 mb = amount * 1024 if unit == "G" else amount
298 assert mb >= 256, f"musehub memory limit {mb}M is below 256M minimum"
299
300 def test_postgres_has_cpu_limit(self) -> None:
301 block = self._service_limits("postgres")
302 assert "cpus:" in block, "postgres service has no CPU limit"
303
304 def test_postgres_has_memory_limit(self) -> None:
305 block = self._service_limits("postgres")
306 assert "memory:" in block, "postgres service has no memory limit"
307
308 def test_runner_has_cpu_limit(self) -> None:
309 block = self._service_limits("musehub-runner")
310 assert "cpus:" in block, "musehub-runner service has no CPU limit"
311
312 def test_runner_has_memory_limit(self) -> None:
313 block = self._service_limits("musehub-runner")
314 assert "memory:" in block, "musehub-runner service has no memory limit"
315
316 def test_all_services_have_deploy_block(self) -> None:
317 """All three services must have a deploy: block (where limits live)."""
318 for svc in ("musehub", "postgres", "musehub-runner"):
319 block = self._service_limits(svc)
320 assert "deploy:" in block, f"{svc} service has no deploy: block"
321
322
323 # ═══════════════════════════════════════════════════════════════════════════════
324 # musehub#194 β€” ECR_REGISTRY must derive from ECR_IMAGE, never be independently
325 # hardcoded. A "prod" deploy passes ECR_IMAGE pointing at Production's registry
326 # (see push.sh's per-environment ECR_REGISTRY map) β€” deploy.sh's docker
327 # login/logout must authenticate against that SAME registry, not a hardcoded
328 # Nonproduction one, or `docker pull` fails with an auth mismatch.
329 # ═══════════════════════════════════════════════════════════════════════════════
330
331 class TestEcrRegistryDerivedFromImage:
332 """Executes the real variable-derivation lines from deploy.sh in a
333 subprocess (not just a text/regex check) β€” this is a bash variable
334 scoping bug, so the test proves actual runtime behavior."""
335
336 _STAGING_IMAGE = "992382692655.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub"
337 _PROD_IMAGE = "672469410277.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub"
338
339 @staticmethod
340 def _extract_var_block() -> str:
341 """Return the exact source lines from ECR_IMAGE's definition through
342 FULL_IMAGE's definition β€” the minimal slice needed to resolve
343 ECR_REGISTRY/ECR_IMAGE/FULL_IMAGE, without deploy.sh's logging setup,
344 `cd`, or anything requiring a real /opt/musehub."""
345 src = _DEPLOY_SH.read_text()
346 start_marker = 'ECR_IMAGE="${ECR_IMAGE:-'
347 end_marker = 'FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"'
348 start = src.index(start_marker)
349 # Back up to the start of that line (the marker may not be at col 0).
350 line_start = src.rfind("\n", 0, start) + 1
351 end = src.index(end_marker) + len(end_marker)
352 return src[line_start:end]
353
354 def _resolve(self, ecr_image: str | None) -> dict[str, str]:
355 block = self._extract_var_block()
356 script = (
357 f"{block}\n"
358 'echo "ECR_REGISTRY=$ECR_REGISTRY"\n'
359 'echo "ECR_IMAGE=$ECR_IMAGE"\n'
360 'echo "FULL_IMAGE=$FULL_IMAGE"\n'
361 )
362 env: dict[str, str] = {"PATH": "/usr/bin:/bin", "IMAGE_TAG": "abc123"}
363 if ecr_image is not None:
364 env["ECR_IMAGE"] = ecr_image
365 result = subprocess.run(
366 ["bash", "-c", script],
367 env=env,
368 capture_output=True,
369 text=True,
370 timeout=5,
371 )
372 assert result.returncode == 0, result.stderr
373 out: dict[str, str] = {}
374 for line in result.stdout.splitlines():
375 key, _, value = line.partition("=")
376 out[key] = value
377 return out
378
379 def test_staging_registry_derived_from_ecr_image(self) -> None:
380 resolved = self._resolve(self._STAGING_IMAGE)
381 assert resolved["ECR_REGISTRY"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com"
382
383 def test_production_registry_derived_from_ecr_image(self) -> None:
384 """musehub#194: the exact regression β€” a prod ECR_IMAGE must resolve
385 to Production's registry, never the hardcoded staging one."""
386 resolved = self._resolve(self._PROD_IMAGE)
387 assert resolved["ECR_REGISTRY"] == "672469410277.dkr.ecr.us-east-1.amazonaws.com"
388 assert resolved["ECR_REGISTRY"] != "992382692655.dkr.ecr.us-east-1.amazonaws.com"
389
390 def test_full_image_matches_passed_ecr_image_and_tag(self) -> None:
391 resolved = self._resolve(self._PROD_IMAGE)
392 assert resolved["FULL_IMAGE"] == f"{self._PROD_IMAGE}:abc123"
393
394 def test_default_ecr_image_falls_back_to_staging_registry(self) -> None:
395 """No ECR_IMAGE set (manual on-instance invocation) defaults to
396 staging β€” matches the script's documented manual-use fallback."""
397 resolved = self._resolve(None)
398 assert resolved["ECR_REGISTRY"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com"
399
400 def test_no_independently_hardcoded_production_or_staging_registry_constant(self) -> None:
401 """Guards against reintroducing a second, independent ECR_REGISTRY
402 source of truth β€” the account ID must only ever appear as part of
403 ECR_IMAGE's default value, never as a bare ECR_REGISTRY="..." literal."""
404 src = _DEPLOY_SH.read_text()
405 assert not re.search(r'^ECR_REGISTRY="\d', src, re.MULTILINE), (
406 "ECR_REGISTRY is hardcoded to a literal account ID again β€” it must "
407 "be derived from ECR_IMAGE so docker login and docker pull always "
408 "target the same registry."
409 )
410
411
412 # ═══════════════════════════════════════════════════════════════════════════════
413 # musehub#199 β€” push.sh bash 3.2 (macOS default) compatibility
414 # ═══════════════════════════════════════════════════════════════════════════════
415
416 _SYSTEM_BASH = Path("/bin/bash")
417
418
419 def _system_bash_major_version() -> int | None:
420 """The major version of whatever /bin/bash actually is on this machine.
421 On macOS this is the real, un-upgradeable system bash 3.2 β€” the exact
422 binary the reported bug ran under. Returns None if /bin/bash is absent."""
423 if not _SYSTEM_BASH.exists():
424 return None
425 result = subprocess.run(
426 [str(_SYSTEM_BASH), "-c", "echo ${BASH_VERSINFO[0]}"],
427 capture_output=True,
428 text=True,
429 timeout=5,
430 )
431 try:
432 return int(result.stdout.strip())
433 except ValueError:
434 return None
435
436
437 def _extract_function(src: str, name: str) -> str:
438 """Return one `name() { ... }` function body verbatim from push.sh."""
439 start = src.index(f"{name}() {{")
440 end = src.index("\n}", start) + len("\n}")
441 return src[start:end]
442
443
444 class TestPushShBash32Compat:
445 """push.sh used `declare -A` (associative arrays, bash 4+ only) with no
446 version guard, so it crashed outright under macOS's default /bin/bash
447 3.2 rather than a Homebrew-installed bash 4/5 β€” 'unbound variable' at
448 the first array reference, before the script did anything useful.
449 Rewritten as `case` dispatch functions, which need no minimum bash
450 version at all."""
451
452 def test_no_declare_a_in_push_sh(self) -> None:
453 src = _PUSH_SH.read_text()
454 assert not re.search(r"^\s*declare -A", src, re.MULTILINE), (
455 "push.sh uses declare -A again, which requires bash 4+ and "
456 "crashes outright under macOS's default bash 3.2."
457 )
458
459 @pytest.mark.skipif(
460 _system_bash_major_version() is None or _system_bash_major_version() >= 4,
461 reason="requires a real bash < 4 (e.g. macOS's /bin/bash) to prove the fix",
462 )
463 def test_usage_message_prints_under_real_bash_32(self) -> None:
464 """The exact reported failure: `bash deploy/push.sh` with no args
465 used to crash at `declare -A` with 'unbound variable' before ever
466 reaching the usage message. Runs the real script end to end."""
467 result = subprocess.run(
468 [str(_SYSTEM_BASH), str(_PUSH_SH)],
469 capture_output=True,
470 text=True,
471 timeout=10,
472 )
473 assert result.returncode == 1
474 assert "Usage: bash deploy/push.sh" in result.stdout
475 assert "unbound variable" not in result.stderr
476
477 @pytest.mark.skipif(
478 _system_bash_major_version() is None or _system_bash_major_version() >= 4,
479 reason="requires a real bash < 4 (e.g. macOS's /bin/bash) to prove the fix",
480 )
481 def test_env_helper_functions_resolve_correctly_under_bash_32(self) -> None:
482 """The replacement case-dispatch helpers must resolve the exact same
483 values the old associative arrays held, not just avoid crashing."""
484 src = _PUSH_SH.read_text()
485 block = "\n".join(
486 _extract_function(src, name)
487 for name in ("instance_id_for", "ecr_registry_for", "aws_profile_for")
488 )
489 script = (
490 f"{block}\n"
491 'echo "INSTANCE_STAGING=$(instance_id_for staging)"\n'
492 'echo "INSTANCE_PROD=$(instance_id_for prod)"\n'
493 'echo "REGISTRY_STAGING=$(ecr_registry_for staging)"\n'
494 'echo "REGISTRY_PROD=$(ecr_registry_for prod)"\n'
495 'echo "PROFILE_STAGING=$(aws_profile_for staging)"\n'
496 'echo "PROFILE_PROD=$(aws_profile_for prod)"\n'
497 )
498 result = subprocess.run(
499 [str(_SYSTEM_BASH), "-c", script],
500 capture_output=True,
501 text=True,
502 timeout=5,
503 )
504 assert result.returncode == 0, result.stderr
505 out = dict(line.split("=", 1) for line in result.stdout.splitlines())
506 assert out["INSTANCE_STAGING"] == "i-07547cd20bee2dea5"
507 assert out["INSTANCE_PROD"] == "i-043aaed71bef11903"
508 assert out["REGISTRY_STAGING"] == "992382692655.dkr.ecr.us-east-1.amazonaws.com"
509 assert out["REGISTRY_PROD"] == "672469410277.dkr.ecr.us-east-1.amazonaws.com"
510 assert out["PROFILE_STAGING"] == "musehub-nonproduction"
511 assert out["PROFILE_PROD"] == "musehub-production"