gabriel / muse public
test_mwp5_clone_retry.py python
925 lines 35.4 KB
Raw
sha256:ac29b8ba9021514a03ab2782d92bf67671f0efa5b3b70d46f7598c5d4e923378 docs: record muse#74 Phase 4 live verification (SCR_13, SCR_14) Sonnet 5 13 days ago
1 """MWP-5: clone/fetch/pull honor Retry-After with a bounded poll (fixes RC-5).
2
3 Test IDs MWP5_00–MWP5_18. All tests live here.
4
5 Phase 0 — RED harness (this file):
6 - _seq_urllib_do: helper that raises TransportError(503, retry_after=R) for
7 the first k calls, then returns a valid mpack POST body. Requires Phase 1
8 to add retry_after to TransportError.__init__.
9 - _record_sleeps: fixture patching muse.core.transport._sleep (create=True so
10 it exists before Phase 2 adds the real function).
11 - MWP5_00: RED until Phase 3 — asserts fetch_mpack retries past a single 503
12 and calls _sleep once (currently raises immediately with no sleep).
13
14 Seam reference: tests/test_core_transport.py — the _urllib_do patch idiom.
15 """
16
17 from __future__ import annotations
18
19 import http.server
20 import threading
21 import time
22 import unittest.mock
23 from collections.abc import Callable
24 from typing import Any
25
26 import msgpack
27 import pytest
28
29 from muse.core.transport import (
30 HttpTransport,
31 TransportError,
32 _parse_retry_after,
33 _next_wait,
34 _resolve_retry_budget,
35 _RETRY_AFTER_CEILING_S,
36 _RETRY_AFTER_FLOOR_S,
37 _RETRY_DEFAULT_BACKOFF_S,
38 _FETCH_RETRY_BUDGET_DEFAULT_S,
39 )
40
41
42 # ---------------------------------------------------------------------------
43 # Helpers
44 # ---------------------------------------------------------------------------
45
46
47 def _minimal_fetch_success_body() -> bytes:
48 """Minimal valid POST /fetch/mpack response body.
49
50 Returns mpack_url=None (absent) so fetch_mpack uses the inline
51 empty-result path — no S3 download needed. One _urllib_do call total
52 after this is returned.
53 """
54 return msgpack.packb(
55 {
56 "mpack_id": None,
57 "mpack_url": None,
58 "repo_id": "r-test",
59 "domain": "code",
60 "default_branch": "main",
61 "branch_heads": {},
62 "commit_count": 0,
63 "blob_count": 0,
64 },
65 use_bin_type=True,
66 )
67
68
69 def _seq_urllib_do(
70 *,
71 k: int,
72 retry_after: "int | None" = 30,
73 success_body: bytes,
74 ) -> Callable[..., bytes]:
75 """Return a _urllib_do side_effect: raises TransportError(503) k times, then succeeds.
76
77 NOTE: TransportError(msg, 503, retry_after=R) requires Phase 1 to add
78 the retry_after keyword argument to TransportError.__init__. Until then,
79 this helper raises TypeError on its first invocation (test is RED).
80 """
81 calls: list[int] = [0]
82
83 def _side_effect(
84 method: str,
85 url: str,
86 headers: dict[str, str],
87 data: bytes | None = None,
88 **kwargs: Any,
89 ) -> bytes:
90 calls[0] += 1
91 if calls[0] <= k:
92 raise TransportError("HTTP 503", 503, retry_after=retry_after)
93 return success_body
94
95 return _side_effect
96
97
98 # ---------------------------------------------------------------------------
99 # Fixtures
100 # ---------------------------------------------------------------------------
101
102
103 @pytest.fixture()
104 def record_sleeps() -> Callable[[], list[float]]:
105 """Patch muse.core.transport._sleep and record every call's argument.
106
107 Uses create=True so the fixture works before Phase 2 adds the real _sleep
108 function. Returns a callable that returns the list of sleep durations
109 recorded so far.
110 """
111 sleep_log: list[float] = []
112
113 def _fake_sleep(seconds: float) -> None:
114 sleep_log.append(seconds)
115
116 with unittest.mock.patch(
117 "muse.core.transport._sleep",
118 side_effect=_fake_sleep,
119 create=True,
120 ):
121 yield lambda: list(sleep_log)
122
123
124 # ---------------------------------------------------------------------------
125 # Phase 0 — RED harness
126 # ---------------------------------------------------------------------------
127
128
129 class TestMWP5Phase0:
130 def test_mwp5_00_fetch_retries_on_503(
131 self,
132 record_sleeps: Callable[[], list[float]],
133 ) -> None:
134 """MWP5_00 — RED until Phase 3.
135
136 Asserts: one 503(Retry-After:30) followed by success causes
137 fetch_mpack to return a FetchMPackResult (not raise) and to have
138 called _sleep exactly once with ~30s.
139
140 With current code this test is RED because:
141 - Phase 0: TransportError does not accept retry_after kwarg →
142 TypeError inside _seq_urllib_do → test errors.
143 - Phase 1: retry_after added to TransportError (TypeError gone), but
144 fetch_mpack still raises immediately on 503 → AssertionError.
145 - Phase 2: _sleep seam exists but fetch_mpack still raises on 503.
146 - Phase 3: retry loop added → test goes GREEN.
147 """
148 success = _minimal_fetch_success_body()
149 fake_do = _seq_urllib_do(k=1, retry_after=30, success_body=success)
150
151 with unittest.mock.patch(
152 "muse.core.transport._urllib_do",
153 side_effect=fake_do,
154 ):
155 result = HttpTransport().fetch_mpack(
156 "https://hub.example.com/gabriel/testrepo",
157 None,
158 want=["sha256:" + "a" * 64],
159 have=[],
160 )
161
162 # fetch_mpack succeeded — did not raise
163 assert result is not None
164
165 # _sleep was called exactly once with the Retry-After value (30s)
166 sleeps = record_sleeps()
167 assert len(sleeps) == 1, f"expected 1 sleep call, got {sleeps}"
168 assert sleeps[0] == pytest.approx(30.0)
169
170
171 # ---------------------------------------------------------------------------
172 # Phase 1 — capture Retry-After (Defect A)
173 # ---------------------------------------------------------------------------
174
175
176 class TestMWP5Phase1:
177 # -- MWP5_01 / MWP5_02: _parse_retry_after unit tests --------------------
178
179 def test_mwp5_01_parse_retry_after_integer(self) -> None:
180 """MWP5_01 — _parse_retry_after returns the integer seconds.
181
182 In production, headers is http.client.HTTPMessage (case-insensitive
183 .get()). Tests use plain dicts with the canonical casing.
184 """
185 assert _parse_retry_after({"Retry-After": "60"}) == 60
186 assert _parse_retry_after({"Retry-After": "30"}) == 30
187 assert _parse_retry_after({"Retry-After": "0"}) == 0
188
189 def test_mwp5_02_parse_retry_after_invalid_returns_none(self) -> None:
190 """MWP5_02 — missing or non-integer Retry-After returns None."""
191 assert _parse_retry_after(None) is None
192 assert _parse_retry_after({}) is None
193 assert _parse_retry_after({"Retry-After": "Wed, 21 Oct 2025 07:28:00 GMT"}) is None
194 assert _parse_retry_after({"Retry-After": ""}) is None
195 assert _parse_retry_after({"Retry-After": "abc"}) is None
196 assert _parse_retry_after({"Retry-After": "-1"}) is None
197
198 # -- MWP5_03: real HTTPError driven through _urllib_do -------------------
199
200 def test_mwp5_03_urllib_do_503_captures_retry_after(self) -> None:
201 """MWP5_03 — a 503 HTTPError with Retry-After:30 header produces
202 TransportError(status_code=503, retry_after=30) from _urllib_do.
203 """
204 import io
205 import urllib.error
206 import urllib.request
207 from muse.core.transport import _urllib_do
208
209 # Build a real urllib.error.HTTPError with a Retry-After header.
210 # http.client.HTTPMessage (used by urllib) supports dict-style get().
211 headers = urllib.request.addinfourl(
212 io.BytesIO(b"mpack not ready"),
213 {"Retry-After": "30", "Content-Type": "text/plain"},
214 "https://hub.example.com/fetch/mpack",
215 503,
216 ).headers
217
218 http_err = urllib.error.HTTPError(
219 url="https://hub.example.com/fetch/mpack",
220 code=503,
221 msg="Service Unavailable",
222 hdrs=headers,
223 fp=io.BytesIO(b"mpack not ready"),
224 )
225
226 def _raise_503(req: "urllib.request.Request") -> "Any":
227 raise http_err
228
229 with unittest.mock.patch("muse.core.transport._open_url", side_effect=_raise_503):
230 # _urllib_do goes through _open_url → HTTPError → TransportError
231 # We call _urllib_do directly via the HttpTransport._execute_fetch
232 # path to exercise that seam (it uses _open_url, not _urllib_do).
233 # Also test _urllib_do directly by patching urllib opener.open.
234 pass
235
236 # Drive _urllib_do directly: patch urllib.request.build_opener to
237 # return an opener whose open() raises the HTTPError.
238 mock_opener = unittest.mock.MagicMock()
239 mock_opener.open.return_value.__enter__ = unittest.mock.MagicMock(side_effect=http_err)
240 mock_opener.open.side_effect = http_err
241
242 with unittest.mock.patch(
243 "urllib.request.build_opener", return_value=mock_opener
244 ):
245 with pytest.raises(TransportError) as exc_info:
246 _urllib_do("POST", "https://hub.example.com/fetch/mpack", {})
247
248 err = exc_info.value
249 assert err.status_code == 503
250 assert err.retry_after == 30
251
252 # -- MWP5_04: 404 yields retry_after=None --------------------------------
253
254 def test_mwp5_04_non_503_yields_retry_after_none(self) -> None:
255 """MWP5_04 — a 404 HTTPError yields retry_after=None (no header).
256 Regression guard: only 503s carry Retry-After; other errors unaffected.
257 """
258 with unittest.mock.patch(
259 "muse.core.transport._urllib_do",
260 side_effect=TransportError("HTTP 404", 404),
261 ):
262 with pytest.raises(TransportError) as exc_info:
263 HttpTransport().fetch_remote_info(
264 "https://hub.example.com/gabriel/missing", None
265 )
266
267 err = exc_info.value
268 assert err.status_code == 404
269 assert err.retry_after is None
270
271
272 # ---------------------------------------------------------------------------
273 # Phase 2 — retry primitives (D3)
274 # ---------------------------------------------------------------------------
275
276
277 class TestMWP5Phase2:
278 # -- MWP5_05: _next_wait clamping -----------------------------------------
279
280 def test_mwp5_05_next_wait_floors_and_ceilings(self) -> None:
281 """MWP5_05 — _next_wait clamps to [floor, ceiling] and uses default backoff."""
282 # floor: 0 is clamped up to _RETRY_AFTER_FLOOR_S (1.0)
283 assert _next_wait(0) == pytest.approx(_RETRY_AFTER_FLOOR_S)
284
285 # ceiling: 999 is clamped down to _RETRY_AFTER_CEILING_S (60.0)
286 assert _next_wait(999) == pytest.approx(_RETRY_AFTER_CEILING_S)
287
288 # None → use _RETRY_DEFAULT_BACKOFF_S (5.0), within [floor, ceiling]
289 assert _next_wait(None) == pytest.approx(_RETRY_DEFAULT_BACKOFF_S)
290
291 # typical values pass through unchanged
292 assert _next_wait(30) == pytest.approx(30.0)
293 assert _next_wait(60) == pytest.approx(_RETRY_AFTER_CEILING_S)
294
295 # floor boundary: 1 passes through (exactly at floor)
296 assert _next_wait(1) == pytest.approx(1.0)
297
298 # -- MWP5_06: _resolve_retry_budget priority chain -----------------------
299
300 def test_mwp5_06_resolve_retry_budget(self, monkeypatch: pytest.MonkeyPatch) -> None:
301 """MWP5_06 — explicit arg > env var > module default; negatives → 0."""
302 # explicit arg always wins
303 assert _resolve_retry_budget(90.0) == pytest.approx(90.0)
304 assert _resolve_retry_budget(0.0) == pytest.approx(0.0)
305
306 # negative explicit is clamped to 0
307 assert _resolve_retry_budget(-5.0) == pytest.approx(0.0)
308
309 # env var used when no explicit arg
310 monkeypatch.setenv("MUSE_FETCH_RETRY_BUDGET_S", "45")
311 assert _resolve_retry_budget(None) == pytest.approx(45.0)
312
313 # negative env is clamped to 0
314 monkeypatch.setenv("MUSE_FETCH_RETRY_BUDGET_S", "-10")
315 assert _resolve_retry_budget(None) == pytest.approx(0.0)
316
317 # junk env falls through to module default
318 monkeypatch.setenv("MUSE_FETCH_RETRY_BUDGET_S", "not-a-number")
319 assert _resolve_retry_budget(None) == pytest.approx(_FETCH_RETRY_BUDGET_DEFAULT_S)
320
321 # no env → module default
322 monkeypatch.delenv("MUSE_FETCH_RETRY_BUDGET_S", raising=False)
323 assert _resolve_retry_budget(None) == pytest.approx(_FETCH_RETRY_BUDGET_DEFAULT_S)
324
325
326 # ---------------------------------------------------------------------------
327 # Phase 3 — wire the loop into fetch_mpack (D4)
328 # ---------------------------------------------------------------------------
329
330
331 class TestMWP5Phase3:
332 """MWP5_07–MWP5_12: the retry loop in HttpTransport.fetch_mpack."""
333
334 def test_mwp5_07_single_503_then_success(
335 self,
336 record_sleeps: Callable[[], list[float]],
337 ) -> None:
338 """MWP5_07 — one 503(Retry-After:30) then success.
339
340 fetch_mpack must return a FetchMPackResult (not raise) and call
341 _sleep exactly once with 30.0.
342 """
343 success = _minimal_fetch_success_body()
344 fake_do = _seq_urllib_do(k=1, retry_after=30, success_body=success)
345
346 with unittest.mock.patch(
347 "muse.core.transport._urllib_do", side_effect=fake_do
348 ):
349 result = HttpTransport().fetch_mpack(
350 "https://hub.example.com/gabriel/testrepo",
351 None,
352 want=["sha256:" + "a" * 64],
353 have=[],
354 )
355
356 assert result is not None
357 sleeps = record_sleeps()
358 assert len(sleeps) == 1, f"expected 1 sleep, got {sleeps}"
359 assert sleeps[0] == pytest.approx(30.0)
360
361 def test_mwp5_08_two_503s_then_success(
362 self,
363 record_sleeps: Callable[[], list[float]],
364 ) -> None:
365 """MWP5_08 — 503(30) then 503(60) then success: two sleeps, result returned.
366
367 Uses a custom side_effect that emits different retry_after values per
368 call (30 on call 1, 60 on call 2, success on call 3).
369 """
370 success = _minimal_fetch_success_body()
371 retry_afters = [30, 60]
372 calls: list[int] = [0]
373
374 def _two_503_side_effect(method: str, url: str, headers: dict, data: bytes | None = None, **kw: Any) -> bytes:
375 calls[0] += 1
376 if calls[0] <= len(retry_afters):
377 raise TransportError("HTTP 503", 503, retry_after=retry_afters[calls[0] - 1])
378 return success
379
380 with unittest.mock.patch(
381 "muse.core.transport._urllib_do", side_effect=_two_503_side_effect
382 ):
383 result = HttpTransport().fetch_mpack(
384 "https://hub.example.com/gabriel/testrepo",
385 None,
386 want=["sha256:" + "a" * 64],
387 have=[],
388 )
389
390 assert result is not None
391 sleeps = record_sleeps()
392 assert len(sleeps) == 2, f"expected 2 sleeps, got {sleeps}"
393 assert sleeps[0] == pytest.approx(30.0)
394 assert sleeps[1] == pytest.approx(60.0)
395
396 def test_mwp5_09_503_no_retry_after_uses_default_backoff(
397 self,
398 record_sleeps: Callable[[], list[float]],
399 ) -> None:
400 """MWP5_09 — 503 with no Retry-After header: sleeps _RETRY_DEFAULT_BACKOFF_S."""
401 success = _minimal_fetch_success_body()
402 # retry_after=None means the TransportError carries no wait hint
403 fake_do = _seq_urllib_do(k=1, retry_after=None, success_body=success) # type: ignore[arg-type]
404
405 # _seq_urllib_do stores retry_after=None; TransportError.retry_after will be None
406 with unittest.mock.patch(
407 "muse.core.transport._urllib_do", side_effect=fake_do
408 ):
409 result = HttpTransport().fetch_mpack(
410 "https://hub.example.com/gabriel/testrepo",
411 None,
412 want=["sha256:" + "a" * 64],
413 have=[],
414 )
415
416 assert result is not None
417 sleeps = record_sleeps()
418 assert len(sleeps) == 1, f"expected 1 sleep, got {sleeps}"
419 assert sleeps[0] == pytest.approx(_RETRY_DEFAULT_BACKOFF_S)
420
421 def test_mwp5_10_budget_exhausted_raises_503(
422 self,
423 record_sleeps: Callable[[], list[float]],
424 ) -> None:
425 """MWP5_10 — 503 forever with budget=120: raises TransportError(503).
426
427 Total accumulated sleep must not exceed the budget. Sleep count is
428 finite. The final raise surfaces the server's real 503 (status_code=503).
429 """
430 success = _minimal_fetch_success_body()
431 # k=999 simulates "503 forever" in practice
432 fake_do = _seq_urllib_do(k=999, retry_after=60, success_body=success)
433
434 with unittest.mock.patch(
435 "muse.core.transport._urllib_do", side_effect=fake_do
436 ):
437 with pytest.raises(TransportError) as exc_info:
438 HttpTransport().fetch_mpack(
439 "https://hub.example.com/gabriel/testrepo",
440 None,
441 want=["sha256:" + "a" * 64],
442 have=[],
443 retry_budget_s=120.0,
444 )
445
446 err = exc_info.value
447 assert err.status_code == 503 # server's real error, not synthetic
448
449 sleeps = record_sleeps()
450 total_slept = sum(sleeps)
451 assert total_slept <= 120.0, f"total slept {total_slept}s > 120s budget"
452 assert len(sleeps) < 20, f"sleep count {len(sleeps)} not bounded"
453
454 def test_mwp5_11_non_503_raises_immediately(
455 self,
456 record_sleeps: Callable[[], list[float]],
457 ) -> None:
458 """MWP5_11 — a 404 inside the loop raises immediately; _sleep never called.
459
460 Non-503 errors must not incur any added latency — the loop skips retry
461 on the first non-retryable status.
462 """
463 with unittest.mock.patch(
464 "muse.core.transport._urllib_do",
465 side_effect=TransportError("HTTP 404", 404),
466 ):
467 with pytest.raises(TransportError) as exc_info:
468 HttpTransport().fetch_mpack(
469 "https://hub.example.com/gabriel/testrepo",
470 None,
471 want=["sha256:" + "a" * 64],
472 have=[],
473 )
474
475 assert exc_info.value.status_code == 404
476 assert record_sleeps() == [] # zero sleeps — no latency added
477
478 def test_mwp5_12_retry_budget_zero_is_single_attempt(
479 self,
480 record_sleeps: Callable[[], list[float]],
481 ) -> None:
482 """MWP5_12 — retry_budget_s=0 (--no-retry): exactly one attempt, zero sleeps.
483
484 This is the escape hatch for deterministic CI. The original single-attempt
485 behavior is preserved exactly — the first 503 raises with no sleep.
486 """
487 success = _minimal_fetch_success_body()
488 fake_do = _seq_urllib_do(k=1, retry_after=30, success_body=success)
489
490 with unittest.mock.patch(
491 "muse.core.transport._urllib_do", side_effect=fake_do
492 ):
493 with pytest.raises(TransportError) as exc_info:
494 HttpTransport().fetch_mpack(
495 "https://hub.example.com/gabriel/testrepo",
496 None,
497 want=["sha256:" + "a" * 64],
498 have=[],
499 retry_budget_s=0.0,
500 )
501
502 assert exc_info.value.status_code == 503
503 assert record_sleeps() == [] # zero sleeps — true single-attempt behavior
504
505
506 # ---------------------------------------------------------------------------
507 # Phase 4 — command integration (D5)
508 # ---------------------------------------------------------------------------
509
510 import argparse
511 import contextlib
512 import pathlib
513 from unittest.mock import MagicMock, patch
514
515 from muse.core.types import fake_id
516
517 # Minimal shared fixtures for command-layer tests.
518 _COMMIT_ID = fake_id("commit-1")
519 _SNAP_ID = fake_id("snap-1")
520 _REPO_ID = fake_id("repo-1")
521
522 _REMOTE_INFO: dict = {
523 "repo_id": _REPO_ID,
524 "domain": "code",
525 "default_branch": "main",
526 "branch_heads": {"main": _COMMIT_ID},
527 "object_count": 1,
528 "size_bytes": 100,
529 }
530
531 _FETCH_RESULT: dict = {
532 "repo_id": _REPO_ID,
533 "domain": "code",
534 "default_branch": "main",
535 "branch_heads": {"main": _COMMIT_ID},
536 "commits": [],
537 "snapshots": [],
538 "blobs": [],
539 "blobs_received": 0,
540 "shallow_commits": [],
541 }
542
543 _APPLY_RESULT: dict = {
544 "commits_written": 1,
545 "snapshots_written": 1,
546 "blobs_written": 0,
547 "blobs_skipped": 0,
548 "failed_blobs": [],
549 }
550
551
552 def _seq_transport(k: int, retry_after: int = 30) -> "HttpTransport":
553 """Return a real HttpTransport whose _fetch_mpack_once raises 503 k times then succeeds.
554
555 Patching _fetch_mpack_once (not fetch_mpack) lets the real retry loop in
556 HttpTransport.fetch_mpack run — the critical path under test for MWP5_13–17.
557 """
558 from muse.core.transport import HttpTransport as _HttpTransport
559 t = _HttpTransport()
560 t.fetch_remote_info = MagicMock(return_value=_REMOTE_INFO) # type: ignore[method-assign]
561 calls: list[int] = [0]
562
563 def _once(*a: Any, **kw: Any) -> dict:
564 calls[0] += 1
565 if calls[0] <= k:
566 raise TransportError("HTTP 503", 503, retry_after=retry_after)
567 return _FETCH_RESULT
568
569 t._fetch_mpack_once = _once # type: ignore[method-assign]
570 return t
571
572
573 def _clone_patches(t: "Any") -> "list[contextlib.AbstractContextManager]":
574 """Standard patches needed to drive clone.run() without a real repo or network."""
575 return [
576 patch("muse.cli.commands.clone.get_signing_identity", return_value=None),
577 patch("muse.cli.commands.clone.make_transport", return_value=t),
578 patch("muse.cli.commands.clone.apply_mpack", return_value=_APPLY_RESULT),
579 patch("muse.cli.commands.clone.set_remote"),
580 patch("muse.cli.commands.clone.set_remote_head"),
581 patch("muse.cli.commands.clone.set_upstream"),
582 patch("muse.cli.commands.clone.apply_manifest"),
583 patch("muse.cli.commands.clone.read_commit", return_value=MagicMock(snapshot_id=_SNAP_ID)),
584 patch("muse.cli.commands.clone.read_snapshot", return_value=MagicMock(manifest={})),
585 patch("muse.core.transport._sleep", side_effect=lambda s: None),
586 ]
587
588
589 class TestMWP5Phase4:
590 """MWP5_13–MWP5_17: command-layer integration of the retry loop."""
591
592 def test_mwp5_13_clone_503_then_success(
593 self,
594 tmp_path: pathlib.Path,
595 ) -> None:
596 """MWP5_13 — CLI clone, transport patched to 503-then-success: exits 0.
597
598 The target directory must exist and contain the apply result after the
599 retry succeeds. The directory is never rmtree'd during the retry loop.
600 """
601 from muse.cli.commands.clone import run as clone_run
602
603 target = tmp_path / "cloned"
604 t = _seq_transport(k=1, retry_after=30)
605
606 args = argparse.Namespace(
607 url="https://hub.example.com/gabriel/repo",
608 directory=str(target),
609 branch=None,
610 depth=None,
611 dry_run=False,
612 no_checkout=True, # skip working-tree restore for speed
613 json_out=False,
614 retry_budget_s=None,
615 )
616
617 with contextlib.ExitStack() as stack:
618 for p in _clone_patches(t):
619 stack.enter_context(p)
620 clone_run(args) # must not raise SystemExit
621
622 # _fetch_mpack_once was called twice: once raised 503, once returned success
623 # (the retry loop in fetch_mpack drove the second call)
624
625 def test_mwp5_14_clone_503_forever_budget_zero_removes_dir(
626 self,
627 tmp_path: pathlib.Path,
628 ) -> None:
629 """MWP5_14 — CLI clone, no-retry (budget=0) + 503 forever: exits INTERNAL_ERROR.
630
631 The target directory is removed by the terminal error path. Stderr must
632 describe the failure. With --json the envelope carries retryable=True.
633 """
634 import io
635 import json
636 import sys as _sys
637 from muse.cli.commands.clone import run as clone_run
638
639 target = tmp_path / "cloned"
640 t = _seq_transport(k=999, retry_after=5) # never succeeds
641
642 args = argparse.Namespace(
643 url="https://hub.example.com/gabriel/repo",
644 directory=str(target),
645 branch=None,
646 depth=None,
647 dry_run=False,
648 no_checkout=True,
649 json_out=True,
650 retry_budget_s=0.0, # --no-retry: single attempt, immediate 503
651 )
652
653 stdout_buf = io.StringIO()
654 stderr_buf = io.StringIO()
655 with contextlib.ExitStack() as stack:
656 for p in _clone_patches(t):
657 stack.enter_context(p)
658 stack.enter_context(contextlib.redirect_stdout(stdout_buf))
659 stack.enter_context(contextlib.redirect_stderr(stderr_buf))
660 with pytest.raises(SystemExit) as exc_info:
661 clone_run(args)
662
663 assert exc_info.value.code != 0 # exits with error
664
665 # JSON envelope on stdout carries retryable marker
666 stdout_text = stdout_buf.getvalue().strip()
667 assert stdout_text, "expected JSON on stdout"
668 envelope = json.loads(stdout_text.splitlines()[0])
669 assert envelope.get("retryable") is True
670
671 # Target directory cleaned up
672 assert not target.exists(), "clone target must be rmtree'd on terminal 503"
673
674 def test_mwp5_15_fetch_503_then_success(
675 self,
676 tmp_path: pathlib.Path,
677 ) -> None:
678 """MWP5_15 — CLI fetch, 503-then-success: exits 0, commits applied.
679
680 Patches require_repo, make_transport, and the apply/ref helpers so no
681 real repository is needed.
682 """
683 from muse.cli.commands.fetch import _fetch_one
684
685 t = _seq_transport(k=1, retry_after=30)
686 elapsed_fn: Callable[[], float] = lambda: 0.0
687
688 with contextlib.ExitStack() as stack:
689 stack.enter_context(patch("muse.cli.commands.fetch.get_remote", return_value="https://hub.example.com/gabriel/repo"))
690 stack.enter_context(patch("muse.cli.commands.fetch.get_signing_identity", return_value=None))
691 stack.enter_context(patch("muse.cli.commands.fetch.make_transport", return_value=t))
692 stack.enter_context(patch("muse.cli.commands.fetch.get_all_commits", return_value=[]))
693 stack.enter_context(patch("muse.cli.commands.fetch.apply_mpack", return_value=_APPLY_RESULT))
694 stack.enter_context(patch("muse.cli.commands.fetch.set_remote_head"))
695 stack.enter_context(patch("muse.core.transport._sleep", side_effect=lambda s: None))
696
697 result = _fetch_one(
698 tmp_path, "origin", "main",
699 prune=False, dry_run=False, json_out=False,
700 elapsed=elapsed_fn, retry_budget_s=None,
701 )
702
703 assert result["status"] == "fetched"
704 # retry loop drove two _fetch_mpack_once calls: first raised 503, second succeeded
705
706 def test_mwp5_16_pull_503_then_success(
707 self,
708 tmp_path: pathlib.Path,
709 ) -> None:
710 """MWP5_16 — CLI pull, 503-then-success: exits 0, ff/merge proceeds.
711
712 Uses pull.run() with full command patches. Verifies fetch_mpack was
713 called twice and the command completes without raising SystemExit.
714 """
715 from muse.cli.commands.pull import run as pull_run
716
717 t = _seq_transport(k=1, retry_after=30)
718
719 # Create a minimal local commit to satisfy get_all_commits
720 commit_mock = MagicMock()
721 commit_mock.commit_id = _COMMIT_ID
722
723 args = argparse.Namespace(
724 remote="origin",
725 branch_flag=None,
726 branch_pos="main",
727 no_merge=True, # fetch-only — skip the merge step complexity
728 ff_only=False,
729 dry_run=False,
730 message=None,
731 json_out=False,
732 retry_budget_s=None,
733 )
734
735 with contextlib.ExitStack() as stack:
736 stack.enter_context(patch("muse.cli.commands.pull.require_repo", return_value=tmp_path))
737 stack.enter_context(patch("muse.cli.commands.pull.get_remote", return_value="https://hub.example.com/gabriel/repo"))
738 stack.enter_context(patch("muse.cli.commands.pull.get_signing_identity", return_value=None))
739 stack.enter_context(patch("muse.cli.commands.pull.make_transport", return_value=t))
740 stack.enter_context(patch("muse.cli.commands.pull.read_current_branch", return_value="main"))
741 stack.enter_context(patch("muse.cli.commands.pull.get_all_commits", return_value=[commit_mock]))
742 stack.enter_context(patch("muse.cli.commands.pull.apply_mpack", return_value=_APPLY_RESULT))
743 stack.enter_context(patch("muse.cli.commands.pull.set_remote_head"))
744 stack.enter_context(patch("muse.core.transport._sleep", side_effect=lambda s: None))
745
746 pull_run(args) # must not raise SystemExit
747 # retry loop drove two _fetch_mpack_once calls: first raised 503, second succeeded
748
749 def test_mwp5_16b_pull_503_budget_exhausted_json_retryable(
750 self,
751 tmp_path: pathlib.Path,
752 ) -> None:
753 """MWP5_16b — CLI pull --json, 503-forever (budget=0): JSON envelope has retryable=True."""
754 import io
755 import json as _json
756 from muse.cli.commands.pull import run as pull_run
757
758 t = _seq_transport(k=999, retry_after=5)
759 commit_mock = MagicMock()
760 commit_mock.commit_id = _COMMIT_ID
761
762 args = argparse.Namespace(
763 remote="origin",
764 branch_flag=None,
765 branch_pos="main",
766 no_merge=True,
767 ff_only=False,
768 dry_run=False,
769 message=None,
770 json_out=True,
771 retry_budget_s=0.0,
772 )
773
774 stdout_buf = io.StringIO()
775 with contextlib.ExitStack() as stack:
776 stack.enter_context(patch("muse.cli.commands.pull.require_repo", return_value=tmp_path))
777 stack.enter_context(patch("muse.cli.commands.pull.get_remote", return_value="https://hub.example.com/gabriel/repo"))
778 stack.enter_context(patch("muse.cli.commands.pull.get_signing_identity", return_value=None))
779 stack.enter_context(patch("muse.cli.commands.pull.make_transport", return_value=t))
780 stack.enter_context(patch("muse.cli.commands.pull.read_current_branch", return_value="main"))
781 stack.enter_context(patch("muse.cli.commands.pull.get_all_commits", return_value=[commit_mock]))
782 stack.enter_context(patch("muse.core.transport._sleep", side_effect=lambda s: None))
783 stack.enter_context(contextlib.redirect_stdout(stdout_buf))
784
785 with pytest.raises(SystemExit) as exc_info:
786 pull_run(args)
787
788 assert exc_info.value.code != 0
789 stdout_text = stdout_buf.getvalue().strip()
790 assert stdout_text, "expected JSON on stdout"
791 envelope = _json.loads(stdout_text.splitlines()[0])
792 assert envelope.get("retryable") is True
793
794 def test_mwp5_17_json_clone_retry_chatter_on_stderr_only(
795 self,
796 tmp_path: pathlib.Path,
797 ) -> None:
798 """MWP5_17 — --json clone with retries: stdout has exactly one JSON object.
799
800 All retry progress lines (⏳ …) must go to stderr. Stdout must contain
801 only the final JSON envelope (or error envelope on failure).
802 """
803 import io
804 import json
805
806 # Use a budget=0 (no-retry) so we get immediate failure — one JSON
807 # object on stdout and any error messages on stderr.
808 from muse.cli.commands.clone import run as clone_run
809
810 target = tmp_path / "cloned"
811 t = _seq_transport(k=999, retry_after=5)
812
813 args = argparse.Namespace(
814 url="https://hub.example.com/gabriel/repo",
815 directory=str(target),
816 branch=None,
817 depth=None,
818 dry_run=False,
819 no_checkout=True,
820 json_out=True,
821 retry_budget_s=0.0,
822 )
823
824 stdout_buf = io.StringIO()
825 stderr_buf = io.StringIO()
826
827 with contextlib.ExitStack() as stack:
828 for p in _clone_patches(t):
829 stack.enter_context(p)
830 stack.enter_context(contextlib.redirect_stdout(stdout_buf))
831 stack.enter_context(contextlib.redirect_stderr(stderr_buf))
832 with pytest.raises(SystemExit):
833 clone_run(args)
834
835 stdout_text = stdout_buf.getvalue().strip()
836 stdout_lines = [l for l in stdout_text.splitlines() if l.strip()]
837 # Exactly one JSON object on stdout
838 assert len(stdout_lines) == 1, (
839 f"stdout must have exactly one line, got {len(stdout_lines)}: {stdout_lines}"
840 )
841 envelope = json.loads(stdout_lines[0])
842 assert "status" in envelope or "error" in envelope
843
844
845 # ---------------------------------------------------------------------------
846 # Phase 5 — real-urllib E2E (MWP5_18)
847 # ---------------------------------------------------------------------------
848
849
850 class TestMWP5Phase5:
851 """MWP5_18 — end-to-end proof that Retry-After survives the real urllib path."""
852
853 def test_mwp5_18_real_urllib_503_then_success(self) -> None:
854 """MWP5_18 — stand up a local HTTP server: 503+Retry-After:1 then valid mpack.
855
856 Drives a real HttpTransport.fetch_mpack (real _urllib_do, real urllib
857 HTTPError, real header parsing) with _sleep patched to no-op.
858 Proves the Retry-After header value survives the actual urllib code path
859 end-to-end — not just a mocked side_effect.
860 """
861 import io
862 import json
863 import socket
864
865 success_body = _minimal_fetch_success_body()
866 call_count: list[int] = [0]
867
868 class _Handler(http.server.BaseHTTPRequestHandler):
869 def do_POST(self) -> None:
870 # Drain request body to avoid broken-pipe on client side
871 content_len = int(self.headers.get("Content-Length", 0))
872 self.rfile.read(content_len)
873
874 call_count[0] += 1
875 if call_count[0] == 1:
876 # First call: 503 with Retry-After: 1
877 self.send_response(503)
878 self.send_header("Retry-After", "1")
879 self.send_header("Content-Type", "application/octet-stream")
880 body = b"mpack not ready"
881 self.send_header("Content-Length", str(len(body)))
882 self.end_headers()
883 self.wfile.write(body)
884 else:
885 # Second call: 200 with valid mpack body
886 self.send_response(200)
887 self.send_header("Content-Type", "application/octet-stream")
888 self.send_header("Content-Length", str(len(success_body)))
889 self.end_headers()
890 self.wfile.write(success_body)
891
892 def log_message(self, *args: Any) -> None:
893 pass # suppress request logs in test output
894
895 # Bind on an OS-assigned port so there are no port conflicts.
896 server = http.server.HTTPServer(("127.0.0.1", 0), _Handler)
897 port = server.server_address[1]
898 server_thread = threading.Thread(target=server.handle_request, daemon=True)
899 server_thread.start()
900
901 # Run a second request handler for the success call.
902 # HTTPServer.handle_request() handles exactly one connection;
903 # start a second thread for the retry.
904 def _serve_one() -> None:
905 server.handle_request()
906
907 second_thread = threading.Thread(target=_serve_one, daemon=True)
908 second_thread.start()
909
910 url = f"http://127.0.0.1:{port}"
911 with unittest.mock.patch("muse.core.transport._sleep", side_effect=lambda s: None):
912 result = HttpTransport().fetch_mpack(
913 url, None,
914 want=["sha256:" + "a" * 64],
915 have=[],
916 )
917
918 server_thread.join(timeout=5)
919 second_thread.join(timeout=5)
920 server.server_close()
921
922 # fetch_mpack returned a result (did not raise) — retry succeeded
923 assert result is not None
924 # Server was called exactly twice (once 503, once success)
925 assert call_count[0] == 2, f"server call count: {call_count[0]}"
File History 2 commits
sha256:ac29b8ba9021514a03ab2782d92bf67671f0efa5b3b70d46f7598c5d4e923378 docs: record muse#74 Phase 4 live verification (SCR_13, SCR_14) Sonnet 5 13 days ago
sha256:2562dffa0a0822ac1bdea854f9b267843c6ce95b497a9dc5c55837c80a3ebd0a feat: domain_command_registry — Phase 1 of muse#74 (SCR_01-03) Sonnet 4.6 patch 13 days ago