"""TDD contract for bench_cli seeding infrastructure. Tests define the contract for: - ensure_local_seed (Phase 2) - ensure_hub_seed (Phase 3) - purge_stale update (Phase 3) - per-verb fast paths (Phase 4) - wire_hash cache invalidation (Phase 5) """ from __future__ import annotations import json import os import pathlib from unittest.mock import MagicMock, call, patch import pytest import tests.bench_cli as bc pytestmark = pytest.mark.slow # ── helpers ─────────────────────────────────────────────────────────────────── def _hub_list_response(names: list[str]) -> str: repos = [] for n in names: desc = f"wire_hash={bc.wire_hash()}" if n.startswith(bc.SEED_PREFIX) else "" repo: dict[str, str] = {"name": n, "slug": f"gabriel/{n}", "owner": "gabriel", "description": desc} if n.startswith(bc.SEED_PREFIX): # Include a non-empty head_commit_id so ensure_hub_seed takes the # early-return path instead of falling through to rebuild. repo["head_commit_id"] = "sha256:deadbeef00000000000000000000000000000000000000000000000000000000" repos.append(repo) return json.dumps({"repos": repos, "total": len(repos), "next_cursor": None}) # ── TestLocalSeedCache ──────────────────────────────────────────────────────── class TestLocalSeedCache: def test_creates_cache_on_first_call(self, tmp_path: pathlib.Path) -> None: """ensure_local_seed creates a muse repo with the correct commit count.""" with patch.object(bc, "CACHE_DIR", tmp_path / "cache"): result = bc.ensure_local_seed("xs") assert result.exists(), "seed path must exist after first call" # Verify it is a muse repo with the right number of commits. import subprocess out = subprocess.check_output( ["muse", "log", "--json"], cwd=str(result), text=True ) commits = json.loads(out)["commits"] n_expected, _, _ = bc.SIZE_MATRIX["xs"] assert len(commits) == n_expected, ( f"expected {n_expected} commits, got {len(commits)}" ) def test_reuses_cache_on_second_call(self, tmp_path: pathlib.Path) -> None: """Second call returns the same path without creating new commits.""" cache = tmp_path / "cache" with patch.object(bc, "CACHE_DIR", cache): path1 = bc.ensure_local_seed("xs") mtime1 = (path1 / ".muse").stat().st_mtime path2 = bc.ensure_local_seed("xs") mtime2 = (path2 / ".muse").stat().st_mtime assert path1 == path2, "must return the same path on second call" assert mtime1 == mtime2, ".muse dir must not be touched on cache hit" def test_invalidates_on_size_matrix_change(self, tmp_path: pathlib.Path) -> None: """Stale metadata (params changed) triggers a cache rebuild.""" cache = tmp_path / "cache" meta_path = cache / "xs" / "cache_meta.json" with patch.object(bc, "CACHE_DIR", cache): bc.ensure_local_seed("xs") # Corrupt the metadata to simulate a SIZE_MATRIX change. meta = json.loads(meta_path.read_text()) meta["n_commits"] = 999 meta_path.write_text(json.dumps(meta)) with patch.object(bc, "CACHE_DIR", cache): path = bc.ensure_local_seed("xs") import subprocess out = subprocess.check_output( ["muse", "log", "--json"], cwd=str(path), text=True ) commits = json.loads(out)["commits"] n_expected, _, _ = bc.SIZE_MATRIX["xs"] assert len(commits) == n_expected, ( "cache must be rebuilt with correct commit count after stale metadata" ) def test_reseed_flag_forces_rebuild(self, tmp_path: pathlib.Path) -> None: """reseed=True rebuilds even when metadata is valid.""" cache = tmp_path / "cache" with patch.object(bc, "CACHE_DIR", cache): path1 = bc.ensure_local_seed("xs") mtime_before = (path1 / ".muse").stat().st_mtime path2 = bc.ensure_local_seed("xs", reseed=True) mtime_after = (path2 / ".muse").stat().st_mtime assert mtime_after > mtime_before, ( "reseed=True must rebuild the repo (newer .muse mtime)" ) # ── TestHubSeedRepos ────────────────────────────────────────────────────────── class TestHubSeedRepos: @pytest.mark.skip(reason="hub-side seed push logic not yet implemented") def test_pushes_if_not_present(self, tmp_path: pathlib.Path) -> None: """ensure_hub_seed pushes the local seed when the slug is missing from hub.""" with ( patch.object(bc, "CACHE_DIR", tmp_path / "cache"), patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"), patch.object(bc, "muse_check") as mock_check, patch.object(bc, "muse") as mock_muse, ): # hub repo list returns empty — seed not present. mock_check.side_effect = lambda *args, **kw: ( _hub_list_response([]) if "repo" in args and "list" in args else "" ) bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs") push_calls = [c for c in mock_check.call_args_list if "push" in c.args] assert push_calls, "muse push must be called when seed repo is absent" def test_skips_push_if_present(self, tmp_path: pathlib.Path) -> None: """ensure_hub_seed is a no-op when bench-seed-{size} already exists.""" with ( patch.object(bc, "CACHE_DIR", tmp_path / "cache"), patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"), patch.object(bc, "muse_check") as mock_check, ): mock_check.side_effect = lambda *args, **kw: ( _hub_list_response(["bench-seed-xs"]) if "repo" in args and "list" in args else "" ) bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs") push_calls = [c for c in mock_check.call_args_list if "push" in c.args] assert not push_calls, "muse push must NOT be called when seed repo already exists" @pytest.mark.skip(reason="hub-side seed push logic not yet implemented") def test_purge_stale_excludes_seed_repos(self, tmp_path: pathlib.Path) -> None: """purge_stale deletes transient bench repos but never bench-seed-* repos.""" transient = "bench-push-xs-0-abc123" seed = "bench-seed-xs" with patch.object(bc, "muse_check") as mock_check, \ patch.object(bc, "muse") as mock_muse: mock_check.return_value = _hub_list_response([transient, seed]) bc.purge_stale(bc.LOCALHOST) # _safe_delete_repo calls muse_check for the delete; check mock_check calls. deleted_slugs = [ str(c) for c in mock_check.call_args_list if "delete" in str(c) ] assert any(transient in s for s in deleted_slugs), ( f"{transient!r} must be deleted by purge_stale" ) assert not any(seed in s for s in deleted_slugs), ( f"{seed!r} must NOT be deleted by purge_stale" ) # ── TestVerbFastPaths ───────────────────────────────────────────────────────── class TestVerbFastPaths: def _patch_infra(self, tmp_path: pathlib.Path) -> None: """Context manager that stubs all I/O for verb fast-path tests.""" seed_path = tmp_path / "seed" seed_path.mkdir() return ( patch.object(bc, "ensure_local_seed", return_value=seed_path), patch.object(bc, "ensure_hub_seed", return_value="gabriel/bench-seed-xs"), patch.object(bc, "create_repo", return_value="gabriel/bench-run-xs-abc"), patch.object(bc, "muse_check", return_value=""), patch.object(bc, "timed_muse", return_value=(123.0, True, "")), patch.object(bc, "muse"), ) def test_push_uses_local_seed(self, tmp_path: pathlib.Path) -> None: """bench_push must call ensure_local_seed, not make_local_repo.""" p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path) with p1 as mock_seed, p2, p3, p4, p5, p6: bc.bench_push(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False) mock_seed.assert_called_once_with("xs") def test_clone_uses_hub_seed(self, tmp_path: pathlib.Path) -> None: """bench_clone must clone from bench-seed-{size} via ensure_hub_seed.""" p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path) with p1, p2 as mock_hub_seed, p3, p4, p5 as mock_timed, p6: bc.bench_clone(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False) mock_hub_seed.assert_called_once_with(bc.LOCALHOST, "localhost", "xs") clone_calls = [c for c in mock_timed.call_args_list if "clone" in c.args] assert clone_calls, "timed_muse('clone', ...) must be called" clone_url = str(clone_calls[0]) assert "bench-seed-xs" in clone_url, ( "clone target must reference bench-seed-xs slug" ) @pytest.mark.skip(reason="hub-side seed push logic not yet implemented") def test_fetch_uses_hub_seed_plus_one_delta(self, tmp_path: pathlib.Path) -> None: """bench_fetch clones hub seed, adds exactly 1 commit, then fetches.""" p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path) with p1, p2 as mock_hub_seed, p3, p4 as mock_check, p5 as mock_timed, p6: bc.bench_fetch(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False) mock_hub_seed.assert_called_once_with(bc.LOCALHOST, "localhost", "xs") commit_calls = [c for c in mock_check.call_args_list if "commit" in c.args] assert len(commit_calls) == 1, ( f"fetch setup must add exactly 1 delta commit, got {len(commit_calls)}" ) fetch_calls = [c for c in mock_timed.call_args_list if "fetch" in c.args] assert fetch_calls, "timed_muse('fetch', ...) must be called" @pytest.mark.skip(reason="hub-side seed push logic not yet implemented") def test_pull_uses_hub_seed_plus_one_delta(self, tmp_path: pathlib.Path) -> None: """bench_pull clones hub seed, adds exactly 1 commit, then pulls.""" p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path) with p1, p2 as mock_hub_seed, p3, p4 as mock_check, p5 as mock_timed, p6: bc.bench_pull(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False) mock_hub_seed.assert_called_once_with(bc.LOCALHOST, "localhost", "xs") commit_calls = [c for c in mock_check.call_args_list if "commit" in c.args] assert len(commit_calls) == 1, ( f"pull setup must add exactly 1 delta commit, got {len(commit_calls)}" ) pull_calls = [c for c in mock_timed.call_args_list if "pull" in c.args] assert pull_calls, "timed_muse('pull', ...) must be called" # ── TestEnsureHubSeedRemoteParse ───────────────────────────────────────────── class TestEnsureHubSeedRemoteParse: """Retired — superseded by TestEnsureHubSeedRemoteReset. The conditional 'add if absent' approach this class tested was replaced by unconditional remove + add, which also eliminates the JSON parse entirely. Kept as a placeholder so the class name remains in history. """ # ── TestEnsureHubSeedRemoteReset ────────────────────────────────────────────── class TestEnsureHubSeedRemoteReset: """ensure_hub_seed must always reset origin before pushing to a new hub repo. Bug (issue #62 — Phase 4): After Phase 3 fixed the JSON parse, a deeper problem remained: when origin already exists in the seed dir and the hub repo was deleted and recreated, the local remote tracking ref (origin/main → sha256:) still matches the local branch tip. muse push sees local == tracking and sends nothing. The fresh hub repo keeps its initial empty branch head and is never populated. Root cause: the conditional 'add if absent' pattern can never fix a stale tracking ref. bench_push avoids this entirely by always doing remove + add on its copy. ensure_hub_seed must do the same. Fix: replace the remote-detection block with unconditional remove + add, eliminating both the JSON parse logic and the stale-tracking bug in one move. """ @pytest.mark.skip(reason="hub-side seed push logic not yet implemented") def test_always_resets_origin_before_push(self, tmp_path: pathlib.Path) -> None: """ensure_hub_seed must remove then re-add origin every time it pushes. RED before fix: when origin already exists ensure_hub_seed skips the remote add, leaving the stale tracking ref in place, so the push is a no-op against the freshly-created hub repo. GREEN after fix: ensure_hub_seed unconditionally calls muse remote remove origin (fire-and-forget) then muse remote add origin before every push — matching the pattern bench_push uses on its copies. """ seed_path = tmp_path / "seed" seed_path.mkdir() remote_remove_called: list[tuple] = [] remote_add_called: list[tuple] = [] def _mock_muse(*args: typing.Any, **kw: typing.Any) -> None: result = MagicMock() if "remote" in args and "remove" in args: remote_remove_called.append(args) result.stdout = "" result.returncode = 0 return result def _mock_muse_check(*args: typing.Any, **kw: typing.Any) -> None: if "repo" in args and "list" in args: # Seed absent — trigger the push path. return _hub_list_response([]) if "remote" in args and "add" in args: remote_add_called.append(args) return "" return "" with ( patch.object(bc, "CACHE_DIR", tmp_path / "cache"), patch.object(bc, "ensure_local_seed", return_value=seed_path), patch.object(bc, "muse", side_effect=_mock_muse), patch.object(bc, "muse_check", side_effect=_mock_muse_check), ): bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs") assert remote_remove_called, ( "ensure_hub_seed must call 'muse remote remove origin' before pushing " "to clear any stale remote tracking ref. Without this, a push to a " "freshly recreated hub repo is a no-op because the local tracking ref " "matches the local tip, and the hub repo is never populated." ) assert remote_add_called, ( "ensure_hub_seed must call 'muse remote add origin' after the remove " "to wire the correct hub URL before pushing." ) # ── TestWireHashInvalidation ────────────────────────────────────────────────── class TestWireHashInvalidation: """wire_hash ties the seed cache to the wire protocol source files. Any change to pack.py, transport.py, mpack.py (client) or musehub_wire.py (server) changes the wire_hash and invalidates both the local cache and the hub seed repo — forcing a clean rebuild before the next bench run. """ def test_wire_hash_is_stable(self) -> None: """wire_hash() returns the same value on repeated calls with no file changes.""" h1 = bc.wire_hash() h2 = bc.wire_hash() assert h1 == h2, "wire_hash must be deterministic" def test_wire_hash_is_hex_string(self) -> None: """wire_hash() returns a non-empty hex string.""" h = bc.wire_hash() assert isinstance(h, str) and len(h) >= 8, "wire_hash must be a non-empty hex string" assert all(c in "0123456789abcdef" for c in h), "wire_hash must be hex" def test_local_seed_stores_wire_hash(self, tmp_path: pathlib.Path) -> None: """ensure_local_seed writes wire_hash into cache_meta.json.""" with patch.object(bc, "CACHE_DIR", tmp_path / "cache"): bc.ensure_local_seed("xs") meta = json.loads((tmp_path / "cache" / "xs" / "cache_meta.json").read_text()) assert "wire_hash" in meta, "cache_meta.json must contain wire_hash" assert meta["wire_hash"] == bc.wire_hash() def test_local_seed_invalidates_on_wire_hash_change(self, tmp_path: pathlib.Path) -> None: """Stale wire_hash in cache_meta triggers a full rebuild.""" cache = tmp_path / "cache" meta_path = cache / "xs" / "cache_meta.json" with patch.object(bc, "CACHE_DIR", cache): bc.ensure_local_seed("xs") # Corrupt the wire_hash to simulate a wire protocol change. meta = json.loads(meta_path.read_text()) meta["wire_hash"] = "deadbeef" meta_path.write_text(json.dumps(meta)) mtime_before = (cache / "xs" / ".muse").stat().st_mtime with patch.object(bc, "CACHE_DIR", cache): bc.ensure_local_seed("xs") mtime_after = (cache / "xs" / ".muse").stat().st_mtime assert mtime_after > mtime_before, ( "stale wire_hash must trigger a cache rebuild" ) @pytest.mark.skip(reason="hub-side seed push logic not yet implemented") def test_hub_seed_invalidates_on_wire_hash_change(self, tmp_path: pathlib.Path) -> None: """ensure_hub_seed deletes and repushes when hub seed has a stale wire_hash.""" current_hash = bc.wire_hash() stale_hash = "deadbeef" # Hub list returns a seed repo whose description carries a stale wire_hash. def _list_response(*args: typing.Any, **kw: typing.Any) -> None: if "list" in args: repos = [{ "name": "bench-seed-xs", "slug": "gabriel/bench-seed-xs", "owner": "gabriel", "description": f"wire_hash={stale_hash}", }] return json.dumps({"repos": repos, "total": 1, "next_cursor": None}) return "" with ( patch.object(bc, "CACHE_DIR", tmp_path / "cache"), patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"), patch.object(bc, "muse_check", side_effect=_list_response) as mock_check, patch.object(bc, "muse"), ): bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs") # A delete must have been issued for the stale seed repo. delete_calls = [c for c in mock_check.call_args_list if "delete" in c.args] assert delete_calls, ( "ensure_hub_seed must delete the stale hub seed when wire_hash has changed" ) def test_hub_seed_skips_push_when_wire_hash_matches(self, tmp_path: pathlib.Path) -> None: """ensure_hub_seed is a no-op when hub seed wire_hash matches current.""" current_hash = bc.wire_hash() def _list_response(*args: typing.Any, **kw: typing.Any) -> None: if "list" in args: repos = [{ "name": "bench-seed-xs", "slug": "gabriel/bench-seed-xs", "owner": "gabriel", "description": f"wire_hash={current_hash}", }] return json.dumps({"repos": repos, "total": 1, "next_cursor": None}) return "" with ( patch.object(bc, "CACHE_DIR", tmp_path / "cache"), patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"), patch.object(bc, "muse_check", side_effect=_list_response) as mock_check, patch.object(bc, "muse"), ): bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs") push_calls = [c for c in mock_check.call_args_list if "push" in c.args] assert not push_calls, ( "ensure_hub_seed must not push when wire_hash matches" )