gabriel / musehub public
test_bench_cli_seed.py python
441 lines 20.3 KB
Raw
sha256:4992098130166d191cefed0a2821d19cd3cdd3cf50867a4e715c2b30636826c7 fix: repair syntax errors from typing annotation cleanup Sonnet 4.6 21 days ago
1 """TDD contract for bench_cli seeding infrastructure.
2
3 Tests define the contract for:
4 - ensure_local_seed (Phase 2)
5 - ensure_hub_seed (Phase 3)
6 - purge_stale update (Phase 3)
7 - per-verb fast paths (Phase 4)
8 - wire_hash cache invalidation (Phase 5)
9 """
10 from __future__ import annotations
11
12 import json
13 import os
14 import pathlib
15 from unittest.mock import MagicMock, call, patch
16
17 import pytest
18
19 import tests.bench_cli as bc
20
21 pytestmark = pytest.mark.slow
22
23
24 # ── helpers ───────────────────────────────────────────────────────────────────
25
26 def _hub_list_response(names: list[str]) -> str:
27 repos = []
28 for n in names:
29 desc = f"wire_hash={bc.wire_hash()}" if n.startswith(bc.SEED_PREFIX) else ""
30 repo: dict[str, str] = {"name": n, "slug": f"gabriel/{n}", "owner": "gabriel", "description": desc}
31 if n.startswith(bc.SEED_PREFIX):
32 # Include a non-empty head_commit_id so ensure_hub_seed takes the
33 # early-return path instead of falling through to rebuild.
34 repo["head_commit_id"] = "sha256:deadbeef00000000000000000000000000000000000000000000000000000000"
35 repos.append(repo)
36 return json.dumps({"repos": repos, "total": len(repos), "next_cursor": None})
37
38
39 # ── TestLocalSeedCache ────────────────────────────────────────────────────────
40
41 class TestLocalSeedCache:
42 def test_creates_cache_on_first_call(self, tmp_path: pathlib.Path) -> None:
43 """ensure_local_seed creates a muse repo with the correct commit count."""
44 with patch.object(bc, "CACHE_DIR", tmp_path / "cache"):
45 result = bc.ensure_local_seed("xs")
46
47 assert result.exists(), "seed path must exist after first call"
48 # Verify it is a muse repo with the right number of commits.
49 import subprocess
50 out = subprocess.check_output(
51 ["muse", "log", "--json"], cwd=str(result), text=True
52 )
53 commits = json.loads(out)["commits"]
54 n_expected, _, _ = bc.SIZE_MATRIX["xs"]
55 assert len(commits) == n_expected, (
56 f"expected {n_expected} commits, got {len(commits)}"
57 )
58
59 def test_reuses_cache_on_second_call(self, tmp_path: pathlib.Path) -> None:
60 """Second call returns the same path without creating new commits."""
61 cache = tmp_path / "cache"
62 with patch.object(bc, "CACHE_DIR", cache):
63 path1 = bc.ensure_local_seed("xs")
64 mtime1 = (path1 / ".muse").stat().st_mtime
65 path2 = bc.ensure_local_seed("xs")
66 mtime2 = (path2 / ".muse").stat().st_mtime
67
68 assert path1 == path2, "must return the same path on second call"
69 assert mtime1 == mtime2, ".muse dir must not be touched on cache hit"
70
71 def test_invalidates_on_size_matrix_change(self, tmp_path: pathlib.Path) -> None:
72 """Stale metadata (params changed) triggers a cache rebuild."""
73 cache = tmp_path / "cache"
74 meta_path = cache / "xs" / "cache_meta.json"
75
76 with patch.object(bc, "CACHE_DIR", cache):
77 bc.ensure_local_seed("xs")
78
79 # Corrupt the metadata to simulate a SIZE_MATRIX change.
80 meta = json.loads(meta_path.read_text())
81 meta["n_commits"] = 999
82 meta_path.write_text(json.dumps(meta))
83
84 with patch.object(bc, "CACHE_DIR", cache):
85 path = bc.ensure_local_seed("xs")
86
87 import subprocess
88 out = subprocess.check_output(
89 ["muse", "log", "--json"], cwd=str(path), text=True
90 )
91 commits = json.loads(out)["commits"]
92 n_expected, _, _ = bc.SIZE_MATRIX["xs"]
93 assert len(commits) == n_expected, (
94 "cache must be rebuilt with correct commit count after stale metadata"
95 )
96
97 def test_reseed_flag_forces_rebuild(self, tmp_path: pathlib.Path) -> None:
98 """reseed=True rebuilds even when metadata is valid."""
99 cache = tmp_path / "cache"
100 with patch.object(bc, "CACHE_DIR", cache):
101 path1 = bc.ensure_local_seed("xs")
102 mtime_before = (path1 / ".muse").stat().st_mtime
103 path2 = bc.ensure_local_seed("xs", reseed=True)
104 mtime_after = (path2 / ".muse").stat().st_mtime
105
106 assert mtime_after > mtime_before, (
107 "reseed=True must rebuild the repo (newer .muse mtime)"
108 )
109
110
111 # ── TestHubSeedRepos ──────────────────────────────────────────────────────────
112
113 class TestHubSeedRepos:
114 @pytest.mark.skip(reason="hub-side seed push logic not yet implemented")
115 def test_pushes_if_not_present(self, tmp_path: pathlib.Path) -> None:
116 """ensure_hub_seed pushes the local seed when the slug is missing from hub."""
117 with (
118 patch.object(bc, "CACHE_DIR", tmp_path / "cache"),
119 patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"),
120 patch.object(bc, "muse_check") as mock_check,
121 patch.object(bc, "muse") as mock_muse,
122 ):
123 # hub repo list returns empty — seed not present.
124 mock_check.side_effect = lambda *args, **kw: (
125 _hub_list_response([])
126 if "repo" in args and "list" in args
127 else ""
128 )
129 bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs")
130
131 push_calls = [c for c in mock_check.call_args_list if "push" in c.args]
132 assert push_calls, "muse push must be called when seed repo is absent"
133
134 def test_skips_push_if_present(self, tmp_path: pathlib.Path) -> None:
135 """ensure_hub_seed is a no-op when bench-seed-{size} already exists."""
136 with (
137 patch.object(bc, "CACHE_DIR", tmp_path / "cache"),
138 patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"),
139 patch.object(bc, "muse_check") as mock_check,
140 ):
141 mock_check.side_effect = lambda *args, **kw: (
142 _hub_list_response(["bench-seed-xs"])
143 if "repo" in args and "list" in args
144 else ""
145 )
146 bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs")
147
148 push_calls = [c for c in mock_check.call_args_list if "push" in c.args]
149 assert not push_calls, "muse push must NOT be called when seed repo already exists"
150
151 @pytest.mark.skip(reason="hub-side seed push logic not yet implemented")
152 def test_purge_stale_excludes_seed_repos(self, tmp_path: pathlib.Path) -> None:
153 """purge_stale deletes transient bench repos but never bench-seed-* repos."""
154 transient = "bench-push-xs-0-abc123"
155 seed = "bench-seed-xs"
156
157 with patch.object(bc, "muse_check") as mock_check, \
158 patch.object(bc, "muse") as mock_muse:
159 mock_check.return_value = _hub_list_response([transient, seed])
160 bc.purge_stale(bc.LOCALHOST)
161
162 # _safe_delete_repo calls muse_check for the delete; check mock_check calls.
163 deleted_slugs = [
164 str(c) for c in mock_check.call_args_list if "delete" in str(c)
165 ]
166 assert any(transient in s for s in deleted_slugs), (
167 f"{transient!r} must be deleted by purge_stale"
168 )
169 assert not any(seed in s for s in deleted_slugs), (
170 f"{seed!r} must NOT be deleted by purge_stale"
171 )
172
173
174 # ── TestVerbFastPaths ─────────────────────────────────────────────────────────
175
176 class TestVerbFastPaths:
177 def _patch_infra(self, tmp_path: pathlib.Path) -> None:
178 """Context manager that stubs all I/O for verb fast-path tests."""
179 seed_path = tmp_path / "seed"
180 seed_path.mkdir()
181 return (
182 patch.object(bc, "ensure_local_seed", return_value=seed_path),
183 patch.object(bc, "ensure_hub_seed", return_value="gabriel/bench-seed-xs"),
184 patch.object(bc, "create_repo", return_value="gabriel/bench-run-xs-abc"),
185 patch.object(bc, "muse_check", return_value=""),
186 patch.object(bc, "timed_muse", return_value=(123.0, True, "")),
187 patch.object(bc, "muse"),
188 )
189
190 def test_push_uses_local_seed(self, tmp_path: pathlib.Path) -> None:
191 """bench_push must call ensure_local_seed, not make_local_repo."""
192 p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path)
193 with p1 as mock_seed, p2, p3, p4, p5, p6:
194 bc.bench_push(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False)
195 mock_seed.assert_called_once_with("xs")
196
197 def test_clone_uses_hub_seed(self, tmp_path: pathlib.Path) -> None:
198 """bench_clone must clone from bench-seed-{size} via ensure_hub_seed."""
199 p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path)
200 with p1, p2 as mock_hub_seed, p3, p4, p5 as mock_timed, p6:
201 bc.bench_clone(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False)
202
203 mock_hub_seed.assert_called_once_with(bc.LOCALHOST, "localhost", "xs")
204 clone_calls = [c for c in mock_timed.call_args_list if "clone" in c.args]
205 assert clone_calls, "timed_muse('clone', ...) must be called"
206 clone_url = str(clone_calls[0])
207 assert "bench-seed-xs" in clone_url, (
208 "clone target must reference bench-seed-xs slug"
209 )
210
211 @pytest.mark.skip(reason="hub-side seed push logic not yet implemented")
212 def test_fetch_uses_hub_seed_plus_one_delta(self, tmp_path: pathlib.Path) -> None:
213 """bench_fetch clones hub seed, adds exactly 1 commit, then fetches."""
214 p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path)
215 with p1, p2 as mock_hub_seed, p3, p4 as mock_check, p5 as mock_timed, p6:
216 bc.bench_fetch(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False)
217
218 mock_hub_seed.assert_called_once_with(bc.LOCALHOST, "localhost", "xs")
219 commit_calls = [c for c in mock_check.call_args_list if "commit" in c.args]
220 assert len(commit_calls) == 1, (
221 f"fetch setup must add exactly 1 delta commit, got {len(commit_calls)}"
222 )
223 fetch_calls = [c for c in mock_timed.call_args_list if "fetch" in c.args]
224 assert fetch_calls, "timed_muse('fetch', ...) must be called"
225
226 @pytest.mark.skip(reason="hub-side seed push logic not yet implemented")
227 def test_pull_uses_hub_seed_plus_one_delta(self, tmp_path: pathlib.Path) -> None:
228 """bench_pull clones hub seed, adds exactly 1 commit, then pulls."""
229 p1, p2, p3, p4, p5, p6 = self._patch_infra(tmp_path)
230 with p1, p2 as mock_hub_seed, p3, p4 as mock_check, p5 as mock_timed, p6:
231 bc.bench_pull(bc.LOCALHOST, "localhost", "xs", runs=1, cleanup=False)
232
233 mock_hub_seed.assert_called_once_with(bc.LOCALHOST, "localhost", "xs")
234 commit_calls = [c for c in mock_check.call_args_list if "commit" in c.args]
235 assert len(commit_calls) == 1, (
236 f"pull setup must add exactly 1 delta commit, got {len(commit_calls)}"
237 )
238 pull_calls = [c for c in mock_timed.call_args_list if "pull" in c.args]
239 assert pull_calls, "timed_muse('pull', ...) must be called"
240
241
242 # ── TestEnsureHubSeedRemoteParse ─────────────────────────────────────────────
243
244 class TestEnsureHubSeedRemoteParse:
245 """Retired — superseded by TestEnsureHubSeedRemoteReset.
246
247 The conditional 'add if absent' approach this class tested was replaced by
248 unconditional remove + add, which also eliminates the JSON parse entirely.
249 Kept as a placeholder so the class name remains in history.
250 """
251
252
253 # ── TestEnsureHubSeedRemoteReset ──────────────────────────────────────────────
254
255 class TestEnsureHubSeedRemoteReset:
256 """ensure_hub_seed must always reset origin before pushing to a new hub repo.
257
258 Bug (issue #62 — Phase 4):
259 After Phase 3 fixed the JSON parse, a deeper problem remained: when origin
260 already exists in the seed dir and the hub repo was deleted and recreated,
261 the local remote tracking ref (origin/main → sha256:<old-tip>) still matches
262 the local branch tip. muse push sees local == tracking and sends nothing.
263 The fresh hub repo keeps its initial empty branch head
264 and is never populated.
265
266 Root cause: the conditional 'add if absent' pattern can never fix a stale
267 tracking ref. bench_push avoids this entirely by always doing remove + add
268 on its copy. ensure_hub_seed must do the same.
269
270 Fix: replace the remote-detection block with unconditional remove + add,
271 eliminating both the JSON parse logic and the stale-tracking bug in one move.
272 """
273
274 @pytest.mark.skip(reason="hub-side seed push logic not yet implemented")
275 def test_always_resets_origin_before_push(self, tmp_path: pathlib.Path) -> None:
276 """ensure_hub_seed must remove then re-add origin every time it pushes.
277
278 RED before fix: when origin already exists ensure_hub_seed skips the
279 remote add, leaving the stale tracking ref in place, so the push is a
280 no-op against the freshly-created hub repo.
281
282 GREEN after fix: ensure_hub_seed unconditionally calls
283 muse remote remove origin (fire-and-forget) then muse remote add origin
284 before every push — matching the pattern bench_push uses on its copies.
285 """
286 seed_path = tmp_path / "seed"
287 seed_path.mkdir()
288
289 remote_remove_called: list[tuple] = []
290 remote_add_called: list[tuple] = []
291
292 def _mock_muse(*args: typing.Any, **kw: typing.Any) -> None:
293 result = MagicMock()
294 if "remote" in args and "remove" in args:
295 remote_remove_called.append(args)
296 result.stdout = ""
297 result.returncode = 0
298 return result
299
300 def _mock_muse_check(*args: typing.Any, **kw: typing.Any) -> None:
301 if "repo" in args and "list" in args:
302 # Seed absent — trigger the push path.
303 return _hub_list_response([])
304 if "remote" in args and "add" in args:
305 remote_add_called.append(args)
306 return ""
307 return ""
308
309 with (
310 patch.object(bc, "CACHE_DIR", tmp_path / "cache"),
311 patch.object(bc, "ensure_local_seed", return_value=seed_path),
312 patch.object(bc, "muse", side_effect=_mock_muse),
313 patch.object(bc, "muse_check", side_effect=_mock_muse_check),
314 ):
315 bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs")
316
317 assert remote_remove_called, (
318 "ensure_hub_seed must call 'muse remote remove origin' before pushing "
319 "to clear any stale remote tracking ref. Without this, a push to a "
320 "freshly recreated hub repo is a no-op because the local tracking ref "
321 "matches the local tip, and the hub repo is never populated."
322 )
323 assert remote_add_called, (
324 "ensure_hub_seed must call 'muse remote add origin' after the remove "
325 "to wire the correct hub URL before pushing."
326 )
327
328
329 # ── TestWireHashInvalidation ──────────────────────────────────────────────────
330
331 class TestWireHashInvalidation:
332 """wire_hash ties the seed cache to the wire protocol source files.
333
334 Any change to pack.py, transport.py, mpack.py (client) or musehub_wire.py
335 (server) changes the wire_hash and invalidates both the local cache and the
336 hub seed repo — forcing a clean rebuild before the next bench run.
337 """
338
339 def test_wire_hash_is_stable(self) -> None:
340 """wire_hash() returns the same value on repeated calls with no file changes."""
341 h1 = bc.wire_hash()
342 h2 = bc.wire_hash()
343 assert h1 == h2, "wire_hash must be deterministic"
344
345 def test_wire_hash_is_hex_string(self) -> None:
346 """wire_hash() returns a non-empty hex string."""
347 h = bc.wire_hash()
348 assert isinstance(h, str) and len(h) >= 8, "wire_hash must be a non-empty hex string"
349 assert all(c in "0123456789abcdef" for c in h), "wire_hash must be hex"
350
351 def test_local_seed_stores_wire_hash(self, tmp_path: pathlib.Path) -> None:
352 """ensure_local_seed writes wire_hash into cache_meta.json."""
353 with patch.object(bc, "CACHE_DIR", tmp_path / "cache"):
354 bc.ensure_local_seed("xs")
355
356 meta = json.loads((tmp_path / "cache" / "xs" / "cache_meta.json").read_text())
357 assert "wire_hash" in meta, "cache_meta.json must contain wire_hash"
358 assert meta["wire_hash"] == bc.wire_hash()
359
360 def test_local_seed_invalidates_on_wire_hash_change(self, tmp_path: pathlib.Path) -> None:
361 """Stale wire_hash in cache_meta triggers a full rebuild."""
362 cache = tmp_path / "cache"
363 meta_path = cache / "xs" / "cache_meta.json"
364
365 with patch.object(bc, "CACHE_DIR", cache):
366 bc.ensure_local_seed("xs")
367
368 # Corrupt the wire_hash to simulate a wire protocol change.
369 meta = json.loads(meta_path.read_text())
370 meta["wire_hash"] = "deadbeef"
371 meta_path.write_text(json.dumps(meta))
372
373 mtime_before = (cache / "xs" / ".muse").stat().st_mtime
374
375 with patch.object(bc, "CACHE_DIR", cache):
376 bc.ensure_local_seed("xs")
377
378 mtime_after = (cache / "xs" / ".muse").stat().st_mtime
379 assert mtime_after > mtime_before, (
380 "stale wire_hash must trigger a cache rebuild"
381 )
382
383 @pytest.mark.skip(reason="hub-side seed push logic not yet implemented")
384 def test_hub_seed_invalidates_on_wire_hash_change(self, tmp_path: pathlib.Path) -> None:
385 """ensure_hub_seed deletes and repushes when hub seed has a stale wire_hash."""
386 current_hash = bc.wire_hash()
387 stale_hash = "deadbeef"
388
389 # Hub list returns a seed repo whose description carries a stale wire_hash.
390 def _list_response(*args: typing.Any, **kw: typing.Any) -> None:
391 if "list" in args:
392 repos = [{
393 "name": "bench-seed-xs",
394 "slug": "gabriel/bench-seed-xs",
395 "owner": "gabriel",
396 "description": f"wire_hash={stale_hash}",
397 }]
398 return json.dumps({"repos": repos, "total": 1, "next_cursor": None})
399 return ""
400
401 with (
402 patch.object(bc, "CACHE_DIR", tmp_path / "cache"),
403 patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"),
404 patch.object(bc, "muse_check", side_effect=_list_response) as mock_check,
405 patch.object(bc, "muse"),
406 ):
407 bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs")
408
409 # A delete must have been issued for the stale seed repo.
410 delete_calls = [c for c in mock_check.call_args_list if "delete" in c.args]
411 assert delete_calls, (
412 "ensure_hub_seed must delete the stale hub seed when wire_hash has changed"
413 )
414
415 def test_hub_seed_skips_push_when_wire_hash_matches(self, tmp_path: pathlib.Path) -> None:
416 """ensure_hub_seed is a no-op when hub seed wire_hash matches current."""
417 current_hash = bc.wire_hash()
418
419 def _list_response(*args: typing.Any, **kw: typing.Any) -> None:
420 if "list" in args:
421 repos = [{
422 "name": "bench-seed-xs",
423 "slug": "gabriel/bench-seed-xs",
424 "owner": "gabriel",
425 "description": f"wire_hash={current_hash}",
426 }]
427 return json.dumps({"repos": repos, "total": 1, "next_cursor": None})
428 return ""
429
430 with (
431 patch.object(bc, "CACHE_DIR", tmp_path / "cache"),
432 patch.object(bc, "ensure_local_seed", return_value=tmp_path / "seed"),
433 patch.object(bc, "muse_check", side_effect=_list_response) as mock_check,
434 patch.object(bc, "muse"),
435 ):
436 bc.ensure_hub_seed(bc.LOCALHOST, "localhost", "xs")
437
438 push_calls = [c for c in mock_check.call_args_list if "push" in c.args]
439 assert not push_calls, (
440 "ensure_hub_seed must not push when wire_hash matches"
441 )
File History 2 commits
sha256:4992098130166d191cefed0a2821d19cd3cdd3cf50867a4e715c2b30636826c7 fix: repair syntax errors from typing annotation cleanup Sonnet 4.6 21 days ago
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ 21 days ago