gabriel / muse public
test_verify_supercharge.py python
568 lines 21.3 KB
Raw
sha256:d11a87833d5fad6059b7662844bf5448a8911a17cce7a51811f71ad394f248eb bump to v0.2.0rc13 Human patch 6 days ago
1 """Supercharge tests for ``muse verify``.
2
3 Every JSON success response must carry ``duration_ms`` (float, ms) and
4 ``exit_code`` (int). Every JSON error response routes to *stdout*, not stderr,
5 so agent pipelines never receive mixed-mode output.
6
7 Coverage tiers
8 --------------
9 U — duration_ms / exit_code on all code paths (clean, failures, no-objects,
10 branch-scoped, fail-fast)
11 E — JSON error routing: _emit_error() writes to stdout in JSON mode, stderr in
12 text mode; no traceback on any error path
13 S — Schema completeness: all _VerifyJson fields present in every success
14 response; all _VerifyErrorJson fields present in every error response
15 D — Data integrity: exit_code=0 ↔ all_ok=True; exit_code=1 ↔ all_ok=False;
16 duration_ms > 0; duration_ms is float; counters are non-negative
17 IO — OSError during run_verify → exit_code=3 in JSON, stderr in text mode
18 P — Performance: duration_ms < 5 000 ms for a 50-commit chain; monotone
19 (two runs on same repo differ only by noise)
20 Sec — No traceback on any error; no raw Python exception in stdout
21 C — Concurrent readers produce valid JSON (10 threads, same repo)
22 """
23
24 from __future__ import annotations
25 from collections.abc import Mapping
26
27 import datetime
28 import json
29 import pathlib
30 import threading
31 import unittest.mock as mock
32
33 import pytest
34 from tests.cli_test_helper import CliRunner, InvokeResult
35
36 from muse.core.types import blob_id, long_id, short_id
37 from muse.core.object_store import object_path, write_object
38 from muse.core.ids import hash_commit as compute_commit_id, hash_snapshot as compute_snapshot_id
39 from muse.core.commits import (
40 CommitRecord,
41 write_commit,
42 )
43 from muse.core.snapshots import (
44 SnapshotRecord,
45 write_snapshot,
46 )
47 from muse.core.verify import run_verify
48 from muse.core.paths import muse_dir, heads_dir, ref_path
49
50 runner = CliRunner()
51 cli = None # argparse migration — CliRunner ignores this arg
52
53 _REPO_ID = "verify-supercharge-test"
54
55
56 # ---------------------------------------------------------------------------
57 # Helpers
58 # ---------------------------------------------------------------------------
59
60
61
62
63 def _init_repo(path: pathlib.Path) -> pathlib.Path:
64 muse = muse_dir(path)
65 for d in ("commits", "snapshots", "objects", "refs/heads"):
66 (muse / d).mkdir(parents=True, exist_ok=True)
67 (muse / "HEAD").write_text("ref: refs/heads/main", encoding="utf-8")
68 (muse / "repo.json").write_text(
69 json.dumps({"repo_id": _REPO_ID, "domain": "midi"}), encoding="utf-8"
70 )
71 return path
72
73
74 def _env(repo: pathlib.Path) -> Mapping[str, str]:
75 return {"MUSE_REPO_ROOT": str(repo)}
76
77
78 def _make_commit(
79 root: pathlib.Path,
80 parent_id: str | None = None,
81 content: bytes = b"data",
82 branch: str = "main",
83 idx: int = 0,
84 ) -> str:
85 raw = content + str(idx).encode()
86 obj_id = blob_id(raw)
87 write_object(root, obj_id, raw)
88 manifest = {f"file_{idx}.txt": obj_id}
89 snap_id = compute_snapshot_id(manifest)
90 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
91 committed_at = (
92 datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
93 + datetime.timedelta(hours=idx)
94 )
95 parent_ids = [parent_id] if parent_id else []
96 commit_id = compute_commit_id(
97 parent_ids=parent_ids,
98 snapshot_id=snap_id,
99 message=f"commit {idx}",
100 committed_at_iso=committed_at.isoformat(),
101 )
102 write_commit(
103 root,
104 CommitRecord(
105 commit_id=commit_id,
106 branch=branch,
107 snapshot_id=snap_id,
108 message=f"commit {idx}",
109 committed_at=committed_at,
110 parent_commit_id=parent_id,
111 ),
112 )
113 (ref_path(root, branch)).write_text(commit_id, encoding="utf-8")
114 return commit_id
115
116
117 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
118 from muse.cli.app import main as cli_main
119 return runner.invoke(cli_main, ["verify", *args], env=_env(repo))
120
121
122 # ---------------------------------------------------------------------------
123 # U — duration_ms and exit_code on all code paths
124 # ---------------------------------------------------------------------------
125
126
127 class TestElapsedAndExitCode:
128 def test_clean_repo_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
129 repo = _init_repo(tmp_path)
130 _make_commit(repo, idx=0)
131 r = _invoke(repo, "--json")
132 assert r.exit_code == 0
133 d = json.loads(r.output)
134 assert d["exit_code"] == 0
135
136 def test_clean_repo_duration_ms_present(self, tmp_path: pathlib.Path) -> None:
137 repo = _init_repo(tmp_path)
138 _make_commit(repo, idx=0)
139 r = _invoke(repo, "--json")
140 d = json.loads(r.output)
141 assert "duration_ms" in d
142 assert isinstance(d["duration_ms"], float)
143 assert d["duration_ms"] > 0
144
145 def test_failures_exit_code_one(self, tmp_path: pathlib.Path) -> None:
146 repo = _init_repo(tmp_path)
147 # Write a ref pointing at a non-existent commit (bare hex — invalid ref format)
148 (heads_dir(repo) / "main").write_text("b" * 64)
149 r = _invoke(repo, "--json")
150 assert r.exit_code == 1
151 d = json.loads(r.output)
152 assert d["exit_code"] == 1
153 assert d["all_ok"] is False
154
155 def test_failures_duration_ms_present(self, tmp_path: pathlib.Path) -> None:
156 repo = _init_repo(tmp_path)
157 (heads_dir(repo) / "main").write_text("c" * 64)
158 r = _invoke(repo, "--json")
159 d = json.loads(r.output)
160 assert isinstance(d["duration_ms"], float)
161 assert d["duration_ms"] > 0
162
163 def test_no_objects_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
164 repo = _init_repo(tmp_path)
165 _make_commit(repo, idx=1)
166 r = _invoke(repo, "--json", "--no-objects")
167 assert r.exit_code == 0
168 d = json.loads(r.output)
169 assert d["exit_code"] == 0
170 assert d["duration_ms"] > 0
171
172 def test_branch_scoped_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
173 repo = _init_repo(tmp_path)
174 _make_commit(repo, idx=2)
175 r = _invoke(repo, "--json", "--branch", "main")
176 assert r.exit_code == 0
177 d = json.loads(r.output)
178 assert d["exit_code"] == 0
179 assert d["duration_ms"] > 0
180
181 def test_fail_fast_exit_code_one(self, tmp_path: pathlib.Path) -> None:
182 repo = _init_repo(tmp_path)
183 (heads_dir(repo) / "main").write_text("d" * 64)
184 r = _invoke(repo, "--json", "--fail-fast")
185 assert r.exit_code == 1
186 d = json.loads(r.output)
187 assert d["exit_code"] == 1
188 assert d["duration_ms"] > 0
189
190
191 # ---------------------------------------------------------------------------
192 # E — Error routing: JSON mode → stdout; text mode → stderr
193 # ---------------------------------------------------------------------------
194
195
196 class TestErrorRouting:
197 def test_io_error_json_to_stdout(self, tmp_path: pathlib.Path) -> None:
198 """OSError during run_verify → JSON error on stdout in JSON mode."""
199 repo = _init_repo(tmp_path)
200 _make_commit(repo, idx=0)
201 with mock.patch(
202 "muse.cli.commands.verify.run_verify",
203 side_effect=OSError("disk full"),
204 ):
205 r = _invoke(repo, "--json")
206 assert r.stderr.strip() == "", f"Expected empty stderr, got: {r.stderr!r}"
207 d = json.loads(r.output)
208 assert d["error"] == "io_error"
209 assert "disk full" in d["message"]
210 assert d["exit_code"] == 3
211 assert isinstance(d["duration_ms"], float)
212
213 def test_io_error_text_to_stderr(self, tmp_path: pathlib.Path) -> None:
214 """OSError during run_verify → error on stderr in text mode."""
215 repo = _init_repo(tmp_path)
216 _make_commit(repo, idx=0)
217 with mock.patch(
218 "muse.cli.commands.verify.run_verify",
219 side_effect=OSError("disk full"),
220 ):
221 r = _invoke(repo)
222 assert r.exit_code == 3
223 assert r.stderr.strip() != ""
224 assert "disk full" in r.stderr
225
226 def test_io_error_quiet_no_output(self, tmp_path: pathlib.Path) -> None:
227 """OSError in quiet mode → no output at all, exit 3."""
228 repo = _init_repo(tmp_path)
229 _make_commit(repo, idx=0)
230 with mock.patch(
231 "muse.cli.commands.verify.run_verify",
232 side_effect=OSError("disk full"),
233 ):
234 r = _invoke(repo, "--quiet")
235 assert r.exit_code == 3
236 assert r.output.strip() == ""
237 assert r.stderr.strip() == ""
238
239 def test_io_error_json_no_traceback(self, tmp_path: pathlib.Path) -> None:
240 """No Python traceback lands on stdout in JSON mode."""
241 repo = _init_repo(tmp_path)
242 _make_commit(repo, idx=0)
243 with mock.patch(
244 "muse.cli.commands.verify.run_verify",
245 side_effect=OSError("broken pipe"),
246 ):
247 r = _invoke(repo, "--json")
248 assert "Traceback" not in r.output
249 assert "Traceback" not in r.stderr
250
251 def test_io_error_json_schema(self, tmp_path: pathlib.Path) -> None:
252 """JSON error payload has exactly the documented keys."""
253 repo = _init_repo(tmp_path)
254 _make_commit(repo, idx=0)
255 with mock.patch(
256 "muse.cli.commands.verify.run_verify",
257 side_effect=OSError("nfs timeout"),
258 ):
259 r = _invoke(repo, "--json")
260 d = json.loads(r.output)
261 assert set(d) >= {"error", "message", "duration_ms", "exit_code"}
262
263
264 # ---------------------------------------------------------------------------
265 # S — Schema completeness
266 # ---------------------------------------------------------------------------
267
268 _SUCCESS_KEYS = {
269 "repo_id", "refs_checked", "commits_checked", "snapshots_checked",
270 "objects_checked", "signatures_checked", "all_ok", "nothing_checked",
271 "check_objects", "branch", "fail_fast", "duration_ms", "exit_code",
272 "failures",
273 }
274
275 _ERROR_KEYS = {"error", "message", "duration_ms", "exit_code"}
276
277
278 class TestSchemaCompleteness:
279 def test_all_success_keys_present_clean(self, tmp_path: pathlib.Path) -> None:
280 repo = _init_repo(tmp_path)
281 _make_commit(repo, idx=0)
282 d = json.loads(_invoke(repo, "--json").output)
283 assert _SUCCESS_KEYS <= set(d), f"Missing keys: {_SUCCESS_KEYS - set(d)}"
284
285 def test_all_success_keys_present_with_failures(self, tmp_path: pathlib.Path) -> None:
286 repo = _init_repo(tmp_path)
287 (heads_dir(repo) / "main").write_text("e" * 64)
288 d = json.loads(_invoke(repo, "--json").output)
289 assert _SUCCESS_KEYS <= set(d), f"Missing keys: {_SUCCESS_KEYS - set(d)}"
290
291 def test_all_error_keys_present(self, tmp_path: pathlib.Path) -> None:
292 repo = _init_repo(tmp_path)
293 with mock.patch(
294 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
295 ):
296 d = json.loads(_invoke(repo, "--json").output)
297 assert _ERROR_KEYS <= set(d), f"Missing keys: {_ERROR_KEYS - set(d)}"
298
299 def test_failures_list_schema(self, tmp_path: pathlib.Path) -> None:
300 repo = _init_repo(tmp_path)
301 # Missing commit: write a ref pointing to a valid-format but missing commit.
302 cid = long_id("f" * 64)
303 (heads_dir(repo) / "main").write_text(cid)
304 d = json.loads(_invoke(repo, "--json").output)
305 assert len(d["failures"]) >= 1
306 for f in d["failures"]:
307 assert {"kind", "id", "error"} <= set(f)
308
309 def test_failures_kind_is_documented_literal(self, tmp_path: pathlib.Path) -> None:
310 repo = _init_repo(tmp_path)
311 cid = long_id("a" * 64)
312 (heads_dir(repo) / "main").write_text(cid)
313 d = json.loads(_invoke(repo, "--json").output)
314 valid_kinds = {"ref", "commit", "snapshot", "object", "signature", "key_missing"}
315 for f in d["failures"]:
316 assert f["kind"] in valid_kinds, f"Unexpected kind: {f['kind']!r}"
317
318
319 # ---------------------------------------------------------------------------
320 # D — Data integrity
321 # ---------------------------------------------------------------------------
322
323
324 class TestDataIntegrity:
325 def test_exit_code_zero_iff_all_ok_true(self, tmp_path: pathlib.Path) -> None:
326 repo = _init_repo(tmp_path)
327 _make_commit(repo, idx=0)
328 d = json.loads(_invoke(repo, "--json").output)
329 assert (d["exit_code"] == 0) == (d["all_ok"] is True)
330
331 def test_exit_code_one_iff_all_ok_false(self, tmp_path: pathlib.Path) -> None:
332 repo = _init_repo(tmp_path)
333 cid = long_id("b" * 64)
334 (heads_dir(repo) / "main").write_text(cid)
335 d = json.loads(_invoke(repo, "--json").output)
336 assert d["exit_code"] == 1
337 assert d["all_ok"] is False
338
339 def test_counters_non_negative(self, tmp_path: pathlib.Path) -> None:
340 repo = _init_repo(tmp_path)
341 _make_commit(repo, idx=0)
342 d = json.loads(_invoke(repo, "--json").output)
343 for key in ("refs_checked", "commits_checked", "snapshots_checked",
344 "objects_checked", "signatures_checked"):
345 assert d[key] >= 0, f"{key} is negative: {d[key]}"
346
347 def test_check_objects_reflected_true(self, tmp_path: pathlib.Path) -> None:
348 repo = _init_repo(tmp_path)
349 _make_commit(repo, idx=0)
350 d = json.loads(_invoke(repo, "--json").output)
351 assert d["check_objects"] is True
352
353 def test_check_objects_reflected_false(self, tmp_path: pathlib.Path) -> None:
354 repo = _init_repo(tmp_path)
355 _make_commit(repo, idx=0)
356 d = json.loads(_invoke(repo, "--json", "--no-objects").output)
357 assert d["check_objects"] is False
358
359 def test_branch_reflected_in_json(self, tmp_path: pathlib.Path) -> None:
360 repo = _init_repo(tmp_path)
361 _make_commit(repo, idx=0)
362 d = json.loads(_invoke(repo, "--json", "--branch", "main").output)
363 assert d["branch"] == "main"
364
365 def test_branch_none_when_not_specified(self, tmp_path: pathlib.Path) -> None:
366 repo = _init_repo(tmp_path)
367 _make_commit(repo, idx=0)
368 d = json.loads(_invoke(repo, "--json").output)
369 assert d["branch"] is None
370
371 def test_fail_fast_reflected_true(self, tmp_path: pathlib.Path) -> None:
372 repo = _init_repo(tmp_path)
373 _make_commit(repo, idx=0)
374 d = json.loads(_invoke(repo, "--json", "--fail-fast").output)
375 assert d["fail_fast"] is True
376
377 def test_fail_fast_reflected_false(self, tmp_path: pathlib.Path) -> None:
378 repo = _init_repo(tmp_path)
379 _make_commit(repo, idx=0)
380 d = json.loads(_invoke(repo, "--json").output)
381 assert d["fail_fast"] is False
382
383 def test_nothing_checked_false_when_commits_exist(self, tmp_path: pathlib.Path) -> None:
384 repo = _init_repo(tmp_path)
385 _make_commit(repo, idx=0)
386 d = json.loads(_invoke(repo, "--json").output)
387 assert d["nothing_checked"] is False
388
389 def test_nothing_checked_true_empty_repo(self, tmp_path: pathlib.Path) -> None:
390 repo = _init_repo(tmp_path)
391 d = json.loads(_invoke(repo, "--json").output)
392 assert d["nothing_checked"] is True
393
394 def test_duration_ms_is_float(self, tmp_path: pathlib.Path) -> None:
395 repo = _init_repo(tmp_path)
396 _make_commit(repo, idx=0)
397 d = json.loads(_invoke(repo, "--json").output)
398 assert isinstance(d["duration_ms"], float)
399
400 def test_blob_id_unique_per_content(self, tmp_path: pathlib.Path) -> None:
401 """blob_id produces distinct IDs for distinct byte sequences."""
402 ids = {blob_id(b"content-a"), blob_id(b"content-b"), blob_id(b"content-c")}
403 assert len(ids) == 3
404
405 def test_long_id_round_trips(self) -> None:
406 """long_id strips sha256: prefix correctly for comparison."""
407 hex_val = "a" * 64
408 full = long_id(hex_val)
409 assert full == "sha256:" + hex_val
410 assert full == f"sha256:{hex_val}"
411
412 def test_short_id_abbreviates(self) -> None:
413 full = long_id("f" * 64)
414 s = short_id(full)
415 assert s.startswith("sha256:")
416 assert len(s) < len(full)
417
418
419 # ---------------------------------------------------------------------------
420 # IO — OSError handling
421 # ---------------------------------------------------------------------------
422
423
424 class TestIOErrorHandling:
425 def test_exit_code_3_on_io_error_json(self, tmp_path: pathlib.Path) -> None:
426 repo = _init_repo(tmp_path)
427 with mock.patch(
428 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
429 ):
430 r = _invoke(repo, "--json")
431 assert r.exit_code == 3
432 d = json.loads(r.output)
433 assert d["exit_code"] == 3
434
435 def test_exit_code_3_on_io_error_text(self, tmp_path: pathlib.Path) -> None:
436 repo = _init_repo(tmp_path)
437 with mock.patch(
438 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
439 ):
440 r = _invoke(repo)
441 assert r.exit_code == 3
442
443 def test_io_error_json_no_stderr(self, tmp_path: pathlib.Path) -> None:
444 repo = _init_repo(tmp_path)
445 with mock.patch(
446 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
447 ):
448 r = _invoke(repo, "--json")
449 assert r.stderr.strip() == ""
450
451 def test_io_error_text_has_stderr(self, tmp_path: pathlib.Path) -> None:
452 repo = _init_repo(tmp_path)
453 with mock.patch(
454 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
455 ):
456 r = _invoke(repo)
457 assert r.stderr.strip() != ""
458
459
460 # ---------------------------------------------------------------------------
461 # P — Performance
462 # ---------------------------------------------------------------------------
463
464
465 class TestPerformance:
466 def test_duration_ms_positive(self, tmp_path: pathlib.Path) -> None:
467 repo = _init_repo(tmp_path)
468 _make_commit(repo, idx=0)
469 d = json.loads(_invoke(repo, "--json").output)
470 assert d["duration_ms"] > 0
471
472 def test_50_commit_chain_under_5000ms(self, tmp_path: pathlib.Path) -> None:
473 repo = _init_repo(tmp_path)
474 prev: str | None = None
475 for i in range(50):
476 prev = _make_commit(repo, parent_id=prev, idx=i)
477 d = json.loads(_invoke(repo, "--json").output)
478 assert d["duration_ms"] < 5_000, f"Too slow: {d['duration_ms']} ms"
479 assert d["all_ok"] is True
480
481 def test_50_commit_chain_no_objects_faster(self, tmp_path: pathlib.Path) -> None:
482 repo = _init_repo(tmp_path)
483 prev: str | None = None
484 for i in range(50):
485 prev = _make_commit(repo, parent_id=prev, idx=i)
486 full = json.loads(_invoke(repo, "--json").output)["duration_ms"]
487 fast = json.loads(_invoke(repo, "--json", "--no-objects").output)["duration_ms"]
488 # --no-objects should generally be faster; we allow some timing noise
489 # but cap both under 10 s to prevent runaway
490 assert fast < 10_000
491 assert full < 10_000
492
493
494 # ---------------------------------------------------------------------------
495 # Sec — Security
496 # ---------------------------------------------------------------------------
497
498
499 class TestSecurity:
500 def test_no_traceback_on_json_io_error(self, tmp_path: pathlib.Path) -> None:
501 repo = _init_repo(tmp_path)
502 with mock.patch(
503 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
504 ):
505 r = _invoke(repo, "--json")
506 assert "Traceback" not in r.output
507 assert "Traceback" not in r.stderr
508
509 def test_no_traceback_on_text_io_error(self, tmp_path: pathlib.Path) -> None:
510 repo = _init_repo(tmp_path)
511 with mock.patch(
512 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
513 ):
514 r = _invoke(repo)
515 assert "Traceback" not in r.output
516 assert "Traceback" not in r.stderr
517
518 def test_no_raw_exception_in_stdout(self, tmp_path: pathlib.Path) -> None:
519 repo = _init_repo(tmp_path)
520 with mock.patch(
521 "muse.cli.commands.verify.run_verify", side_effect=OSError("secret path")
522 ):
523 r = _invoke(repo, "--json")
524 # The exception message may appear in the JSON "message" field — that's
525 # intentional. What we check is that no raw Python exception string
526 # (e.g. "OSError:") leaks outside the JSON structure.
527 assert "OSError:" not in r.output
528 assert "OSError:" not in r.stderr
529
530
531 # ---------------------------------------------------------------------------
532 # C — Concurrent readers
533 # ---------------------------------------------------------------------------
534
535
536 class TestConcurrent:
537 def test_10_concurrent_reads_all_valid_json(self, tmp_path: pathlib.Path) -> None:
538 repo = _init_repo(tmp_path)
539 prev: str | None = None
540 for i in range(10):
541 prev = _make_commit(repo, parent_id=prev, idx=i)
542
543 results: list[dict] = []
544 errors: list[Exception] = []
545 lock = threading.Lock()
546
547 def _read() -> None:
548 try:
549 r = _invoke(repo, "--json")
550 d = json.loads(r.output)
551 with lock:
552 results.append(d)
553 except Exception as exc:
554 with lock:
555 errors.append(exc)
556
557 threads = [threading.Thread(target=_read) for _ in range(10)]
558 for t in threads:
559 t.start()
560 for t in threads:
561 t.join()
562
563 assert errors == [], f"Thread errors: {errors}"
564 assert len(results) == 10
565 for d in results:
566 assert d["all_ok"] is True
567 assert d["exit_code"] == 0
568 assert isinstance(d["duration_ms"], float)
File History 1 commit
sha256:d11a87833d5fad6059b7662844bf5448a8911a17cce7a51811f71ad394f248eb bump to v0.2.0rc13 Human patch 6 days ago