gabriel / muse public
test_integrity_I1_read_verify.py python
853 lines 33.3 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
1 """I-1: Read-time SHA-256 verification in read_object.
2
3 Every read now re-verifies the digest of the bytes returned against the
4 object_id. This test suite:
5
6 Unit — confirms clean objects pass, corrupt objects raise OSError.
7 Unit — MAX_FILE_BYTES size limit enforcement.
8 Unit — CRITICAL log emission on corruption.
9 Integration — confirms the plumbing cat-object command surfaces the error.
10 Integration — verify-pack catches bit-flipped LOCAL STORE objects.
11 Integration — verify-object --all audits the full local store.
12 Perf — 256 MiB object re-hash must complete within 500 ms on NVMe.
13 Stress — bit-flips at every byte position; multi-bit and random fuzz.
14 Regression — write→corrupt→read round-trip never silently returns bad data.
15 """
16 from __future__ import annotations
17
18 type _ObjPayload = dict[str, str | bytes]
19
20 import hashlib
21 import json
22 import logging
23 import os
24 import pathlib
25 import random
26 import struct
27 import tempfile
28 import time
29 from typing import TypedDict
30
31 import msgpack
32 import pytest
33
34 from muse.core.object_store import (
35 objects_dir,
36 object_path,
37 read_object,
38 write_object,
39 )
40 from muse.core.validation import MAX_FILE_BYTES
41 from muse.core._types import Manifest
42 from tests.cli_test_helper import CliRunner, InvokeResult
43
44
45 # ---------------------------------------------------------------------------
46 # Helpers
47 # ---------------------------------------------------------------------------
48
49 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
50 """Minimal .muse/ skeleton."""
51 (tmp_path / ".muse").mkdir()
52 return tmp_path
53
54
55 def _write(repo: pathlib.Path, data: bytes) -> str:
56 oid = hashlib.sha256(data).hexdigest()
57 write_object(repo, oid, data)
58 return oid
59
60
61 def _stored_path(repo: pathlib.Path, oid: str) -> pathlib.Path:
62 return object_path(repo, oid)
63
64
65 def _flip_bit(data: bytes, byte_idx: int, bit_idx: int) -> bytes:
66 """Return *data* with one bit flipped at position (byte_idx, bit_idx)."""
67 ba = bytearray(data)
68 ba[byte_idx] ^= 1 << bit_idx
69 return bytes(ba)
70
71
72 def _corrupt_file(p: pathlib.Path, new_content: bytes) -> None:
73 """Overwrite *p* with *new_content*, temporarily lifting the 0o444 guard.
74
75 Object files are written with mode 0o444 (read-only) to enforce
76 content-addressability at the OS level. Corruption tests must override
77 that protection to simulate disk errors, cosmic-ray bit flips, etc.
78 The permission is restored to 0o444 after the write.
79 """
80 os.chmod(p, 0o644)
81 try:
82 p.write_bytes(new_content)
83 finally:
84 os.chmod(p, 0o444)
85
86
87 def _corrupt_stored(repo: pathlib.Path, oid: str, byte_idx: int = 0, bit_idx: int = 0) -> None:
88 """Flip one bit in the on-disk object file."""
89 p = _stored_path(repo, oid)
90 data = p.read_bytes()
91 _corrupt_file(p, _flip_bit(data, byte_idx, bit_idx))
92
93
94 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
95 runner = CliRunner()
96 env = {"MUSE_REPO_ROOT": str(repo)}
97 return runner.invoke(None, ["cat-object", *args], env=env)
98
99
100 # ---------------------------------------------------------------------------
101 # Unit: happy path — clean objects always pass
102 # ---------------------------------------------------------------------------
103
104 class TestCleanObjectsPass:
105 def test_empty_bytes(self, tmp_path: pathlib.Path) -> None:
106 repo = _repo(tmp_path)
107 oid = _write(repo, b"")
108 assert read_object(repo, oid) == b""
109
110 def test_small_ascii(self, tmp_path: pathlib.Path) -> None:
111 repo = _repo(tmp_path)
112 data = b"hello muse"
113 oid = _write(repo, data)
114 assert read_object(repo, oid) == data
115
116 def test_binary_blob(self, tmp_path: pathlib.Path) -> None:
117 repo = _repo(tmp_path)
118 data = bytes(range(256)) * 100
119 oid = _write(repo, data)
120 assert read_object(repo, oid) == data
121
122 def test_1_mib_object(self, tmp_path: pathlib.Path) -> None:
123 repo = _repo(tmp_path)
124 data = os.urandom(1024 * 1024)
125 oid = _write(repo, data)
126 assert read_object(repo, oid) == data
127
128 def test_read_twice_same_result(self, tmp_path: pathlib.Path) -> None:
129 repo = _repo(tmp_path)
130 data = b"idempotent read"
131 oid = _write(repo, data)
132 assert read_object(repo, oid) == read_object(repo, oid)
133
134 def test_absent_returns_none(self, tmp_path: pathlib.Path) -> None:
135 repo = _repo(tmp_path)
136 absent = "a" * 64
137 assert read_object(repo, absent) is None
138
139
140 # ---------------------------------------------------------------------------
141 # Unit: single-bit corruption always raises OSError
142 # ---------------------------------------------------------------------------
143
144 class TestSingleBitCorruption:
145 def test_flip_first_byte_first_bit(self, tmp_path: pathlib.Path) -> None:
146 repo = _repo(tmp_path)
147 oid = _write(repo, b"critical data")
148 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=0)
149 with pytest.raises(OSError, match="integrity check"):
150 read_object(repo, oid)
151
152 def test_flip_first_byte_last_bit(self, tmp_path: pathlib.Path) -> None:
153 repo = _repo(tmp_path)
154 oid = _write(repo, b"critical data")
155 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=7)
156 with pytest.raises(OSError, match="integrity check"):
157 read_object(repo, oid)
158
159 def test_flip_last_byte(self, tmp_path: pathlib.Path) -> None:
160 repo = _repo(tmp_path)
161 data = b"end matters too"
162 oid = _write(repo, data)
163 _corrupt_stored(repo, oid, byte_idx=len(data) - 1, bit_idx=3)
164 with pytest.raises(OSError, match="integrity check"):
165 read_object(repo, oid)
166
167 def test_flip_middle_byte(self, tmp_path: pathlib.Path) -> None:
168 repo = _repo(tmp_path)
169 data = b"middle byte flip"
170 oid = _write(repo, data)
171 mid = len(data) // 2
172 _corrupt_stored(repo, oid, byte_idx=mid, bit_idx=4)
173 with pytest.raises(OSError, match="integrity check"):
174 read_object(repo, oid)
175
176 def test_error_message_contains_expected_prefix(self, tmp_path: pathlib.Path) -> None:
177 repo = _repo(tmp_path)
178 oid = _write(repo, b"check message content")
179 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=1)
180 with pytest.raises(OSError) as exc_info:
181 read_object(repo, oid)
182 msg = str(exc_info.value)
183 assert oid[:8] in msg
184 assert "SHA-256" in msg or "integrity" in msg.lower()
185
186 def test_error_suggests_verify_pack(self, tmp_path: pathlib.Path) -> None:
187 repo = _repo(tmp_path)
188 oid = _write(repo, b"suggest remedy")
189 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=0)
190 with pytest.raises(OSError) as exc_info:
191 read_object(repo, oid)
192 assert "verify-pack" in str(exc_info.value)
193
194 def test_clean_sibling_unaffected(self, tmp_path: pathlib.Path) -> None:
195 """Corruption of one object must not affect a sibling object."""
196 repo = _repo(tmp_path)
197 oid_a = _write(repo, b"object a - clean")
198 oid_b = _write(repo, b"object b - will corrupt")
199 _corrupt_stored(repo, oid_b, byte_idx=0, bit_idx=0)
200 # b is corrupt
201 with pytest.raises(OSError):
202 read_object(repo, oid_b)
203 # a is still fine
204 assert read_object(repo, oid_a) == b"object a - clean"
205
206 def test_truncated_file_raises(self, tmp_path: pathlib.Path) -> None:
207 """A file truncated to zero bytes is caught by the hash check."""
208 repo = _repo(tmp_path)
209 data = b"will be truncated"
210 oid = _write(repo, data)
211 _corrupt_file(_stored_path(repo, oid), b"")
212 with pytest.raises(OSError, match="integrity check"):
213 read_object(repo, oid)
214
215 def test_fully_zeroed_file_raises(self, tmp_path: pathlib.Path) -> None:
216 repo = _repo(tmp_path)
217 data = b"zeroed out"
218 oid = _write(repo, data)
219 _corrupt_file(_stored_path(repo, oid), b"\x00" * len(data))
220 with pytest.raises(OSError, match="integrity check"):
221 read_object(repo, oid)
222
223 def test_appended_byte_raises(self, tmp_path: pathlib.Path) -> None:
224 """A byte appended to the end is caught."""
225 repo = _repo(tmp_path)
226 data = b"exact bytes"
227 oid = _write(repo, data)
228 _corrupt_file(_stored_path(repo, oid), data + b"\xff")
229 with pytest.raises(OSError, match="integrity check"):
230 read_object(repo, oid)
231
232 def test_prepended_byte_raises(self, tmp_path: pathlib.Path) -> None:
233 """A byte prepended to the start is caught."""
234 repo = _repo(tmp_path)
235 data = b"exact bytes"
236 oid = _write(repo, data)
237 _corrupt_file(_stored_path(repo, oid), b"\x00" + data)
238 with pytest.raises(OSError, match="integrity check"):
239 read_object(repo, oid)
240
241
242 # ---------------------------------------------------------------------------
243 # Stress: exhaustive single-bit sweep
244 # ---------------------------------------------------------------------------
245
246 class TestExhaustiveBitFlip:
247 def test_every_bit_in_32_byte_object(self, tmp_path: pathlib.Path) -> None:
248 """Every one of the 256 single-bit flips in a 32-byte payload is caught."""
249 repo = _repo(tmp_path)
250 data = bytes(range(32))
251 oid = _write(repo, data)
252 p = _stored_path(repo, oid)
253 original = p.read_bytes()
254 caught = 0
255 for byte_idx in range(len(original)):
256 for bit_idx in range(8):
257 flipped = _flip_bit(original, byte_idx, bit_idx)
258 _corrupt_file(p, flipped)
259 try:
260 read_object(repo, oid)
261 # A flip that happens to produce a valid SHA-256 preimage
262 # is theoretically impossible — if this branch is hit, fail.
263 pytest.fail(
264 f"Bit flip at byte={byte_idx} bit={bit_idx} "
265 "was not caught — corrupt data returned silently"
266 )
267 except OSError:
268 caught += 1
269 finally:
270 _corrupt_file(p, original)
271 assert caught == len(original) * 8
272
273 @pytest.mark.slow
274 def test_every_bit_in_4096_byte_object(self, tmp_path: pathlib.Path) -> None:
275 """Every bit flip in a 4 KiB object is caught (32 768 checks)."""
276 repo = _repo(tmp_path)
277 data = os.urandom(4096)
278 oid = _write(repo, data)
279 p = _stored_path(repo, oid)
280 original = p.read_bytes()
281 for byte_idx in range(len(original)):
282 for bit_idx in range(8):
283 flipped = _flip_bit(original, byte_idx, bit_idx)
284 _corrupt_file(p, flipped)
285 with pytest.raises(OSError):
286 read_object(repo, oid)
287 _corrupt_file(p, original)
288
289
290 # ---------------------------------------------------------------------------
291 # Stress: multi-bit and random fuzz
292 # ---------------------------------------------------------------------------
293
294 class TestFuzzCorruption:
295 def test_5_random_bits_1000_iterations(self, tmp_path: pathlib.Path) -> None:
296 """Random 5-bit corruption: zero silent passes in 1000 trials."""
297 repo = _repo(tmp_path)
298 data = os.urandom(256)
299 oid = _write(repo, data)
300 p = _stored_path(repo, oid)
301 original = p.read_bytes()
302 rng = random.Random(42)
303 silent_passes = 0
304 for _ in range(1000):
305 ba = bytearray(original)
306 for _ in range(5):
307 idx = rng.randrange(len(ba))
308 bit = rng.randrange(8)
309 ba[idx] ^= 1 << bit
310 _corrupt_file(p, bytes(ba))
311 try:
312 read_object(repo, oid)
313 silent_passes += 1
314 except OSError:
315 pass
316 finally:
317 _corrupt_file(p, original)
318 assert silent_passes == 0, f"{silent_passes} corrupt reads went undetected"
319
320 def test_completely_random_content_1000_iterations(self, tmp_path: pathlib.Path) -> None:
321 """Replacing the file with entirely random bytes is always caught."""
322 repo = _repo(tmp_path)
323 data = os.urandom(128)
324 oid = _write(repo, data)
325 p = _stored_path(repo, oid)
326 original = p.read_bytes()
327 rng = random.Random(99)
328 for _ in range(1000):
329 garbage = bytes(rng.randrange(256) for _ in range(len(original)))
330 _corrupt_file(p, garbage)
331 with pytest.raises(OSError):
332 read_object(repo, oid)
333 _corrupt_file(p, original)
334
335 def test_struct_pack_corruption(self, tmp_path: pathlib.Path) -> None:
336 """Struct-level 4-byte word corruption is always caught."""
337 repo = _repo(tmp_path)
338 data = b"struct corruption test " * 10
339 oid = _write(repo, data)
340 p = _stored_path(repo, oid)
341 original = p.read_bytes()
342 for word_offset in range(0, len(original) - 3, 4):
343 ba = bytearray(original)
344 # XOR one 32-bit word
345 word = struct.unpack_from(">I", ba, word_offset)[0]
346 struct.pack_into(">I", ba, word_offset, word ^ 0xDEADBEEF)
347 _corrupt_file(p, bytes(ba))
348 with pytest.raises(OSError):
349 read_object(repo, oid)
350 _corrupt_file(p, original)
351
352
353 # ---------------------------------------------------------------------------
354 # Integration: plumbing cat-object surfaces the error
355 # ---------------------------------------------------------------------------
356
357 class TestCatObjectIntegration:
358 def test_cat_clean_object_json(self, tmp_path: pathlib.Path) -> None:
359 repo = _repo(tmp_path)
360 data = b"cat-object integration test"
361 oid = _write(repo, data)
362 r = _invoke(repo, "--json", oid)
363 assert r.exit_code == 0
364 import json
365 d = json.loads(r.output)
366 assert d["object_id"] == oid
367
368 def test_cat_corrupt_object_errors(self, tmp_path: pathlib.Path) -> None:
369 """cat-object raw mode on a bit-flipped object must exit non-zero.
370
371 The --json (info) mode only checks file existence/size — it intentionally
372 does not read content. The raw mode MUST verify the hash before streaming
373 any bytes to stdout.
374 """
375 repo = _repo(tmp_path)
376 data = b"will be corrupted for cat-object test"
377 oid = _write(repo, data)
378 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=0)
379 # Raw mode (no --json) is the mode that reads and streams bytes.
380 r = _invoke(repo, oid)
381 assert r.exit_code != 0
382
383 def test_cat_corrupt_object_no_raw_data_in_output(self, tmp_path: pathlib.Path) -> None:
384 """A corrupt object must NEVER have its raw bytes echoed to stdout."""
385 repo = _repo(tmp_path)
386 sentinel = b"TOP_SECRET_PAYLOAD_MUST_NOT_LEAK"
387 oid = _write(repo, sentinel)
388 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=0)
389 r = _invoke(repo, oid)
390 # The sentinel string must not appear anywhere in stdout
391 assert b"TOP_SECRET_PAYLOAD" not in r.stdout_bytes
392
393
394 # ---------------------------------------------------------------------------
395 # Regression: write → corrupt → read never returns bad data
396 # ---------------------------------------------------------------------------
397
398 class TestRegressionSilentCorruption:
399 def test_concurrent_read_after_corruption(self, tmp_path: pathlib.Path) -> None:
400 """Simulate a read race: write clean, corrupt disk, read — must raise."""
401 import threading
402 repo = _repo(tmp_path)
403 data = os.urandom(4096)
404 oid = _write(repo, data)
405 results: list[str] = []
406
407 def corrupt_then_read() -> None:
408 _corrupt_stored(repo, oid, byte_idx=100, bit_idx=3)
409 try:
410 read_object(repo, oid)
411 results.append("silent_pass")
412 except OSError:
413 results.append("caught")
414
415 t = threading.Thread(target=corrupt_then_read)
416 t.start()
417 t.join()
418 assert "silent_pass" not in results, "Corrupt data returned silently in thread"
419
420 def test_large_object_stream_integrity(self, tmp_path: pathlib.Path) -> None:
421 """16 MiB object: corruption in the second chunk boundary is caught."""
422 repo = _repo(tmp_path)
423 # 16 MiB — forces multiple 64 KiB streaming chunks
424 data = os.urandom(16 * 1024 * 1024)
425 oid = _write(repo, data)
426 p = _stored_path(repo, oid)
427 original = p.read_bytes()
428 # Corrupt a byte at the second chunk boundary (64 KiB + 1)
429 _corrupt_file(p, _flip_bit(original, 65537, 0))
430 with pytest.raises(OSError, match="integrity check"):
431 read_object(repo, oid)
432 # Restore and confirm clean read works
433 _corrupt_file(p, original)
434 assert read_object(repo, oid) == data
435
436
437 # ---------------------------------------------------------------------------
438 # Gap 2: CRITICAL log emission on corruption
439 # ---------------------------------------------------------------------------
440
441 class TestCriticalLogOnCorruption:
442 """Corruption must be logged at CRITICAL — agents parsing structured logs
443 must receive a severity signal, not just a silent Python exception."""
444
445 def test_critical_logged_on_bit_flip(
446 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
447 ) -> None:
448 repo = _repo(tmp_path)
449 data = b"must log at critical"
450 oid = _write(repo, data)
451 _corrupt_stored(repo, oid, byte_idx=0, bit_idx=0)
452
453 with caplog.at_level(logging.CRITICAL, logger="muse.core.object_store"):
454 with pytest.raises(OSError):
455 read_object(repo, oid)
456
457 critical_records = [r for r in caplog.records if r.levelno >= logging.CRITICAL]
458 assert critical_records, "No CRITICAL log emitted on bit-flip corruption"
459 assert any(oid[:8] in r.getMessage() for r in critical_records), (
460 "CRITICAL log does not include the object ID"
461 )
462
463 def test_critical_message_mentions_corruption(
464 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
465 ) -> None:
466 repo = _repo(tmp_path)
467 data = b"critical message check"
468 oid = _write(repo, data)
469 _corrupt_stored(repo, oid, byte_idx=5, bit_idx=2)
470
471 with caplog.at_level(logging.CRITICAL, logger="muse.core.object_store"):
472 with pytest.raises(OSError):
473 read_object(repo, oid)
474
475 messages = " ".join(r.getMessage() for r in caplog.records)
476 assert "corrupt" in messages.lower() or "integrity" in messages.lower(), (
477 f"CRITICAL log does not mention corruption: {messages!r}"
478 )
479
480 def test_no_critical_on_clean_read(
481 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
482 ) -> None:
483 """A clean read must NOT emit CRITICAL — no false alarms."""
484 repo = _repo(tmp_path)
485 data = b"clean - no alarm"
486 oid = _write(repo, data)
487
488 with caplog.at_level(logging.CRITICAL, logger="muse.core.object_store"):
489 result = read_object(repo, oid)
490
491 assert result == data
492 critical_records = [r for r in caplog.records if r.levelno >= logging.CRITICAL]
493 assert critical_records == [], f"False CRITICAL alarm on clean read: {critical_records}"
494
495
496 # ---------------------------------------------------------------------------
497 # Gap 3: MAX_FILE_BYTES size limit enforcement
498 # ---------------------------------------------------------------------------
499
500 class TestMaxFileBytesLimit:
501 """read_object must reject objects that exceed MAX_FILE_BYTES before
502 reading their content into memory — preventing OOM on pathological input."""
503
504 def test_oversized_object_raises_oserror(self, tmp_path: pathlib.Path) -> None:
505 """A file exceeding MAX_FILE_BYTES raises OSError before any read."""
506 from unittest.mock import patch as _patch, MagicMock
507 repo = _repo(tmp_path)
508 data = b"placeholder"
509 oid = _write(repo, data)
510
511 # Inject a fake stat result with an inflated st_size so we don't
512 # need to allocate gigabytes of real data.
513 mock_stat = MagicMock()
514 mock_stat.st_size = MAX_FILE_BYTES + 1
515
516 with _patch.object(pathlib.Path, "stat", return_value=mock_stat):
517 with pytest.raises(OSError, match="MiB read limit"):
518 read_object(repo, oid)
519
520 def test_exactly_max_size_allowed(self, tmp_path: pathlib.Path) -> None:
521 """An object exactly at MAX_FILE_BYTES must be readable (boundary check)."""
522 from unittest.mock import patch as _patch, MagicMock
523 repo = _repo(tmp_path)
524 data = b"boundary"
525 oid = _write(repo, data)
526
527 # st_size == MAX_FILE_BYTES: the guard is strict greater-than, so this
528 # should not fire. The actual (small) file is then read and verified.
529 mock_stat = MagicMock()
530 mock_stat.st_size = MAX_FILE_BYTES
531 with _patch.object(pathlib.Path, "stat", return_value=mock_stat):
532 result = read_object(repo, oid)
533
534 assert result == data
535
536 def test_error_message_includes_mib_limit(self, tmp_path: pathlib.Path) -> None:
537 """The OSError message must include the configured MiB limit."""
538 from unittest.mock import patch as _patch, MagicMock
539 repo = _repo(tmp_path)
540 data = b"size limit error msg"
541 oid = _write(repo, data)
542
543 mock_stat = MagicMock()
544 mock_stat.st_size = MAX_FILE_BYTES + 1024
545
546 with _patch.object(pathlib.Path, "stat", return_value=mock_stat):
547 with pytest.raises(OSError) as exc_info:
548 read_object(repo, oid)
549
550 assert "MiB" in str(exc_info.value), (
551 f"Error message does not include MiB limit: {exc_info.value}"
552 )
553
554
555 # ---------------------------------------------------------------------------
556 # Gap 4: verify-pack catches bit-flipped LOCAL STORE objects
557 # ---------------------------------------------------------------------------
558
559 def _full_repo(tmp_path: pathlib.Path) -> pathlib.Path:
560 """Create a minimal repo with objects, HEAD and config for CLI invocation."""
561 muse_dir = tmp_path / ".muse"
562 muse_dir.mkdir()
563 for d in ("objects", "commits", "snapshots", "refs/heads"):
564 (muse_dir / d).mkdir(parents=True, exist_ok=True)
565 (muse_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")
566 (muse_dir / "repo.json").write_text(
567 '{"repo_id": "test-repo", "domain": "code", "default_branch": "main"}',
568 encoding="utf-8",
569 )
570 return tmp_path
571
572
573 class _BundleSnapEntry(TypedDict, total=False):
574 snapshot_id: str
575 manifest: Manifest
576
577
578 class _BundleDict(TypedDict):
579 objects: list[dict[str, str | bytes]]
580 snapshots: list[_BundleSnapEntry]
581 commits: list[dict[str, str]]
582
583
584 def _good_bundle_obj(data: bytes) -> _ObjPayload:
585 oid = hashlib.sha256(data).hexdigest()
586 return {"object_id": oid, "content": data}
587
588
589 def _make_pack(objects: list[_ObjPayload]) -> bytes:
590 bundle: _BundleDict = {"objects": objects, "snapshots": [], "commits": []}
591 packed: bytes = msgpack.packb(bundle, use_bin_type=True)
592 return packed
593
594
595 def _snap_bundle(snap_id: str, manifest: Manifest) -> bytes:
596 """Build a bundle whose snapshot references objects in the LOCAL STORE."""
597 bundle: _BundleDict = {
598 "objects": [],
599 "snapshots": [{"snapshot_id": snap_id, "manifest": manifest}],
600 "commits": [],
601 }
602 packed: bytes = msgpack.packb(bundle, use_bin_type=True)
603 return packed
604
605
606 class TestVerifyPackLocalStoreIntegrity:
607 """verify-pack must catch SHA-256 mismatches in LOCAL STORE objects that
608 are referenced by bundle snapshots — not just objects inside the bundle.
609
610 Before the fix: has_object() (existence check only) was used.
611 After the fix: read_object() (hash-verified read) is used — a bit-flipped
612 local store object is reported as a failure.
613 """
614
615 def _vp(
616 self,
617 repo: pathlib.Path,
618 extra_args: list[str],
619 env_root: pathlib.Path | None = None,
620 ) -> "InvokeResult":
621 env_root = env_root or repo
622 runner = CliRunner()
623 return runner.invoke(None, ["verify-pack"] + extra_args,
624 env={"MUSE_REPO_ROOT": str(env_root)})
625
626 def test_clean_local_store_object_passes(self, tmp_path: pathlib.Path) -> None:
627 """A snapshot referencing a clean local store object must verify OK."""
628 repo = _full_repo(tmp_path)
629 data = b"clean local object"
630 oid = hashlib.sha256(data).hexdigest()
631 write_object(repo, oid, data)
632
633 snap_id = hashlib.sha256(b"snap1").hexdigest()
634 bf = tmp_path / "bundle.muse"
635 bf.write_bytes(_snap_bundle(snap_id, {"file.py": oid}))
636
637 r = self._vp(repo, ["--file", str(bf)])
638 assert r.exit_code == 0, f"Expected 0 but got {r.exit_code}: {r.output}"
639 d = json.loads(r.output)
640 assert d["all_ok"] is True
641
642 def test_bit_flipped_local_store_object_fails(self, tmp_path: pathlib.Path) -> None:
643 """A snapshot referencing a bit-flipped local store object must fail.
644
645 This is the core regression test for the has_object→read_object fix.
646 Before the fix, verify-pack would report all_ok=True even when the local
647 store contained a corrupt object.
648 """
649 repo = _full_repo(tmp_path)
650 data = b"will be bit-flipped in local store"
651 oid = hashlib.sha256(data).hexdigest()
652 write_object(repo, oid, data)
653
654 # Flip a bit in the locally stored object.
655 stored = object_path(repo, oid)
656 original = stored.read_bytes()
657 _corrupt_file(stored, _flip_bit(original, 0, 0))
658
659 snap_id = hashlib.sha256(b"snap2").hexdigest()
660 bf = tmp_path / "bundle.muse"
661 bf.write_bytes(_snap_bundle(snap_id, {"code.py": oid}))
662
663 r = self._vp(repo, ["--file", str(bf)])
664 assert r.exit_code != 0, (
665 "verify-pack reported all_ok=True on a bit-flipped local store object "
666 "(regression: has_object() was used instead of read_object())"
667 )
668 d = json.loads(r.output)
669 assert d["all_ok"] is False
670 assert any(
671 "integrity" in f["error"].lower() or "sha-256" in f["error"].lower()
672 for f in d["failures"]
673 ), f"No integrity error in failures: {d['failures']}"
674
675 def test_zeroed_local_store_object_fails(self, tmp_path: pathlib.Path) -> None:
676 """Zeroing the stored file content is caught by verify-pack."""
677 repo = _full_repo(tmp_path)
678 data = b"will be zeroed"
679 oid = hashlib.sha256(data).hexdigest()
680 write_object(repo, oid, data)
681 _corrupt_file(object_path(repo, oid), b"\x00" * len(data))
682
683 snap_id = hashlib.sha256(b"snap3").hexdigest()
684 bf = tmp_path / "bundle.muse"
685 bf.write_bytes(_snap_bundle(snap_id, {"z.py": oid}))
686
687 r = self._vp(repo, ["--file", str(bf)])
688 d = json.loads(r.output)
689 assert d["all_ok"] is False
690
691 def test_no_local_flag_skips_local_check(self, tmp_path: pathlib.Path) -> None:
692 """--no-local skips local store checks entirely — corrupt local object not reported."""
693 repo = _full_repo(tmp_path)
694 data = b"corrupt but skipped"
695 oid = hashlib.sha256(data).hexdigest()
696 write_object(repo, oid, data)
697 stored = object_path(repo, oid)
698 _corrupt_file(stored, _flip_bit(stored.read_bytes(), 0, 0))
699
700 snap_id = hashlib.sha256(b"snap4").hexdigest()
701 bf = tmp_path / "bundle.muse"
702 bf.write_bytes(_snap_bundle(snap_id, {"s.py": oid}))
703
704 r = self._vp(repo, ["--file", str(bf), "--no-local"])
705 # --no-local skips the integrity check; the object is "missing" from
706 # the bundle (not present in bundle_object_ids) and local check is skipped,
707 # so the snapshot manifest entry reports a missing object — not a corruption.
708 d = json.loads(r.output)
709 # Either all_ok (if no manifest check happens) or a "missing" failure —
710 # crucially NOT an integrity/SHA-256 failure.
711 if not d["all_ok"]:
712 for f in d["failures"]:
713 assert "integrity" not in f["error"].lower(), (
714 "--no-local should not report integrity failures"
715 )
716
717
718 # ---------------------------------------------------------------------------
719 # Gap 5: verify-object --all audits the full local store
720 # ---------------------------------------------------------------------------
721
722 class TestVerifyObjectAllCorrupt:
723 """muse plumbing verify-object --all must surface bit-flipped objects
724 across the entire local store — not just objects passed on the CLI."""
725
726 def _vobj(
727 self, repo: pathlib.Path, args: list[str]
728 ) -> "InvokeResult":
729 runner = CliRunner()
730 return runner.invoke(
731 None, ["verify-object"] + args,
732 env={"MUSE_REPO_ROOT": str(repo)},
733 )
734
735 def test_verify_all_clean_store_passes(self, tmp_path: pathlib.Path) -> None:
736 repo = _full_repo(tmp_path)
737 for i in range(5):
738 write_object(repo, hashlib.sha256(f"obj{i}".encode()).hexdigest(), f"obj{i}".encode())
739 r = self._vobj(repo, ["--all"])
740 assert r.exit_code == 0
741 d = json.loads(r.output)
742 assert d["all_ok"] is True
743 assert d["checked"] == 5
744
745 def test_verify_all_catches_single_bit_flip(self, tmp_path: pathlib.Path) -> None:
746 """--all must detect a bit-flip in one object among many clean ones."""
747 repo = _full_repo(tmp_path)
748 oids: list[str] = []
749 for i in range(10):
750 data = f"object-{i}".encode()
751 oid = hashlib.sha256(data).hexdigest()
752 write_object(repo, oid, data)
753 oids.append(oid)
754
755 # Corrupt exactly one object.
756 corrupt_oid = oids[4]
757 p = object_path(repo, corrupt_oid)
758 _corrupt_file(p, _flip_bit(p.read_bytes(), 0, 0))
759
760 r = self._vobj(repo, ["--all"])
761 assert r.exit_code != 0
762 d = json.loads(r.output)
763 assert d["all_ok"] is False
764 assert d["failed"] == 1
765 assert any(res["object_id"] == corrupt_oid for res in d["results"] if not res["ok"])
766
767 def test_verify_all_catches_multiple_corruptions(self, tmp_path: pathlib.Path) -> None:
768 """--all must catch ALL corrupt objects, not just the first."""
769 repo = _full_repo(tmp_path)
770 oids: list[str] = []
771 for i in range(6):
772 data = f"multi-corrupt-{i}".encode()
773 oid = hashlib.sha256(data).hexdigest()
774 write_object(repo, oid, data)
775 oids.append(oid)
776
777 # Corrupt three objects.
778 corrupt = {oids[0], oids[2], oids[5]}
779 for oid in corrupt:
780 p = object_path(repo, oid)
781 _corrupt_file(p, _flip_bit(p.read_bytes(), 0, 0))
782
783 r = self._vobj(repo, ["--all"])
784 d = json.loads(r.output)
785 assert d["all_ok"] is False
786 failed_ids = {res["object_id"] for res in d["results"] if not res["ok"]}
787 assert failed_ids == corrupt, f"Expected {corrupt}, got {failed_ids}"
788
789 def test_verify_explicit_id_corrupt(self, tmp_path: pathlib.Path) -> None:
790 """Passing a corrupt object ID explicitly must detect the failure."""
791 repo = _full_repo(tmp_path)
792 data = b"explicit check"
793 oid = hashlib.sha256(data).hexdigest()
794 write_object(repo, oid, data)
795 p = object_path(repo, oid)
796 _corrupt_file(p, _flip_bit(p.read_bytes(), 3, 1))
797
798 r = self._vobj(repo, [oid])
799 assert r.exit_code != 0
800 d = json.loads(r.output)
801 assert d["all_ok"] is False
802
803
804 # ---------------------------------------------------------------------------
805 # Gap 6: Performance benchmark — 256 MiB re-hash < 500 ms
806 # ---------------------------------------------------------------------------
807
808 class TestPerformanceBenchmark:
809 """The read-time re-hash must not introduce unacceptable latency.
810
811 Plan requirement: overhead of re-hashing a 256 MiB object must be
812 < 500 ms on modern hardware (streaming SHA-256 is I/O-bound, not
813 CPU-bound — the bottleneck is NVMe throughput, not the hash itself).
814
815 This test writes to a tmpfs / in-memory filesystem (tmp_path) so it
816 measures pure CPU and memory bandwidth, which is the lower bound.
817 Real NVMe latency may be higher but SHA-256 itself adds < 50 ms for
818 256 MiB on any modern CPU.
819 """
820
821 @pytest.mark.perf
822 def test_256_mib_hash_under_500ms(self, tmp_path: pathlib.Path) -> None:
823 """read_object on a 256 MiB blob must complete within 500 ms."""
824 repo = _repo(tmp_path)
825 size = 256 * 1024 * 1024
826 data = os.urandom(size)
827 oid = _write(repo, data)
828
829 start = time.perf_counter()
830 result = read_object(repo, oid)
831 elapsed_ms = (time.perf_counter() - start) * 1000
832
833 assert result == data, "256 MiB object content corrupted"
834 assert elapsed_ms < 500, (
835 f"read_object re-hash took {elapsed_ms:.1f} ms on a 256 MiB object "
836 f"— exceeds the 500 ms budget. SHA-256 performance regression detected."
837 )
838
839 @pytest.mark.perf
840 def test_1_mib_hash_under_10ms(self, tmp_path: pathlib.Path) -> None:
841 """1 MiB object hash must be sub-10 ms — baseline for small commits."""
842 repo = _repo(tmp_path)
843 data = os.urandom(1024 * 1024)
844 oid = _write(repo, data)
845
846 start = time.perf_counter()
847 result = read_object(repo, oid)
848 elapsed_ms = (time.perf_counter() - start) * 1000
849
850 assert result == data
851 assert elapsed_ms < 10, (
852 f"1 MiB read_object took {elapsed_ms:.1f} ms — performance regression"
853 )
File History 5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 26 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e Merge branch 'dev' into main Human 29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce Merge branch 'dev' into main Human 48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 100 days ago