gabriel / muse public
test_integrity_I4_msgpack_size.py python
766 lines 31.0 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 27 days ago
1 """I-4: Msgpack read size limit — prevent OOM from oversized store files.
2
3 Problem (pre-fix): ``_read_msgpack`` called ``path.read_bytes()`` with no
4 size guard. A 10 GiB corrupt or adversarially crafted ``.msgpack`` file
5 would allocate 10 GiB of RAM, crashing the process or triggering the OOM
6 killer — a critical data-integrity and availability failure.
7
8 ``read_object`` in the object store already had a 256 MiB cap. The commit,
9 snapshot, tag, release, and index stores did not.
10
11 Fix: added to both ``muse/core/store.py`` and ``muse/core/indices.py``:
12
13 1. ``MAX_MSGPACK_BYTES = 64 MiB`` — ``stat().st_size`` is checked *before*
14 ``read_bytes()`` so no allocation ever occurs.
15 2. Per-value limits on ``msgpack.unpackb`` — ``max_str_len``,
16 ``max_bin_len``, ``max_array_len``, ``max_map_len`` — prevent deeply
17 nested or pathologically large single-value documents from consuming
18 unbounded memory even within the size cap.
19
20 This file proves every aspect of the fix:
21
22 Tier 0 — constant export
23 Tier 1 — stat check before read (OOM prevention)
24 Tier 2 — per-value unpack limits
25 Tier 3 — all high-level read functions (read_commit, read_snapshot, …)
26 Tier 4 — index file protection
27 Tier 5 — CLI plumbing command (clean JSON error, no traceback)
28 Tier 6 — boundary / exact-limit behaviour
29 Tier 7 — performance (size check adds < 1 ms overhead)
30 Tier 8 — warning log on oversized file
31 """
32 from __future__ import annotations
33
34 import datetime
35 import hashlib
36 import logging
37 import pathlib
38 import time
39 from unittest.mock import patch, MagicMock
40
41 import msgpack
42 import pytest
43
44 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
45 from muse.core.store import (
46 MAX_MSGPACK_BYTES,
47 MsgpackValue,
48 CommitRecord,
49 TagRecord,
50 SnapshotRecord,
51 read_commit,
52 read_snapshot,
53 write_commit,
54 write_snapshot,
55 write_tag,
56 get_all_tags,
57 list_releases,
58 )
59
60 from muse.core._types import Manifest, MsgpackDict
61 from muse.core.indices import (
62 load_symbol_history,
63 load_hash_occurrence,
64 )
65
66
67 # ---------------------------------------------------------------------------
68 # Helpers
69 # ---------------------------------------------------------------------------
70
71 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
72 muse = tmp_path / ".muse"
73 (muse / "commits").mkdir(parents=True)
74 (muse / "snapshots").mkdir()
75 (muse / "tags").mkdir()
76 (muse / "releases").mkdir()
77 (muse / "indices").mkdir()
78 (muse / "refs" / "heads").mkdir(parents=True)
79 (muse / "HEAD").write_text("ref: refs/heads/main\n")
80 return tmp_path
81
82
83 def _sha(seed: str) -> str:
84 return hashlib.sha256(seed.encode()).hexdigest()
85
86
87 def _commit(idx: int = 0) -> CommitRecord:
88 snapshot_id = compute_snapshot_id({})
89 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
90 message = f"commit {idx}"
91 commit_id = compute_commit_id([], snapshot_id, message, committed_at.isoformat())
92 return CommitRecord(
93 commit_id=commit_id,
94 repo_id="test-repo",
95 branch="main",
96 snapshot_id=snapshot_id,
97 message=message,
98 committed_at=committed_at,
99 author="tester",
100 parent_commit_id=None,
101 parent2_commit_id=None,
102 )
103
104
105 def _snapshot(idx: int = 0) -> SnapshotRecord:
106 manifest: Manifest = {f"__idx__": _sha(f"snap-{idx}")}
107 sid = compute_snapshot_id(manifest)
108 return SnapshotRecord(
109 snapshot_id=sid,
110 manifest=manifest,
111 )
112
113
114 def _tag(idx: int = 0) -> TagRecord:
115 return TagRecord(
116 tag_id=_sha(f"tag-id-{idx}"),
117 repo_id="test-repo",
118 commit_id=_sha(f"tag-commit-{idx}"),
119 tag=f"v{idx}.0.0",
120 )
121
122
123 # ---------------------------------------------------------------------------
124 # Tier 0 — constant export
125 # ---------------------------------------------------------------------------
126
127 class TestConstantExport:
128 """MAX_MSGPACK_BYTES must be importable and have the correct value."""
129
130 def test_max_msgpack_bytes_is_exported(self) -> None:
131 from muse.core.store import MAX_MSGPACK_BYTES as cap
132 assert cap == 64 * 1024 * 1024, (
133 f"Expected 64 MiB (67108864), got {cap}"
134 )
135
136 def test_max_msgpack_bytes_is_int(self) -> None:
137 assert isinstance(MAX_MSGPACK_BYTES, int)
138
139 def test_max_msgpack_bytes_less_than_256mib(self) -> None:
140 """Commit/snapshot records should be capped well below 256 MiB."""
141 assert MAX_MSGPACK_BYTES < 256 * 1024 * 1024, (
142 "Msgpack records should be capped below the object store's 256 MiB limit"
143 )
144
145
146 # ---------------------------------------------------------------------------
147 # Tier 1 — stat check fires BEFORE read_bytes (the OOM prevention)
148 # ---------------------------------------------------------------------------
149
150 class TestStatCheckBeforeRead:
151 """The size guard must fire before any read_bytes() call.
152
153 We prove this by mocking stat to report an oversized file while keeping
154 the actual file tiny — if read_bytes() were called first, we would NOT
155 trigger the OSError from the stat check.
156 """
157
158 def _oversized_stat(self, real_path: pathlib.Path) -> MagicMock:
159 """Return a MagicMock that reports st_size = MAX_MSGPACK_BYTES + 1."""
160 stat_result = MagicMock()
161 stat_result.st_size = MAX_MSGPACK_BYTES + 1
162 return stat_result
163
164 def test_read_commit_checks_stat_before_read_bytes(
165 self, tmp_path: pathlib.Path
166 ) -> None:
167 root = _repo(tmp_path)
168 c = _commit(0)
169 write_commit(root, c)
170
171 commit_path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
172 real_stat = commit_path.stat # preserve reference
173
174 with patch.object(type(commit_path), "stat") as mock_stat:
175 mock_stat.return_value = self._oversized_stat(commit_path)
176 read_bytes_called = [False]
177 real_read_bytes = commit_path.read_bytes
178
179 def tracking_read_bytes() -> bytes:
180 read_bytes_called[0] = True
181 return real_read_bytes()
182
183 with patch.object(type(commit_path), "read_bytes", tracking_read_bytes):
184 result = read_commit(root, c.commit_id)
185
186 assert result is None, "read_commit should return None for oversized file"
187 assert not read_bytes_called[0], (
188 "read_bytes() was called BEFORE the stat size check — OOM risk!"
189 )
190
191 def test_read_snapshot_checks_stat_before_read_bytes(
192 self, tmp_path: pathlib.Path
193 ) -> None:
194 root = _repo(tmp_path)
195 s = _snapshot(0)
196 write_snapshot(root, s)
197
198 snap_path = root / ".muse" / "snapshots" / f"{s.snapshot_id}.msgpack"
199 read_bytes_called = [False]
200 real_read_bytes = snap_path.read_bytes
201
202 def tracking_read_bytes() -> bytes:
203 read_bytes_called[0] = True
204 return real_read_bytes()
205
206 with patch.object(type(snap_path), "stat") as mock_stat:
207 mock_stat.return_value = self._oversized_stat(snap_path)
208 with patch.object(type(snap_path), "read_bytes", tracking_read_bytes):
209 result = read_snapshot(root, s.snapshot_id)
210
211 assert result is None
212 assert not read_bytes_called[0], (
213 "read_bytes() was called before the stat size check in read_snapshot"
214 )
215
216
217 # ---------------------------------------------------------------------------
218 # Tier 2 — high-level read functions return None for oversized files
219 # ---------------------------------------------------------------------------
220
221 class TestReadFunctionsReturnNoneOnOversize:
222 """All public read functions must gracefully handle oversized files.
223
224 We patch MAX_MSGPACK_BYTES to a small value so we can create real files
225 that exceed it without writing gigabytes to disk.
226 """
227
228 def _write_oversized_commit(
229 self, root: pathlib.Path, c: CommitRecord, limit: int
230 ) -> None:
231 """Write a commit, then inflate the file beyond *limit* bytes."""
232 write_commit(root, c)
233 path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
234 # Overwrite with limit+1 bytes of valid-looking (but unparseable) data.
235 path.write_bytes(b"\x00" * (limit + 1))
236
237 def test_read_commit_returns_none_for_oversized_file(
238 self, tmp_path: pathlib.Path
239 ) -> None:
240 root = _repo(tmp_path)
241 c = _commit(1)
242 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
243 self._write_oversized_commit(root, c, 100)
244 result = read_commit(root, c.commit_id)
245 assert result is None, "read_commit must return None, not raise, for oversized file"
246
247 def test_read_snapshot_returns_none_for_oversized_file(
248 self, tmp_path: pathlib.Path
249 ) -> None:
250 root = _repo(tmp_path)
251 s = _snapshot(1)
252 write_snapshot(root, s)
253 snap_path = root / ".muse" / "snapshots" / f"{s.snapshot_id}.msgpack"
254 snap_path.write_bytes(b"\x00" * 101)
255 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
256 result = read_snapshot(root, s.snapshot_id)
257 assert result is None
258
259 def test_get_all_tags_skips_oversized_files(
260 self, tmp_path: pathlib.Path
261 ) -> None:
262 """get_all_tags iterates all tag files — oversized ones are skipped."""
263 root = _repo(tmp_path)
264 good = _tag(0)
265 bad = _tag(1)
266 write_tag(root, good)
267 write_tag(root, bad)
268
269 # A real tag record is ~200 bytes packed (64-char IDs + timestamp).
270 # Choose a limit above a real tag but below our inflated bad file.
271 good_path = root / ".muse" / "tags" / "test-repo" / f"{good.tag_id}.msgpack"
272 real_size = good_path.stat().st_size
273 test_limit = real_size * 2 # real tag fits; we'll inflate the bad tag to 3×
274
275 bad_path = root / ".muse" / "tags" / "test-repo" / f"{bad.tag_id}.msgpack"
276 bad_path.write_bytes(b"\x00" * (real_size * 3)) # definitely exceeds limit
277
278 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
279 tags = get_all_tags(root, "test-repo")
280 tag_ids = {t.tag_id for t in tags}
281 assert good.tag_id in tag_ids, "Good tag was incorrectly dropped"
282 assert bad.tag_id not in tag_ids, "Oversized tag was not skipped"
283
284 def test_list_releases_skips_oversized_files(
285 self, tmp_path: pathlib.Path
286 ) -> None:
287 """list_releases must skip oversized release files."""
288 root = _repo(tmp_path)
289 releases_dir = root / ".muse" / "releases" / "test-repo"
290 releases_dir.mkdir(parents=True)
291 # Write a fake oversized release file.
292 fake_release = releases_dir / f"{'a' * 64}.msgpack"
293 fake_release.write_bytes(b"\x00" * 101)
294 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
295 results = list_releases(root, "test-repo")
296 assert results == [], "Oversized release should be skipped, not crash"
297
298
299 # ---------------------------------------------------------------------------
300 # Tier 3 — exact boundary behaviour
301 # ---------------------------------------------------------------------------
302
303 class TestExactBoundary:
304 """At the boundary: MAX_MSGPACK_BYTES is the last allowed size."""
305
306 def test_file_exactly_at_limit_is_read(self, tmp_path: pathlib.Path) -> None:
307 """A file of exactly MAX_MSGPACK_BYTES bytes passes the size check.
308
309 The content may be unparseable (zeros are not valid msgpack), but the
310 OSError raised is a parse error, not a size-limit error.
311 """
312 test_limit = 256 # small limit for test speed
313 path = tmp_path / "exactly_at_limit.msgpack"
314 path.write_bytes(b"\x00" * test_limit)
315 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
316 # Should raise a parse error (invalid msgpack), NOT an OSError about size.
317 from muse.core.store import _read_msgpack
318 try:
319 _read_msgpack(path)
320 pytest.fail("Expected an error for invalid msgpack content")
321 except OSError as exc:
322 assert "MiB read limit" not in str(exc), (
323 f"Got size-limit OSError at the boundary — should be parse error: {exc}"
324 )
325 except Exception:
326 pass # Any non-size-limit error is acceptable here
327
328 def test_file_one_byte_over_limit_raises_oslimit_error(
329 self, tmp_path: pathlib.Path
330 ) -> None:
331 """A file of MAX_MSGPACK_BYTES + 1 bytes raises OSError before reading."""
332 test_limit = 256
333 path = tmp_path / "one_over.msgpack"
334 path.write_bytes(b"\x00" * (test_limit + 1))
335 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
336 from muse.core.store import _read_msgpack
337 with pytest.raises(OSError, match="read limit"):
338 _read_msgpack(path)
339
340 def test_zero_byte_file_does_not_trigger_size_limit(
341 self, tmp_path: pathlib.Path
342 ) -> None:
343 """An empty file passes the size check but fails msgpack parse."""
344 path = tmp_path / "empty.msgpack"
345 path.write_bytes(b"")
346 from muse.core.store import _read_msgpack
347 with pytest.raises(Exception): # parse error, not size error
348 _read_msgpack(path)
349
350 def test_size_limit_error_message_includes_filename_and_limit(
351 self, tmp_path: pathlib.Path
352 ) -> None:
353 """The OSError message must include the file name and limit in MiB."""
354 test_limit = 1024 # 1 KiB for test speed
355 path = tmp_path / "big.msgpack"
356 path.write_bytes(b"\x00" * (test_limit + 1))
357 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
358 from muse.core.store import _read_msgpack
359 with pytest.raises(OSError) as exc_info:
360 _read_msgpack(path)
361 msg = str(exc_info.value)
362 assert "big.msgpack" in msg, f"Filename missing from error: {msg}"
363 assert "KiB" in msg or "MiB" in msg or "bytes" in msg, (
364 f"Size info missing from error: {msg}"
365 )
366
367
368 # ---------------------------------------------------------------------------
369 # Tier 4 — per-value unpack limits
370 # ---------------------------------------------------------------------------
371
372 class TestPerValueUnpackLimits:
373 """Verify that per-value limits from msgpack.unpackb are enforced."""
374
375 def _pack_to_path(self, tmp_path: pathlib.Path, data: MsgpackValue) -> pathlib.Path:
376 path = tmp_path / "test.msgpack"
377 path.write_bytes(msgpack.packb(data, use_bin_type=True))
378 return path
379
380 def test_string_exceeding_max_str_len_rejected(self, tmp_path: pathlib.Path) -> None:
381 """A string longer than _MSGPACK_MAX_STR_LEN must raise an exception."""
382 huge_str = "x" * 200
383 path = self._pack_to_path(tmp_path, {"key": huge_str})
384 from muse.core.store import _read_msgpack
385 with patch("muse.core.store._MSGPACK_MAX_STR_LEN", 100):
386 with pytest.raises(Exception):
387 _read_msgpack(path)
388
389 def test_string_within_max_str_len_accepted(self, tmp_path: pathlib.Path) -> None:
390 """A string within the limit unpacks normally."""
391 path = self._pack_to_path(tmp_path, {"key": "short"})
392 from muse.core.store import _read_msgpack
393 result = _read_msgpack(path)
394 assert isinstance(result, dict)
395
396 def test_binary_blob_rejected_in_store_records(self, tmp_path: pathlib.Path) -> None:
397 """Binary data (msgpack bin type) must be rejected for store records.
398
399 Commit/snapshot/tag records contain no binary fields. A file with
400 binary data is either corrupt or tampered. max_bin_len=0 ensures
401 this is caught immediately during unpack rather than producing a
402 ``bytes`` value that callers are not prepared to handle.
403 """
404 path = self._pack_to_path(tmp_path, {"body": b"some binary blob"})
405 from muse.core.store import _read_msgpack
406 # max_bin_len=0 means any bin-type value raises an error.
407 with pytest.raises(Exception):
408 _read_msgpack(path)
409
410 def test_map_exceeding_max_map_len_rejected(self, tmp_path: pathlib.Path) -> None:
411 """A map with more than _MSGPACK_MAX_MAP_LEN entries must raise."""
412 big_map: MsgpackDict = {str(i): i for i in range(200)}
413 path = self._pack_to_path(tmp_path, big_map)
414 from muse.core.store import _read_msgpack
415 with patch("muse.core.store._MSGPACK_MAX_MAP_LEN", 100):
416 with pytest.raises(Exception):
417 _read_msgpack(path)
418
419 def test_array_exceeding_max_array_len_rejected(self, tmp_path: pathlib.Path) -> None:
420 """An array with more than _MSGPACK_MAX_ARRAY_LEN entries must raise."""
421 big_list: list[MsgpackValue] = list(range(200))
422 path = self._pack_to_path(tmp_path, big_list)
423 from muse.core.store import _read_msgpack
424 with patch("muse.core.store._MSGPACK_MAX_ARRAY_LEN", 100):
425 with pytest.raises(Exception):
426 _read_msgpack(path)
427
428 def _make_deep_nested_msgpack(self, depth: int) -> bytes:
429 """Build msgpack bytes for a *depth*-deep nested dict without Python recursion.
430
431 ``msgpack.packb`` uses Python-level recursion so packing a 600-deep
432 dict hits the default recursion limit. We build the bytes directly:
433
434 fixmap(1) fixstr("x") fixmap(1) fixstr("x") ... fixmap(0)
435
436 Each level is 3 bytes: ``0x81`` (fixmap 1 entry) + ``0xa1 0x78``
437 (fixstr "x"). The leaf is ``0x80`` (fixmap 0 entries).
438
439 This produces a valid msgpack binary that ``unpackb`` will parse up
440 to its stack limit and then raise ``StackError``.
441 """
442 # 0x81 = fixmap with 1 item; 0xa1 0x78 = fixstr "x"
443 frame = b"\x81\xa1x"
444 leaf = b"\x80" # fixmap with 0 items
445 return frame * depth + leaf
446
447 def test_deeply_nested_map_raises_stack_error(self, tmp_path: pathlib.Path) -> None:
448 """A pathologically nested document hits msgpack's StackError.
449
450 At extreme depth (10 000 levels), msgpack's C-extension stack limit is
451 exceeded and an exception is raised. The file is only ~30 KiB so the
452 size check passes; the protection comes from msgpack's internal stack
453 guard, not the 64 MiB cap.
454 """
455 packed = self._make_deep_nested_msgpack(10_000)
456 path = tmp_path / "deep_nest.msgpack"
457 path.write_bytes(packed)
458 from muse.core.store import _read_msgpack
459 with pytest.raises(Exception): # msgpack.exceptions.StackError
460 _read_msgpack(path)
461
462 def test_deeply_nested_terminates_quickly(self, tmp_path: pathlib.Path) -> None:
463 """The StackError for deeply nested documents is raised in < 1 second."""
464 packed = self._make_deep_nested_msgpack(10_000)
465 path = tmp_path / "deep_nest_perf.msgpack"
466 path.write_bytes(packed)
467 from muse.core.store import _read_msgpack
468 start = time.perf_counter()
469 try:
470 _read_msgpack(path)
471 except Exception:
472 pass
473 elapsed = time.perf_counter() - start
474 assert elapsed < 1.0, (
475 f"Deeply nested document took {elapsed:.3f}s to fail — not fast enough"
476 )
477
478 def test_valid_large_map_within_limits_is_accepted(self, tmp_path: pathlib.Path) -> None:
479 """A large but within-limit map (simulating a 1k-file snapshot) unpacks cleanly."""
480 # Simulate a 1000-file snapshot manifest: {path: object_id}
481 manifest = {f"src/file_{i:04d}.py": _sha(f"obj-{i}") for i in range(1000)}
482 path = tmp_path / "big_valid.msgpack"
483 path.write_bytes(msgpack.packb(manifest, use_bin_type=True))
484 from muse.core.store import _read_msgpack
485 result = _read_msgpack(path)
486 assert isinstance(result, dict)
487 assert len(result) == 1000
488
489
490 # ---------------------------------------------------------------------------
491 # Tier 5 — index file protection
492 # ---------------------------------------------------------------------------
493
494 class TestIndexReadProtection:
495 """muse/core/indices.py has its own _read_msgpack — must also be protected."""
496
497 def test_load_symbol_history_skips_oversized_index(
498 self, tmp_path: pathlib.Path
499 ) -> None:
500 """An oversized symbol history index returns an empty dict, not OOM."""
501 (tmp_path / ".muse" / "indices").mkdir(parents=True)
502 index_path = tmp_path / ".muse" / "indices" / "symbol_history.msgpack"
503 index_path.write_bytes(b"\x00" * 101)
504 with patch("muse.core.indices._MAX_INDEX_BYTES", 100):
505 result = load_symbol_history(tmp_path)
506 assert result == {}, "Oversized index must return empty dict, not crash"
507
508 def test_load_hash_occurrence_skips_oversized_index(
509 self, tmp_path: pathlib.Path
510 ) -> None:
511 """An oversized hash_occurrence index returns an empty dict."""
512 (tmp_path / ".muse" / "indices").mkdir(parents=True)
513 index_path = tmp_path / ".muse" / "indices" / "hash_occurrence.msgpack"
514 index_path.write_bytes(b"\x00" * 101)
515 with patch("muse.core.indices._MAX_INDEX_BYTES", 100):
516 result = load_hash_occurrence(tmp_path)
517 assert result == {}
518
519 def test_index_size_limit_is_more_generous_than_store(self) -> None:
520 """Index files are allowed to be larger than store records."""
521 from muse.core.indices import _MAX_INDEX_BYTES
522 assert _MAX_INDEX_BYTES > MAX_MSGPACK_BYTES, (
523 "Index limit should be larger than store limit — indices grow with repo size"
524 )
525
526 def test_index_read_checks_stat_before_read_bytes(
527 self, tmp_path: pathlib.Path
528 ) -> None:
529 """The index stat check must fire before read_bytes (no allocation)."""
530 (tmp_path / ".muse" / "indices").mkdir(parents=True)
531 index_path = tmp_path / ".muse" / "indices" / "symbol_history.msgpack"
532 index_path.write_bytes(b"\x85") # 1 byte — well within any size limit
533 read_bytes_called = [False]
534 real_rb = index_path.read_bytes
535
536 def tracking_rb() -> bytes:
537 read_bytes_called[0] = True
538 return real_rb()
539
540 stat_result = MagicMock()
541 stat_result.st_size = 1024 * 1024 * 1024 # 1 GiB — way over limit
542
543 with patch.object(type(index_path), "stat", return_value=stat_result):
544 with patch.object(type(index_path), "read_bytes", tracking_rb):
545 result = load_symbol_history(tmp_path)
546
547 assert result == {}
548 assert not read_bytes_called[0], "read_bytes was called before the stat check!"
549
550
551 # ---------------------------------------------------------------------------
552 # Tier 6 — warning log on oversized file
553 # ---------------------------------------------------------------------------
554
555 class TestWarningLogOnOversizedFile:
556 """Operators need to know when oversized files are detected.
557
558 read_commit / read_snapshot log a WARNING when they catch the OSError
559 from _read_msgpack — this surfaces corruption or tampering in monitoring.
560 """
561
562 def test_warning_logged_for_oversized_commit(
563 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
564 ) -> None:
565 root = _repo(tmp_path)
566 c = _commit(10)
567 with patch("muse.core.store.MAX_MSGPACK_BYTES", 50):
568 path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
569 path.write_bytes(b"\x00" * 51)
570 with caplog.at_level(logging.WARNING, logger="muse.core.store"):
571 result = read_commit(root, c.commit_id)
572 assert result is None
573 # A warning must have been emitted for the corrupt/oversized file.
574 assert any(
575 "Corrupt" in rec.message or "corrupt" in rec.message
576 or "oversized" in rec.message or "limit" in rec.message.lower()
577 for rec in caplog.records
578 ), f"No warning logged for oversized commit. Records: {[r.message for r in caplog.records]}"
579
580 def test_warning_logged_for_oversized_snapshot(
581 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
582 ) -> None:
583 root = _repo(tmp_path)
584 s = _snapshot(10)
585 with patch("muse.core.store.MAX_MSGPACK_BYTES", 50):
586 path = root / ".muse" / "snapshots" / f"{s.snapshot_id}.msgpack"
587 path.write_bytes(b"\x00" * 51)
588 with caplog.at_level(logging.WARNING, logger="muse.core.store"):
589 result = read_snapshot(root, s.snapshot_id)
590 assert result is None
591 assert any(
592 "Corrupt" in rec.message or "corrupt" in rec.message
593 for rec in caplog.records
594 ), f"No warning logged. Records: {[r.message for r in caplog.records]}"
595
596
597 # ---------------------------------------------------------------------------
598 # Tier 7 — CLI plumbing: clean JSON error, no traceback
599 # ---------------------------------------------------------------------------
600
601 class TestPlumbingReadCommitOversized:
602 """muse plumbing read-commit with an oversized commit file must produce
603 a clean, machine-readable JSON error — no Python traceback, no process crash.
604 """
605
606 def test_oversized_commit_produces_json_error_not_traceback(
607 self, tmp_path: pathlib.Path
608 ) -> None:
609 """write a commit, corrupt its file, run read-commit — must get JSON error."""
610 import json
611 import sys
612 from tests.cli_test_helper import CliRunner
613
614 root = _repo(tmp_path)
615 c = _commit(99)
616 write_commit(root, c)
617
618 # Corrupt the commit file to exceed the limit.
619 commit_path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
620 commit_path.write_bytes(b"\x00" * 101)
621
622 runner = CliRunner()
623 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
624 result = runner.invoke(None, ["read-commit", c.commit_id],
625 env={"MUSE_REPO_ROOT": str(root)})
626
627 # Must not crash (exit code may be non-zero, but not a Python traceback).
628 assert "Traceback" not in (result.output or ""), (
629 f"CLI produced a Python traceback for oversized commit:\n{result.output}"
630 )
631 assert "Traceback" not in (result.stderr or ""), (
632 f"CLI stderr has a Python traceback:\n{result.stderr}"
633 )
634 # The error output must be valid JSON (or include a meaningful error).
635 combined = (result.output or "") + (result.stderr or "")
636 try:
637 # Check if any JSON blob exists in the output.
638 for line in combined.splitlines():
639 line = line.strip()
640 if line.startswith("{"):
641 parsed = json.loads(line)
642 assert "error" in parsed, f"JSON lacks 'error' key: {parsed}"
643 break
644 else:
645 # If no JSON line found, at minimum confirm no traceback and
646 # that "not found" or "error" appears in the output.
647 assert (
648 "not found" in combined.lower()
649 or "error" in combined.lower()
650 ), f"No useful error in CLI output:\n{combined}"
651 except json.JSONDecodeError as exc:
652 pytest.fail(f"Output is not valid JSON: {exc}\nOutput:\n{combined}")
653
654
655 # ---------------------------------------------------------------------------
656 # Tier 8 — round-trip: valid files still read correctly
657 # ---------------------------------------------------------------------------
658
659 class TestValidFilesUnaffected:
660 """The size guard must not regress normal reads."""
661
662 def test_read_commit_roundtrip_unaffected(self, tmp_path: pathlib.Path) -> None:
663 root = _repo(tmp_path)
664 c = _commit(42)
665 write_commit(root, c)
666 got = read_commit(root, c.commit_id)
667 assert got is not None
668 assert got.commit_id == c.commit_id
669 assert got.message == c.message
670
671 def test_read_snapshot_roundtrip_unaffected(self, tmp_path: pathlib.Path) -> None:
672 root = _repo(tmp_path)
673 s = _snapshot(42)
674 write_snapshot(root, s)
675 got = read_snapshot(root, s.snapshot_id)
676 assert got is not None
677 assert got.snapshot_id == s.snapshot_id
678
679 def test_snapshot_with_large_manifest_reads_correctly(
680 self, tmp_path: pathlib.Path
681 ) -> None:
682 """A 1000-file snapshot manifest (realistic scale) reads without issue."""
683 root = _repo(tmp_path)
684 manifest = {f"src/file_{i:05d}.py": _sha(f"obj-{i}") for i in range(1000)}
685 sid = compute_snapshot_id(manifest)
686 s = SnapshotRecord(
687 snapshot_id=sid,
688 manifest=manifest,
689 )
690 write_snapshot(root, s)
691 got = read_snapshot(root, sid)
692 assert got is not None
693 assert len(got.manifest) == 1000
694
695 def test_commit_with_long_message_reads_correctly(
696 self, tmp_path: pathlib.Path
697 ) -> None:
698 """A commit with a 64 KiB message reads correctly (well within 1 MiB str limit)."""
699 root = _repo(tmp_path)
700 long_msg = "a" * 65536
701 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
702 snapshot_id = compute_snapshot_id({})
703 cid = compute_commit_id([], snapshot_id, long_msg, committed_at.isoformat())
704 c = CommitRecord(
705 commit_id=cid,
706 repo_id="test-repo",
707 branch="main",
708 snapshot_id=snapshot_id,
709 message=long_msg,
710 committed_at=committed_at,
711 author="tester",
712 parent_commit_id=None,
713 parent2_commit_id=None,
714 )
715 write_commit(root, c)
716 got = read_commit(root, cid)
717 assert got is not None
718 assert len(got.message) == 65536
719
720
721 # ---------------------------------------------------------------------------
722 # Tier 9 — performance: size check adds < 1 ms per read
723 # ---------------------------------------------------------------------------
724
725 class TestSizeCheckPerformance:
726 """The stat() check should add negligible overhead to normal reads."""
727
728 @pytest.mark.perf
729 def test_stat_check_overhead_under_1ms_per_read(
730 self, tmp_path: pathlib.Path
731 ) -> None:
732 """100 sequential read_commit calls with the size guard active < 100ms total."""
733 root = _repo(tmp_path)
734 commits = [_commit(i) for i in range(100)]
735 for c in commits:
736 write_commit(root, c)
737
738 start = time.perf_counter()
739 for c in commits:
740 result = read_commit(root, c.commit_id)
741 assert result is not None
742 elapsed = time.perf_counter() - start
743
744 assert elapsed < 0.1, (
745 f"100 read_commit calls took {elapsed:.3f}s — "
746 "size check is adding too much overhead (< 100ms expected)"
747 )
748
749 @pytest.mark.perf
750 def test_oversized_rejection_under_1ms(self, tmp_path: pathlib.Path) -> None:
751 """Rejecting an oversized file (via stat) takes < 1ms — no disk I/O."""
752 root = _repo(tmp_path)
753 c = _commit(200)
754 write_commit(root, c)
755 path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
756 path.write_bytes(b"\x00" * 101)
757
758 start = time.perf_counter()
759 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
760 for _ in range(1000):
761 read_commit(root, c.commit_id)
762 elapsed = time.perf_counter() - start
763
764 assert elapsed < 1.0, (
765 f"1000 oversized-rejection calls took {elapsed:.3f}s (> 1ms each)"
766 )
File History 5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 27 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 27 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e Merge branch 'dev' into main Human 29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce Merge branch 'dev' into main Human 48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 101 days ago