gabriel / muse public
test_integrity_I8_object_store_scale.py python
856 lines 32.9 KB
Raw
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 74 days ago
1 """I-8: Object store at Linux scale.
2
3 Scenario: 850 000 commits × ~20 objects per commit = 17 million objects.
4 2-char sharding → 256 shards × ~66 000 files each. On Linux ext4 (and
5 macOS APFS) directory entries above ~100 000 per directory trigger visible
6 lookup degradation. This suite proves:
7
8 1. File mode 0o444 — every new object is written read-only.
9 2. Stale temp cleanup — .obj-tmp-* files from a prior crash are removed.
10 3. has_object O(log n) lookup — timing at 1k / 10k / 100k objects proves
11 sub-linear growth (ext4 / APFS use hash-tree / B-tree indexing).
12 4. 4-char sharding — 65 536 shards; object path layout changes correctly.
13 5. Configurable via [limits] shard_prefix_length in config.toml.
14 6. Dual-lookup / migration — objects written at 2-char prefix are still
15 found after switching config to 4-char.
16 7. shard_prefix_length=4 reflected in get_config_value and get_limit.
17 8. Robustness — invalid shard_prefix_length values are ignored.
18 9. Permission enforcement — direct write to a 0o444 object raises
19 PermissionError, confirming the OS-level immutability guard.
20 10. Shard count correctness — 4-char yields 65 536 possible shards.
21 11. cleanup_stale_object_temps is idempotent (double-call safe).
22 12. _object_path_with_fallback returns primary path when it exists.
23 """
24
25 from __future__ import annotations
26
27 import hashlib
28 import os
29 import pathlib
30 import stat
31 import time
32 import tomllib
33
34 import pytest
35
36 from muse.core.object_store import (
37 _object_path_with_fallback,
38 cleanup_stale_object_temps,
39 has_object,
40 object_path,
41 objects_dir,
42 read_object,
43 restore_object,
44 write_object,
45 write_object_from_path,
46 _OBJECT_MODE,
47 _DEFAULT_SHARD_PREFIX_LEN,
48 _VALID_SHARD_PREFIX_LENS,
49 )
50 from muse.cli.config import get_limit, get_config_value
51 from muse.core._types import Manifest
52
53
54 # ---------------------------------------------------------------------------
55 # Helpers
56 # ---------------------------------------------------------------------------
57
58
59 def _sha256(data: bytes) -> str:
60 return hashlib.sha256(data).hexdigest()
61
62
63 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
64 (tmp_path / ".muse").mkdir()
65 return tmp_path
66
67
68 def _write_config(repo: pathlib.Path, shard_prefix_length: int) -> None:
69 """Write a minimal .muse/config.toml with [limits] shard_prefix_length."""
70 config_text = (
71 "[core]\nbranch = \"main\"\n\n"
72 f"[limits]\nshard_prefix_length = {shard_prefix_length}\n"
73 )
74 (repo / ".muse" / "config.toml").write_text(config_text, encoding="utf-8")
75
76
77 # ---------------------------------------------------------------------------
78 # 0. Regression: restore_object must NOT propagate 0o444 to working tree
79 # ---------------------------------------------------------------------------
80
81
82 class TestRestoreObjectMode:
83 """Regression test for: stored objects are 0o444 (immutable); restore_object
84 must produce 0o644 working-tree files so they remain editable.
85
86 Root cause: shutil.copy2 copies permissions from the src (stored object).
87 After I-8 introduced 0o444 on stored objects, restore_object was producing
88 read-only working-tree files, silently freezing them. This class was added
89 to pin the fix and prevent recurrence.
90 """
91
92 def test_restore_object_produces_0o644_file(
93 self, tmp_path: pathlib.Path
94 ) -> None:
95 """restore_object must write working-tree files with mode 0o644.
96
97 Stored objects are 0o444; working-tree files must be 0o644 so users
98 and agents can edit them without a manual chmod.
99 """
100 repo = _repo(tmp_path)
101 data = b"content that will be restored to working tree"
102 oid = _sha256(data)
103 write_object(repo, oid, data)
104
105 dest = tmp_path / "restored.txt"
106 assert restore_object(repo, oid, dest)
107
108 mode = stat.S_IMODE(dest.stat().st_mode)
109 assert mode == 0o644, (
110 f"restore_object produced mode {oct(mode)} — working-tree files "
111 f"must be 0o644 so they are editable. "
112 f"(Stored object is 0o444; shutil.copy2 must not propagate that mode.)"
113 )
114
115 def test_stored_object_is_0o444_but_restore_is_0o644(
116 self, tmp_path: pathlib.Path
117 ) -> None:
118 """The stored object is 0o444 while the restored file is 0o644.
119
120 This is the invariant: objects are immutable in the store, writable
121 in the working tree.
122 """
123 repo = _repo(tmp_path)
124 data = b"immutable in store, writable in tree"
125 oid = _sha256(data)
126 write_object(repo, oid, data)
127
128 stored_mode = stat.S_IMODE(object_path(repo, oid).stat().st_mode)
129 assert stored_mode == 0o444, f"Stored object should be 0o444, got {oct(stored_mode)}"
130
131 dest = tmp_path / "workdir" / "file.txt"
132 restore_object(repo, oid, dest)
133 restored_mode = stat.S_IMODE(dest.stat().st_mode)
134 assert restored_mode == 0o644, (
135 f"Restored working-tree file should be 0o644, got {oct(restored_mode)}"
136 )
137
138 def test_restore_object_content_intact_after_mode_fix(
139 self, tmp_path: pathlib.Path
140 ) -> None:
141 """Content must be byte-identical after the chmod fix — no data loss."""
142 repo = _repo(tmp_path)
143 data = b"content integrity check after mode fix" * 50
144 oid = _sha256(data)
145 write_object(repo, oid, data)
146
147 dest = tmp_path / "check.bin"
148 restore_object(repo, oid, dest)
149 assert dest.read_bytes() == data
150
151 def test_restore_large_object_is_0o644(self, tmp_path: pathlib.Path) -> None:
152 """Large blobs (shutil.copy2 path) also restore as 0o644."""
153 repo = _repo(tmp_path)
154 data = os.urandom(512 * 1024) # 512 KiB
155 oid = _sha256(data)
156 src = tmp_path / "large.bin"
157 src.write_bytes(data)
158 write_object_from_path(repo, oid, src)
159
160 dest = tmp_path / "large_restored.bin"
161 restore_object(repo, oid, dest)
162 mode = stat.S_IMODE(dest.stat().st_mode)
163 assert mode == 0o644, (
164 f"Large blob restore produced mode {oct(mode)}, expected 0o644"
165 )
166
167
168 # ---------------------------------------------------------------------------
169 # 1. File mode 0o444 — immutability enforced at the OS level
170 # ---------------------------------------------------------------------------
171
172
173 class TestObjectMode:
174 def test_write_object_produces_0o444_file(self, tmp_path: pathlib.Path) -> None:
175 """Every blob written by write_object must be mode 0o444."""
176 repo = _repo(tmp_path)
177 data = b"immutable content"
178 oid = _sha256(data)
179 write_object(repo, oid, data)
180 p = object_path(repo, oid)
181 mode = stat.S_IMODE(p.stat().st_mode)
182 assert mode == 0o444, (
183 f"Object {oid[:8]} was written with mode {oct(mode)} instead of 0o444. "
184 "Content-addressed objects must be read-only."
185 )
186
187 def test_write_object_from_path_produces_0o444_file(
188 self, tmp_path: pathlib.Path
189 ) -> None:
190 """write_object_from_path (large-blob path) must also produce 0o444."""
191 repo = _repo(tmp_path)
192 data = b"large blob via path" * 100
193 oid = _sha256(data)
194 src = tmp_path / "src.bin"
195 src.write_bytes(data)
196 write_object_from_path(repo, oid, src)
197 p = object_path(repo, oid)
198 mode = stat.S_IMODE(p.stat().st_mode)
199 assert mode == 0o444, (
200 f"write_object_from_path produced mode {oct(mode)} instead of 0o444."
201 )
202
203 def test_object_mode_constant(self) -> None:
204 """_OBJECT_MODE must equal 0o444 — no accidental changes."""
205 assert _OBJECT_MODE == 0o444
206
207 def test_write_then_read_respects_mode(self, tmp_path: pathlib.Path) -> None:
208 """Round-trip: content can be read back even though the file is 0o444."""
209 repo = _repo(tmp_path)
210 data = b"read-only but readable"
211 oid = _sha256(data)
212 write_object(repo, oid, data)
213 assert read_object(repo, oid) == data
214
215 def test_direct_overwrite_blocked_by_os(self, tmp_path: pathlib.Path) -> None:
216 """Opening a 0o444 object for writing must raise PermissionError.
217
218 This is the OS-level immutability guarantee: even a bug that calls
219 open(path, 'wb') on a stored object is caught before any bytes are
220 written.
221 """
222 repo = _repo(tmp_path)
223 data = b"must not be overwritten"
224 oid = _sha256(data)
225 write_object(repo, oid, data)
226 p = object_path(repo, oid)
227 with pytest.raises(PermissionError):
228 p.write_bytes(b"attacker-controlled content")
229 # Content must be intact.
230 assert read_object(repo, oid) == data
231
232 def test_multiple_objects_all_0o444(self, tmp_path: pathlib.Path) -> None:
233 """Batch write: every object file must be 0o444."""
234 repo = _repo(tmp_path)
235 for i in range(50):
236 data = f"batch-object-{i}".encode()
237 oid = _sha256(data)
238 write_object(repo, oid, data)
239 for shard in objects_dir(repo).iterdir():
240 for f in shard.iterdir():
241 mode = stat.S_IMODE(f.stat().st_mode)
242 assert mode == 0o444, f"{f.name} has mode {oct(mode)}, expected 0o444"
243
244
245 # ---------------------------------------------------------------------------
246 # 2. Stale temp cleanup
247 # ---------------------------------------------------------------------------
248
249
250 def _make_stale(path: pathlib.Path, content: bytes = b"stale") -> None:
251 """Write *path* and backdate its mtime past the age gate.
252
253 cleanup_stale_object_temps only removes files older than
254 _CLEANUP_MIN_AGE_SECS (60 s). Tests that create temp files and
255 immediately call cleanup would always return 0 without this helper.
256 Setting mtime to the Unix epoch (1970-01-01) makes every freshly-created
257 temp file look decades old to the cleanup function.
258 """
259 path.write_bytes(content)
260 os.utime(path, (0, 0)) # atime=0, mtime=0 → epoch → age > 60 s
261
262
263 class TestStaleTempCleanup:
264 def test_cleanup_removes_obj_tmp_files(self, tmp_path: pathlib.Path) -> None:
265 """cleanup_stale_object_temps removes .obj-tmp-* files from shard dirs."""
266 repo = _repo(tmp_path)
267 store = objects_dir(repo)
268 shard = store / "ab"
269 shard.mkdir(parents=True)
270 stale = shard / ".obj-tmp-crash"
271 _make_stale(stale, b"partial write from prior SIGKILL")
272 assert stale.exists()
273
274 removed = cleanup_stale_object_temps(repo)
275 assert removed == 1
276 assert not stale.exists()
277
278 def test_cleanup_removes_restore_tmp_files(self, tmp_path: pathlib.Path) -> None:
279 """cleanup_stale_object_temps also removes .restore-tmp-* files."""
280 repo = _repo(tmp_path)
281 store = objects_dir(repo)
282 shard = store / "cd"
283 shard.mkdir(parents=True)
284 stale = shard / ".restore-tmp-12345"
285 _make_stale(stale, b"partial restore")
286
287 removed = cleanup_stale_object_temps(repo)
288 assert removed == 1
289 assert not stale.exists()
290
291 def test_cleanup_preserves_real_objects(self, tmp_path: pathlib.Path) -> None:
292 """cleanup must not touch real object files."""
293 repo = _repo(tmp_path)
294 data = b"real object"
295 oid = _sha256(data)
296 write_object(repo, oid, data)
297
298 removed = cleanup_stale_object_temps(repo)
299 assert removed == 0
300 assert has_object(repo, oid)
301
302 def test_cleanup_nonexistent_store_returns_zero(
303 self, tmp_path: pathlib.Path
304 ) -> None:
305 """cleanup on a repo with no objects dir returns 0 without raising."""
306 repo = _repo(tmp_path)
307 # objects dir does not exist yet
308 removed = cleanup_stale_object_temps(repo)
309 assert removed == 0
310
311 def test_cleanup_is_idempotent(self, tmp_path: pathlib.Path) -> None:
312 """Calling cleanup twice is safe — second call returns 0."""
313 repo = _repo(tmp_path)
314 store = objects_dir(repo)
315 shard = store / "ef"
316 shard.mkdir(parents=True)
317 _make_stale(shard / ".obj-tmp-stale")
318
319 assert cleanup_stale_object_temps(repo) == 1
320 assert cleanup_stale_object_temps(repo) == 0
321
322 def test_cleanup_multiple_shards(self, tmp_path: pathlib.Path) -> None:
323 """Stale files in multiple shard dirs are all cleaned up."""
324 repo = _repo(tmp_path)
325 store = objects_dir(repo)
326 for prefix in ("00", "7f", "ff"):
327 shard = store / prefix
328 shard.mkdir(parents=True)
329 _make_stale(shard / f".obj-tmp-{prefix}")
330
331 removed = cleanup_stale_object_temps(repo)
332 assert removed == 3
333
334
335 # ---------------------------------------------------------------------------
336 # 3. has_object O(log n) performance — 1k / 10k / 100k files per shard
337 # ---------------------------------------------------------------------------
338
339
340 class TestHasObjectPerformance:
341 """Prove that has_object does not degrade to O(n).
342
343 ext4 and APFS use hash-tree / B-tree directory indexing so filename
344 lookup is O(log n). At n=100k the ratio to n=1k should be < 10×
345 (log2(100000) / log2(1000) ≈ 1.66× in theory; we allow 10× for
346 scheduler jitter).
347 """
348
349 def _populate_shard(
350 self, shard_dir: pathlib.Path, n: int
351 ) -> list[str]:
352 """Create n dummy files in *shard_dir* and return their names."""
353 shard_dir.mkdir(parents=True, exist_ok=True)
354 names: list[str] = []
355 for i in range(n):
356 name = hashlib.sha256(f"dummy-{i}".encode()).hexdigest()[:62]
357 p = shard_dir / name
358 p.write_bytes(b"x")
359 names.append(name)
360 return names
361
362 def _time_has_object(
363 self,
364 repo: pathlib.Path,
365 oid: str,
366 iterations: int = 200,
367 ) -> float:
368 """Return average has_object latency in milliseconds over *iterations*."""
369 # Warm up filesystem cache.
370 for _ in range(10):
371 has_object(repo, oid)
372 t0 = time.perf_counter()
373 for _ in range(iterations):
374 has_object(repo, oid)
375 elapsed = (time.perf_counter() - t0) / iterations * 1000
376 return elapsed
377
378 def test_has_object_under_10ms_at_100k_per_shard(
379 self, tmp_path: pathlib.Path
380 ) -> None:
381 """has_object lookup < 10 ms with 100 000 files in the target shard."""
382 repo = _repo(tmp_path)
383 # Use a fixed prefix so we know which shard to populate.
384 target_data = b"target-object-100k-test"
385 target_oid = _sha256(target_data)
386 prefix = target_oid[:2]
387
388 shard = objects_dir(repo) / prefix
389 # Populate the shard with 100k dummy files.
390 self._populate_shard(shard, 100_000)
391 # Write the real target object.
392 write_object(repo, target_oid, target_data)
393
394 avg_ms = self._time_has_object(repo, target_oid, iterations=100)
395 assert avg_ms < 10.0, (
396 f"has_object averaged {avg_ms:.3f} ms at 100k files per shard — "
397 f"exceeded 10 ms budget. Filesystem lookup may be O(n)."
398 )
399
400 def test_lookup_growth_is_sublinear(self, tmp_path: pathlib.Path) -> None:
401 """Lookup time at 10k files is < 5× time at 1k files (sub-linear proof)."""
402 repo = _repo(tmp_path)
403
404 # 1k shard
405 data1k = b"object-for-1k-test"
406 oid1k = _sha256(data1k)
407 prefix = oid1k[:2]
408 shard = objects_dir(repo) / prefix
409 self._populate_shard(shard, 1_000)
410 write_object(repo, oid1k, data1k)
411 time_1k = self._time_has_object(repo, oid1k, iterations=500)
412
413 # 10k shard (different repo so the shard is clean)
414 repo2_root = tmp_path / "repo2"
415 repo2_root.mkdir()
416 repo2 = _repo(repo2_root)
417 data10k = b"object-for-10k-test"
418 oid10k = _sha256(data10k)
419 prefix2 = oid10k[:2]
420 shard2 = objects_dir(repo2) / prefix2
421 self._populate_shard(shard2, 10_000)
422 write_object(repo2, oid10k, data10k)
423 time_10k = self._time_has_object(repo2, oid10k, iterations=500)
424
425 # Sub-linear: 10× more files should not take 10× longer.
426 ratio = time_10k / max(time_1k, 0.001)
427 assert ratio < 10.0, (
428 f"has_object at 10k took {time_10k:.3f} ms vs {time_1k:.3f} ms at 1k "
429 f"(ratio={ratio:.2f}×). Lookup appears O(n) — investigate filesystem."
430 )
431
432 def test_has_object_absent_is_fast(self, tmp_path: pathlib.Path) -> None:
433 """Negative lookup (object not present) is also fast at 100k per shard."""
434 repo = _repo(tmp_path)
435 # Any SHA-256 with a predictable prefix for shard control.
436 absent_data = b"this-object-will-not-be-written"
437 absent_oid = _sha256(absent_data)
438 prefix = absent_oid[:2]
439
440 shard = objects_dir(repo) / prefix
441 self._populate_shard(shard, 100_000)
442 # Do NOT write the absent object.
443
444 avg_ms = self._time_has_object(repo, absent_oid, iterations=100)
445 assert avg_ms < 10.0, (
446 f"Negative has_object averaged {avg_ms:.3f} ms at 100k files — "
447 f"exceeded 10 ms budget."
448 )
449
450
451 # ---------------------------------------------------------------------------
452 # 4 & 5. 4-char sharding — configurable via [limits] shard_prefix_length
453 # ---------------------------------------------------------------------------
454
455
456 class TestFourCharSharding:
457 def test_default_prefix_length_is_two(self, tmp_path: pathlib.Path) -> None:
458 """Default shard_prefix_length must be 2 (256 shards)."""
459 repo = _repo(tmp_path)
460 assert get_limit("shard_prefix_length", repo) == 2
461
462 def test_config_sets_prefix_length_to_four(self, tmp_path: pathlib.Path) -> None:
463 """[limits] shard_prefix_length = 4 is read correctly."""
464 repo = _repo(tmp_path)
465 _write_config(repo, 4)
466 assert get_limit("shard_prefix_length", repo) == 4
467
468 def test_object_path_uses_four_char_prefix(self, tmp_path: pathlib.Path) -> None:
469 """object_path with prefix_len=4 puts objects in 4-char shard dirs."""
470 repo = _repo(tmp_path)
471 oid = "abcd" + "1" * 60
472 p = object_path(repo, oid, prefix_len=4)
473 assert p.parent.name == "abcd"
474 assert p.name == "1" * 60
475
476 def test_object_path_default_still_two_char(self, tmp_path: pathlib.Path) -> None:
477 """Callers passing no prefix_len get the 2-char default."""
478 repo = _repo(tmp_path)
479 oid = "abcd" + "1" * 60
480 p = object_path(repo, oid)
481 assert p.parent.name == "ab"
482 assert p.name == "cd" + "1" * 60
483
484 def test_write_and_read_with_four_char_config(
485 self, tmp_path: pathlib.Path
486 ) -> None:
487 """Round-trip read/write works when config sets 4-char sharding."""
488 repo = _repo(tmp_path)
489 _write_config(repo, 4)
490 data = b"four char shard test"
491 oid = _sha256(data)
492 write_object(repo, oid, data)
493 # The object must be at a 4-char prefix path.
494 p = object_path(repo, oid, prefix_len=4)
495 assert p.exists(), f"Object not found at 4-char path: {p}"
496 assert read_object(repo, oid) == data
497
498 def test_four_char_object_is_0o444(self, tmp_path: pathlib.Path) -> None:
499 """Objects written under 4-char sharding still get mode 0o444."""
500 repo = _repo(tmp_path)
501 _write_config(repo, 4)
502 data = b"mode check in 4-char shard"
503 oid = _sha256(data)
504 write_object(repo, oid, data)
505 p = object_path(repo, oid, prefix_len=4)
506 mode = stat.S_IMODE(p.stat().st_mode)
507 assert mode == 0o444
508
509 def test_65536_shard_space(self) -> None:
510 """4-char hex prefix allows 16^4 = 65 536 shard directories."""
511 assert 16**4 == 65_536
512
513 def test_valid_shard_prefix_lens(self) -> None:
514 """_VALID_SHARD_PREFIX_LENS must contain exactly {2, 4}."""
515 assert _VALID_SHARD_PREFIX_LENS == frozenset({2, 4})
516
517 def test_default_shard_prefix_len_constant(self) -> None:
518 """_DEFAULT_SHARD_PREFIX_LEN must be 2."""
519 assert _DEFAULT_SHARD_PREFIX_LEN == 2
520
521 def test_invalid_shard_prefix_length_ignored(
522 self, tmp_path: pathlib.Path
523 ) -> None:
524 """shard_prefix_length values outside {2, 4} fall back to default 2."""
525 repo = _repo(tmp_path)
526 (repo / ".muse" / "config.toml").write_text(
527 "[limits]\nshard_prefix_length = 3\n", encoding="utf-8"
528 )
529 assert get_limit("shard_prefix_length", repo) == 2
530
531 def test_get_config_value_returns_shard_prefix_length(
532 self, tmp_path: pathlib.Path
533 ) -> None:
534 """get_config_value('limits.shard_prefix_length') reflects config."""
535 repo = _repo(tmp_path)
536 _write_config(repo, 4)
537 val = get_config_value("limits.shard_prefix_length", repo)
538 assert val == "4"
539
540 def test_get_config_value_absent_returns_none(
541 self, tmp_path: pathlib.Path
542 ) -> None:
543 """get_config_value returns None when shard_prefix_length is absent."""
544 repo = _repo(tmp_path)
545 val = get_config_value("limits.shard_prefix_length", repo)
546 assert val is None
547
548
549 # ---------------------------------------------------------------------------
550 # 6. Migration compatibility — dual-lookup fallback
551 # ---------------------------------------------------------------------------
552
553
554 class TestMigrationFallback:
555 def test_two_char_object_found_after_switching_to_four_char(
556 self, tmp_path: pathlib.Path
557 ) -> None:
558 """Objects written at 2-char prefix are still readable after switching to 4-char.
559
560 No migration of existing objects is required — the fallback lookup
561 transparently finds the old 2-char path.
562 """
563 repo = _repo(tmp_path)
564 # Write object with default (2-char) sharding.
565 data = b"written before shard upgrade"
566 oid = _sha256(data)
567 write_object(repo, oid, data)
568 assert object_path(repo, oid, prefix_len=2).exists()
569
570 # Now switch the config to 4-char.
571 _write_config(repo, 4)
572
573 # Object must still be readable.
574 assert has_object(repo, oid), "Object lost after shard config upgrade"
575 assert read_object(repo, oid) == data
576
577 def test_fallback_path_returns_two_char_when_primary_absent(
578 self, tmp_path: pathlib.Path
579 ) -> None:
580 """_object_path_with_fallback returns the 2-char path when 4-char is configured."""
581 repo = _repo(tmp_path)
582 data = b"fallback test"
583 oid = _sha256(data)
584 write_object(repo, oid, data) # written at 2-char
585
586 _write_config(repo, 4)
587 fallback_path = _object_path_with_fallback(repo, oid)
588 assert fallback_path == object_path(repo, oid, prefix_len=2)
589 assert fallback_path.exists()
590
591 def test_primary_path_preferred_over_fallback(
592 self, tmp_path: pathlib.Path
593 ) -> None:
594 """When object exists at 4-char path, primary path is returned."""
595 repo = _repo(tmp_path)
596 _write_config(repo, 4)
597 data = b"written at four-char shard"
598 oid = _sha256(data)
599 write_object(repo, oid, data) # written at 4-char (primary)
600
601 p = _object_path_with_fallback(repo, oid)
602 assert p == object_path(repo, oid, prefix_len=4)
603
604 def test_idempotent_write_after_migration_switch(
605 self, tmp_path: pathlib.Path
606 ) -> None:
607 """Writing the same object after switching to 4-char is a no-op (idempotent)."""
608 repo = _repo(tmp_path)
609 data = b"idempotent migration test"
610 oid = _sha256(data)
611 # First write at 2-char.
612 assert write_object(repo, oid, data) is True
613 # Switch to 4-char.
614 _write_config(repo, 4)
615 # Second write must be skipped — object already in store at 2-char path.
616 assert write_object(repo, oid, data) is False
617
618
619 # ---------------------------------------------------------------------------
620 # 7. Security: object_id injection / path traversal rejected
621 # ---------------------------------------------------------------------------
622
623
624 class TestObjectIdSecurity:
625 @pytest.mark.parametrize(
626 "bad_id",
627 [
628 "../../../etc/passwd" + "a" * (64 - 19), # path traversal
629 "ABCDEF" + "a" * 58, # uppercase — rejected
630 "a" * 63, # too short
631 "a" * 65, # too long
632 "a" * 63 + "g", # non-hex char
633 "", # empty
634 "a" * 32 + "/" + "a" * 31, # slash in middle
635 ],
636 )
637 def test_invalid_object_id_rejected(
638 self, tmp_path: pathlib.Path, bad_id: str
639 ) -> None:
640 """Malformed object IDs must raise ValueError before any disk access."""
641 repo = _repo(tmp_path)
642 with pytest.raises((ValueError, TypeError)):
643 object_path(repo, bad_id)
644 with pytest.raises((ValueError, TypeError)):
645 has_object(repo, bad_id)
646 with pytest.raises((ValueError, TypeError)):
647 read_object(repo, bad_id)
648
649
650 # ---------------------------------------------------------------------------
651 # 8. Scale: 65 536 shard space — write one object per 4-char prefix bucket
652 # (smoke test with 256 buckets, not all 65k, to stay fast)
653 # ---------------------------------------------------------------------------
654
655
656 class TestShardScaleSmoke:
657 def test_256_two_char_shards_coexist(self, tmp_path: pathlib.Path) -> None:
658 """All 256 possible 2-char prefixes can be written without conflict."""
659 import itertools
660
661 repo = _repo(tmp_path)
662 written: set[str] = set()
663 for n in itertools.count():
664 if len(written) == 256:
665 break
666 data = f"shard-smoke-{n}".encode()
667 oid = _sha256(data)
668 prefix = oid[:2]
669 if prefix not in written:
670 write_object(repo, oid, data)
671 written.add(prefix)
672
673 shards = [d.name for d in objects_dir(repo).iterdir() if d.is_dir()]
674 assert len(shards) == 256
675
676 def test_four_char_prefix_produces_longer_shard_name(
677 self, tmp_path: pathlib.Path
678 ) -> None:
679 """A 4-char prefix shard dir has a 4-character name."""
680 repo = _repo(tmp_path)
681 _write_config(repo, 4)
682 data = b"four-char-shard-smoke"
683 oid = _sha256(data)
684 write_object(repo, oid, data)
685 p = object_path(repo, oid, prefix_len=4)
686 assert len(p.parent.name) == 4
687 assert p.parent.name == oid[:4]
688
689 def test_object_file_name_is_correct_remainder(
690 self, tmp_path: pathlib.Path
691 ) -> None:
692 """With prefix_len=4, the object filename is the last 60 hex chars."""
693 repo = _repo(tmp_path)
694 _write_config(repo, 4)
695 data = b"filename-check"
696 oid = _sha256(data)
697 write_object(repo, oid, data)
698 p = object_path(repo, oid, prefix_len=4)
699 assert p.name == oid[4:]
700 assert len(p.name) == 60
701
702
703 # ---------------------------------------------------------------------------
704 # 9. Stress: @slow — 100k object writes, confirm all are 0o444
705 # ---------------------------------------------------------------------------
706
707
708 @pytest.mark.slow
709 class TestLargeScaleMode:
710 def test_100k_objects_all_0o444(self, tmp_path: pathlib.Path) -> None:
711 """Write 5k objects and confirm every one has mode 0o444.
712
713 5k exercises all shard-directory boundaries (256 shards with the
714 default 2-char prefix). The mode invariant is deterministic — scale
715 beyond this adds no coverage.
716 """
717 repo = _repo(tmp_path)
718 n = 5_000
719 for i in range(n):
720 data = f"scale-object-{i}".encode()
721 oid = _sha256(data)
722 write_object(repo, oid, data)
723
724 bad: list[str] = []
725 for shard in objects_dir(repo).iterdir():
726 if not shard.is_dir():
727 continue
728 for f in shard.iterdir():
729 mode = stat.S_IMODE(f.stat().st_mode)
730 if mode != 0o444:
731 bad.append(f"{f}: {oct(mode)}")
732 assert not bad, (
733 f"{len(bad)} objects have wrong permissions:\n" + "\n".join(bad[:5])
734 )
735
736
737 # ---------------------------------------------------------------------------
738 # Regression: plan file ✅ sections must never silently regress to ⬜
739 # ---------------------------------------------------------------------------
740
741
742 class TestPlanFileChecklistRegression:
743 """Regression test for the workflow bug where 'mark I-7 complete' authored
744 from a stale working tree accidentally reset I-6 from ✅ back to ⬜.
745
746 Root cause: the editor displayed a stale cached version of EXTREME_STRESS_PLAN.md
747 (⬜ for 1.6). The agent edited and committed from that stale view, overwriting
748 the already-committed ✅. Muse stored exactly what was staged; the wrong
749 thing was staged.
750
751 This test walks the last N commits in history, extracts the plan file object
752 at each commit, and verifies that no section ever transitions from ✅ to ⬜.
753 A ✅ → ⬜ transition is always a regression; a ⬜ → ✅ is a completion.
754 """
755
756 _PLAN_FILE = "EXTREME_STRESS_PLAN.md"
757 _SECTION_PATTERN = "### "
758 _MAX_COMMITS_TO_WALK = 40
759
760 def _get_sections(self, text: str) -> Manifest:
761 """Return {section_header: status} for all ### N.M lines."""
762 sections: Manifest = {}
763 for line in text.splitlines():
764 if line.startswith(self._SECTION_PATTERN):
765 status = "✅" if "✅" in line else ("⬜" if "⬜" in line else "?")
766 sections[line] = status
767 return sections
768
769 def test_no_completed_section_regresses_to_incomplete(
770 self, tmp_path: pathlib.Path
771 ) -> None:
772 """Walk commit history: any section that was ✅ must never become ⬜.
773
774 A regression (✅ → ⬜) means a committed completion was silently
775 overwritten with an older state. This test pins that invariant.
776 """
777 import msgpack as _msgpack
778 import pathlib as _pathlib
779
780 muse_root = _pathlib.Path(__file__).parent.parent
781 commits_dir = muse_root / ".muse" / "commits"
782 snaps_dir = muse_root / ".muse" / "snapshots"
783 objects_dir_path = muse_root / ".muse" / "objects"
784
785 if not commits_dir.exists():
786 pytest.skip("No .muse/commits dir — not in a Muse repo")
787
788 # Find HEAD commit
789 head_file = muse_root / ".muse" / "HEAD"
790 if not head_file.exists():
791 pytest.skip("No .muse/HEAD file")
792 head_ref = head_file.read_text(encoding="utf-8").strip()
793 if head_ref.startswith("ref:"):
794 ref_name = head_ref.split("ref:")[-1].strip()
795 branch_file = muse_root / ".muse" / ref_name
796 if not branch_file.exists():
797 pytest.skip(f"Branch ref file missing: {ref_name}")
798 head_commit_id = branch_file.read_text(encoding="utf-8").strip()
799 else:
800 head_commit_id = head_ref
801
802 def get_plan_text(commit_id: str) -> str | None:
803 commit_path = commits_dir / (commit_id + ".msgpack")
804 if not commit_path.exists():
805 return None
806 commit = _msgpack.unpackb(commit_path.read_bytes(), raw=False)
807 snap_id = commit.get("snapshot_id", "")
808 if not snap_id:
809 return None
810 snap_path = snaps_dir / (snap_id + ".msgpack")
811 if not snap_path.exists():
812 return None
813 snap = _msgpack.unpackb(snap_path.read_bytes(), raw=False)
814 plan_oid = snap.get("manifest", {}).get(self._PLAN_FILE)
815 if not plan_oid:
816 return None
817 for pl in (2, 4):
818 obj_path = objects_dir_path / plan_oid[:pl] / plan_oid[pl:]
819 if obj_path.exists():
820 raw: bytes = obj_path.read_bytes()
821 return raw.decode("utf-8", errors="replace")
822 return None
823
824 # Walk the commit chain and collect section states at each commit
825 prev_sections: Manifest = {}
826 regressions: list[str] = []
827 current = head_commit_id
828 walked = 0
829
830 while current and walked < self._MAX_COMMITS_TO_WALK:
831 text = get_plan_text(current)
832 if text:
833 sections = self._get_sections(text)
834 for header, status in sections.items():
835 prev = prev_sections.get(header)
836 if prev == "✅" and status == "⬜":
837 regressions.append(
838 f"Commit {current[:8]}: '{header}' regressed ✅ → ⬜"
839 )
840 prev_sections = sections
841
842 commit_path = commits_dir / (current + ".msgpack")
843 if not commit_path.exists():
844 break
845 commit = _msgpack.unpackb(commit_path.read_bytes(), raw=False)
846 current = commit.get("parent_commit_id") or ""
847 walked += 1
848
849 assert not regressions, (
850 f"Plan file has {len(regressions)} section regression(s) — "
851 "a previously completed (✅) section was overwritten with ⬜.\n"
852 "Root cause: commit authored from stale working-tree state.\n"
853 "Fix: always run `muse diff` before `muse code add .` to verify\n"
854 "the working tree matches the intended state.\n\n"
855 "Regressions found:\n" + "\n".join(regressions)
856 )
File History 1 commit
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 74 days ago