gabriel / muse public
test_integrity_I9_sigkill.py python
953 lines 36.4 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
1 """I-9 — Crash safety: SIGKILL simulation and startup GC sweep.
2
3 Validates three guarantees:
4
5 1. **Startup GC correctness** — :func:`muse.core.repo.require_repo` sweeps
6 every stale temp-file family on the next command after a crash:
7 * ``.obj-tmp-*`` / ``.restore-tmp-*`` — object-store shard directories
8 * ``.muse-tmp-*`` — store/config writes in ``.muse/`` subdirectories
9 * ``.stat_cache_*.tmp`` — :class:`~muse.core.stat_cache.StatCache`
10
11 2. **SIGKILL safety at timing windows** — a process killed at T+50 ms,
12 T+100 ms, and T+200 ms into a write sequence leaves the repository in a
13 consistent state. The subsequent startup GC removes any orphan temps so
14 the *next* ``muse commit`` succeeds.
15
16 3. **Push idempotency under SIGKILL** — a partial push leaves no corruption
17 on the remote because :func:`~muse.core.object_store.write_object` is
18 content-addressed and atomic; incomplete object writes are swept by the
19 remote-side startup GC.
20
21 Test classes
22 ------------
23 * ``TestCleanupMuseDirTemps`` — unit tests for ``_cleanup_muse_dir_temps``
24 * ``TestStartupGcObjectTemps`` — object-store orphan files swept by GC
25 * ``TestStartupGcMuseTemps`` — ``.muse-tmp-*`` files swept by GC
26 * ``TestStartupGcStatCacheTemps`` — ``.stat_cache_*.tmp`` swept by GC
27 * ``TestRequireRepoCallsGc`` — ``require_repo`` triggers the sweep
28 * ``TestMultipleSigkills`` — stale files from *N* crashes all swept
29 * ``TestSigkillAtTimingWindows`` — subprocess SIGKILL at T+50/100/200 ms
30 * ``TestSigkillDuringCommit`` — full CLI commit survives SIGKILL
31 * ``TestSigkillDuringPush`` — push path is idempotent under SIGKILL
32 * ``TestGcPreservesRealObjects`` — GC never deletes valid stored objects
33 * ``TestGcSweeperPerformance`` — sweep ≤ 10 ms with 1 000 stale files
34 * ``TestRestoreTempWorkdirBound`` — restore-tmp in workdir: documented scope
35 """
36
37 from __future__ import annotations
38
39 import hashlib
40 import multiprocessing
41 import os
42 import pathlib
43 import signal
44 import tempfile
45 import time
46
47 import pytest
48
49 from muse.core.object_store import (
50 cleanup_stale_object_temps,
51 read_object,
52 write_object,
53 )
54 from muse.core.repo import (
55 _MUSE_TEMP_PREFIXES,
56 _MUSE_SWEEP_DIRS,
57 _cleanup_muse_dir_temps,
58 _startup_gc,
59 require_repo,
60 )
61 from muse.core.store import write_text_atomic
62
63
64 # ---------------------------------------------------------------------------
65 # Helpers
66 # ---------------------------------------------------------------------------
67
68
69 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
70 """Minimal .muse/ layout for unit tests."""
71 muse = tmp_path / ".muse"
72 muse.mkdir()
73 (muse / "commits").mkdir()
74 (muse / "snapshots").mkdir()
75 (muse / "branches").mkdir()
76 (muse / "refs").mkdir()
77 (muse / "refs" / "heads").mkdir()
78 (muse / "objects").mkdir()
79 return tmp_path
80
81
82 def _oid(data: bytes) -> str:
83 return hashlib.sha256(data).hexdigest()
84
85
86 def _plant_stale_muse_tmp(muse_dir: pathlib.Path, subdir: str = "") -> pathlib.Path:
87 """Create a fake .muse-tmp-* file as would be left by a SIGKILL'd write_text_atomic."""
88 target = muse_dir / subdir if subdir else muse_dir
89 target.mkdir(parents=True, exist_ok=True)
90 fd, path = tempfile.mkstemp(dir=target, prefix=".muse-tmp-")
91 os.close(fd)
92 pathlib.Path(path).write_text("partial content", encoding="utf-8")
93 return pathlib.Path(path)
94
95
96 def _plant_stale_stat_cache_tmp(muse_dir: pathlib.Path) -> pathlib.Path:
97 """Create a fake .stat_cache_*.tmp file as would be left by a SIGKILL'd StatCache.save."""
98 fd, path = tempfile.mkstemp(dir=muse_dir, prefix=".stat_cache_", suffix=".tmp")
99 os.close(fd)
100 pathlib.Path(path).write_bytes(b"\x00" * 64)
101 return pathlib.Path(path)
102
103
104 def _make_stale(path: pathlib.Path) -> pathlib.Path:
105 """Backdate *path* mtime past the 60-second age gate in cleanup_stale_object_temps.
106
107 cleanup_stale_object_temps skips files younger than _CLEANUP_MIN_AGE_SECS (60 s).
108 Setting mtime to the Unix epoch (1970-01-01) makes freshly-created temp files
109 look decades old so cleanup picks them up immediately in tests.
110 """
111 os.utime(str(path), (0, 0))
112 return path
113
114
115 def _plant_stale_obj_tmp(objects_shard: pathlib.Path) -> pathlib.Path:
116 """Create a fake .obj-tmp-* file as would be left by a SIGKILL'd write_object."""
117 fd, path = tempfile.mkstemp(dir=objects_shard, prefix=".obj-tmp-")
118 os.close(fd)
119 pathlib.Path(path).write_bytes(b"partial object bytes")
120 return _make_stale(pathlib.Path(path))
121
122
123 def _plant_stale_restore_tmp(shard: pathlib.Path) -> pathlib.Path:
124 """Create a fake .restore-tmp-* file as would be left by a SIGKILL'd restore_object."""
125 fd, path = tempfile.mkstemp(dir=shard, prefix=".restore-tmp-")
126 os.close(fd)
127 pathlib.Path(path).write_bytes(b"partial restore")
128 return _make_stale(pathlib.Path(path))
129
130
131 def _count_stale_files(repo_root: pathlib.Path) -> int:
132 """Count all stale temp files in .muse/ (all families)."""
133 muse = repo_root / ".muse"
134 total = 0
135 for f in muse.rglob("*"):
136 if f.is_file() and any(
137 f.name.startswith(p)
138 for p in (".muse-tmp-", ".stat_cache_", ".obj-tmp-", ".restore-tmp-")
139 ):
140 total += 1
141 return total
142
143
144 # ---------------------------------------------------------------------------
145 # Subprocess workers — defined at module level for picklability under "spawn"
146 # ---------------------------------------------------------------------------
147
148
149 def _write_objects_worker(root: pathlib.Path, count: int) -> None:
150 """Write objects in a tight loop — killed midway to simulate SIGKILL."""
151 import hashlib as _hl
152 import time as _t
153
154 from muse.core.object_store import write_object as _wo
155
156 for i in range(count):
157 payload = f"sigkill-object-{i:06d}".encode()
158 oid = _hl.sha256(payload).hexdigest()
159 _wo(root, oid, payload)
160 _t.sleep(0.0002)
161
162
163 def _write_store_worker(root: pathlib.Path, count: int) -> None:
164 """Write commit-dir text atomically in a loop — killed midway."""
165 import time as _t
166
167 from muse.core.store import write_text_atomic as _wta
168
169 for i in range(count):
170 path = root / ".muse" / "commits" / f"fake-{i:06d}.msgpack"
171 _wta(path, f"fake commit {i}")
172 _t.sleep(0.0002)
173
174
175 def _full_commit_worker(repo_path: pathlib.Path, commit_msg: str) -> None:
176 """Run `muse commit -m <msg>` in a subprocess target (for SIGKILL testing)."""
177 import subprocess
178 import sys as _sys
179
180 subprocess.run(
181 ["muse", "commit", "-m", commit_msg],
182 cwd=str(repo_path),
183 stdout=_sys.stdout,
184 stderr=_sys.stderr,
185 )
186
187
188 # ---------------------------------------------------------------------------
189 # 1. _cleanup_muse_dir_temps — unit tests
190 # ---------------------------------------------------------------------------
191
192
193 class TestCleanupMuseDirTemps:
194 """Unit tests for the _cleanup_muse_dir_temps helper."""
195
196 def test_removes_muse_tmp_from_root(self, tmp_path: pathlib.Path) -> None:
197 muse = tmp_path / ".muse"
198 muse.mkdir()
199 stale = _plant_stale_muse_tmp(muse)
200 assert stale.exists()
201 removed = _cleanup_muse_dir_temps(muse)
202 assert removed == 1
203 assert not stale.exists()
204
205 def test_removes_muse_tmp_from_commits_subdir(self, tmp_path: pathlib.Path) -> None:
206 muse = tmp_path / ".muse"
207 muse.mkdir()
208 stale = _plant_stale_muse_tmp(muse, "commits")
209 removed = _cleanup_muse_dir_temps(muse)
210 assert removed == 1
211 assert not stale.exists()
212
213 def test_removes_muse_tmp_from_branches_subdir(self, tmp_path: pathlib.Path) -> None:
214 muse = tmp_path / ".muse"
215 muse.mkdir()
216 (muse / "branches").mkdir()
217 stale = _plant_stale_muse_tmp(muse, "branches")
218 removed = _cleanup_muse_dir_temps(muse)
219 assert removed == 1
220 assert not stale.exists()
221
222 def test_removes_muse_tmp_from_snapshots_subdir(self, tmp_path: pathlib.Path) -> None:
223 muse = tmp_path / ".muse"
224 muse.mkdir()
225 (muse / "snapshots").mkdir()
226 stale = _plant_stale_muse_tmp(muse, "snapshots")
227 removed = _cleanup_muse_dir_temps(muse)
228 assert removed == 1
229 assert not stale.exists()
230
231 def test_removes_stat_cache_tmp(self, tmp_path: pathlib.Path) -> None:
232 muse = tmp_path / ".muse"
233 muse.mkdir()
234 stale = _plant_stale_stat_cache_tmp(muse)
235 removed = _cleanup_muse_dir_temps(muse)
236 assert removed == 1
237 assert not stale.exists()
238
239 def test_preserves_real_muse_files(self, tmp_path: pathlib.Path) -> None:
240 muse = tmp_path / ".muse"
241 muse.mkdir()
242 (muse / "commits").mkdir()
243 # Real files must NOT be deleted
244 real_head = muse / "HEAD"
245 real_head.write_text("ref: refs/heads/main", encoding="utf-8")
246 real_commit = muse / "commits" / "abc123.msgpack"
247 real_commit.write_bytes(b"fake msgpack")
248 real_config = muse / "config.toml"
249 real_config.write_text("[core]\n", encoding="utf-8")
250 removed = _cleanup_muse_dir_temps(muse)
251 assert removed == 0
252 assert real_head.exists()
253 assert real_commit.exists()
254 assert real_config.exists()
255
256 def test_multiple_stale_files_across_subdirs(self, tmp_path: pathlib.Path) -> None:
257 muse = tmp_path / ".muse"
258 muse.mkdir()
259 stale: list[pathlib.Path] = []
260 stale.append(_plant_stale_muse_tmp(muse))
261 stale.append(_plant_stale_muse_tmp(muse, "commits"))
262 stale.append(_plant_stale_muse_tmp(muse, "snapshots"))
263 stale.append(_plant_stale_stat_cache_tmp(muse))
264 removed = _cleanup_muse_dir_temps(muse)
265 assert removed == 4
266 for f in stale:
267 assert not f.exists()
268
269 def test_nonexistent_muse_dir_returns_zero(self, tmp_path: pathlib.Path) -> None:
270 result = _cleanup_muse_dir_temps(tmp_path / ".muse")
271 assert result == 0
272
273 def test_missing_subdir_is_skipped_silently(self, tmp_path: pathlib.Path) -> None:
274 muse = tmp_path / ".muse"
275 muse.mkdir()
276 # Only root exists; subdirs (branches, commits, …) do not
277 stale = _plant_stale_muse_tmp(muse)
278 removed = _cleanup_muse_dir_temps(muse)
279 assert removed == 1
280 assert not stale.exists()
281
282 def test_idempotent_second_call(self, tmp_path: pathlib.Path) -> None:
283 muse = tmp_path / ".muse"
284 muse.mkdir()
285 _plant_stale_muse_tmp(muse)
286 _cleanup_muse_dir_temps(muse)
287 # Second call on clean dir must not raise and return 0
288 removed = _cleanup_muse_dir_temps(muse)
289 assert removed == 0
290
291 def test_temp_prefixes_constant_non_empty(self) -> None:
292 """_MUSE_TEMP_PREFIXES is exported and non-empty — agents can introspect it."""
293 assert len(_MUSE_TEMP_PREFIXES) >= 2
294 assert ".muse-tmp-" in _MUSE_TEMP_PREFIXES
295 assert ".stat_cache_" in _MUSE_TEMP_PREFIXES
296
297 def test_sweep_dirs_constant_includes_all_write_sites(self) -> None:
298 """_MUSE_SWEEP_DIRS covers every directory that uses write_text_atomic / _write_msgpack_atomic."""
299 required = {"", "branches", "commits", "snapshots", "tags", "releases"}
300 assert required.issubset(set(_MUSE_SWEEP_DIRS))
301
302
303 # ---------------------------------------------------------------------------
304 # 2. Startup GC — object-store orphans
305 # ---------------------------------------------------------------------------
306
307
308 class TestStartupGcObjectTemps:
309 """Object-store stale temps (.obj-tmp-*, .restore-tmp-*) are swept by startup GC."""
310
311 def test_obj_tmp_removed_by_cleanup(self, tmp_path: pathlib.Path) -> None:
312 repo = _repo(tmp_path)
313 shard = repo / ".muse" / "objects" / "ab"
314 shard.mkdir(parents=True, exist_ok=True)
315 stale = _plant_stale_obj_tmp(shard)
316 assert stale.exists()
317 removed = cleanup_stale_object_temps(repo)
318 assert removed >= 1
319 assert not stale.exists()
320
321 def test_restore_tmp_removed_by_cleanup(self, tmp_path: pathlib.Path) -> None:
322 repo = _repo(tmp_path)
323 shard = repo / ".muse" / "objects" / "cd"
324 shard.mkdir(parents=True, exist_ok=True)
325 stale = _plant_stale_restore_tmp(shard)
326 removed = cleanup_stale_object_temps(repo)
327 assert removed >= 1
328 assert not stale.exists()
329
330 def test_startup_gc_delegates_to_object_cleanup(self, tmp_path: pathlib.Path) -> None:
331 repo = _repo(tmp_path)
332 shard = repo / ".muse" / "objects" / "ef"
333 shard.mkdir(parents=True, exist_ok=True)
334 stale_obj = _plant_stale_obj_tmp(shard)
335 stale_restore = _plant_stale_restore_tmp(shard)
336 _startup_gc(repo)
337 assert not stale_obj.exists()
338 assert not stale_restore.exists()
339
340 def test_real_objects_preserved_by_startup_gc(self, tmp_path: pathlib.Path) -> None:
341 repo = _repo(tmp_path)
342 data = b"real object content"
343 oid = _oid(data)
344 write_object(repo, oid, data)
345 _startup_gc(repo)
346 result = read_object(repo, oid)
347 assert result == data
348
349 def test_stale_and_real_coexist_only_stale_removed(self, tmp_path: pathlib.Path) -> None:
350 repo = _repo(tmp_path)
351 data = b"survivor"
352 oid = _oid(data)
353 write_object(repo, oid, data)
354 # Plant stale temp in same shard as the real object
355 shard_name = oid[:2]
356 shard = repo / ".muse" / "objects" / shard_name
357 stale = _plant_stale_obj_tmp(shard)
358 _startup_gc(repo)
359 assert not stale.exists()
360 assert read_object(repo, oid) == data
361
362
363 # ---------------------------------------------------------------------------
364 # 3. Startup GC — .muse-tmp-* in subdirectories
365 # ---------------------------------------------------------------------------
366
367
368 class TestStartupGcMuseTemps:
369 """.muse-tmp-* files in all .muse/ subdirs are swept by _startup_gc."""
370
371 def test_muse_tmp_in_root_swept(self, tmp_path: pathlib.Path) -> None:
372 repo = _repo(tmp_path)
373 stale = _plant_stale_muse_tmp(repo / ".muse")
374 _startup_gc(repo)
375 assert not stale.exists()
376
377 def test_muse_tmp_in_commits_swept(self, tmp_path: pathlib.Path) -> None:
378 repo = _repo(tmp_path)
379 stale = _plant_stale_muse_tmp(repo / ".muse", "commits")
380 _startup_gc(repo)
381 assert not stale.exists()
382
383 def test_muse_tmp_in_branches_swept(self, tmp_path: pathlib.Path) -> None:
384 repo = _repo(tmp_path)
385 stale = _plant_stale_muse_tmp(repo / ".muse", "branches")
386 _startup_gc(repo)
387 assert not stale.exists()
388
389 def test_muse_tmp_in_snapshots_swept(self, tmp_path: pathlib.Path) -> None:
390 repo = _repo(tmp_path)
391 stale = _plant_stale_muse_tmp(repo / ".muse", "snapshots")
392 _startup_gc(repo)
393 assert not stale.exists()
394
395 def test_muse_tmp_in_tags_swept(self, tmp_path: pathlib.Path) -> None:
396 repo = _repo(tmp_path)
397 (repo / ".muse" / "tags").mkdir()
398 stale = _plant_stale_muse_tmp(repo / ".muse", "tags")
399 _startup_gc(repo)
400 assert not stale.exists()
401
402 def test_muse_tmp_in_releases_swept(self, tmp_path: pathlib.Path) -> None:
403 repo = _repo(tmp_path)
404 (repo / ".muse" / "releases").mkdir()
405 stale = _plant_stale_muse_tmp(repo / ".muse", "releases")
406 _startup_gc(repo)
407 assert not stale.exists()
408
409 def test_real_msgpack_in_commits_preserved(self, tmp_path: pathlib.Path) -> None:
410 repo = _repo(tmp_path)
411 real = repo / ".muse" / "commits" / "deadbeef.msgpack"
412 real.write_bytes(b"\x82\xa9commit_id\xa8deadbeef")
413 _startup_gc(repo)
414 assert real.exists()
415
416
417 # ---------------------------------------------------------------------------
418 # 4. Startup GC — .stat_cache_*.tmp
419 # ---------------------------------------------------------------------------
420
421
422 class TestStartupGcStatCacheTemps:
423 """.stat_cache_*.tmp files (StatCache.save) are swept by _startup_gc."""
424
425 def test_stat_cache_tmp_swept(self, tmp_path: pathlib.Path) -> None:
426 repo = _repo(tmp_path)
427 stale = _plant_stale_stat_cache_tmp(repo / ".muse")
428 _startup_gc(repo)
429 assert not stale.exists()
430
431 def test_real_stat_cache_msgpack_preserved(self, tmp_path: pathlib.Path) -> None:
432 repo = _repo(tmp_path)
433 real = repo / ".muse" / "stat_cache.msgpack"
434 real.write_bytes(b"\x82\xa7version\x02")
435 _startup_gc(repo)
436 assert real.exists()
437
438 def test_multiple_stat_cache_tmps_all_swept(self, tmp_path: pathlib.Path) -> None:
439 repo = _repo(tmp_path)
440 stales = [_plant_stale_stat_cache_tmp(repo / ".muse") for _ in range(5)]
441 _startup_gc(repo)
442 for s in stales:
443 assert not s.exists()
444
445
446 # ---------------------------------------------------------------------------
447 # 5. require_repo calls the startup GC
448 # ---------------------------------------------------------------------------
449
450
451 class TestRequireRepoCallsGc:
452 """require_repo() triggers the full startup GC sweep."""
453
454 def test_require_repo_removes_obj_tmp(self, tmp_path: pathlib.Path) -> None:
455 repo = _repo(tmp_path)
456 shard = repo / ".muse" / "objects" / "aa"
457 shard.mkdir(parents=True, exist_ok=True)
458 stale = _plant_stale_obj_tmp(shard)
459 # Call require_repo with MUSE_REPO_ROOT override so it finds the repo
460 ctx = require_repo(start=repo)
461 assert ctx == repo
462 assert not stale.exists()
463
464 def test_require_repo_removes_muse_tmp(self, tmp_path: pathlib.Path) -> None:
465 repo = _repo(tmp_path)
466 stale = _plant_stale_muse_tmp(repo / ".muse")
467 require_repo(start=repo)
468 assert not stale.exists()
469
470 def test_require_repo_removes_stat_cache_tmp(self, tmp_path: pathlib.Path) -> None:
471 repo = _repo(tmp_path)
472 stale = _plant_stale_stat_cache_tmp(repo / ".muse")
473 require_repo(start=repo)
474 assert not stale.exists()
475
476 def test_require_repo_sweeps_all_families_at_once(self, tmp_path: pathlib.Path) -> None:
477 repo = _repo(tmp_path)
478 shard = repo / ".muse" / "objects" / "bb"
479 shard.mkdir(parents=True, exist_ok=True)
480 f1 = _plant_stale_obj_tmp(shard)
481 f2 = _plant_stale_restore_tmp(shard)
482 f3 = _plant_stale_muse_tmp(repo / ".muse")
483 f4 = _plant_stale_muse_tmp(repo / ".muse", "commits")
484 f5 = _plant_stale_stat_cache_tmp(repo / ".muse")
485 require_repo(start=repo)
486 for f in (f1, f2, f3, f4, f5):
487 assert not f.exists(), f"{f.name} should have been swept"
488
489 def test_require_repo_not_in_repo_still_raises(self, tmp_path: pathlib.Path) -> None:
490 """require_repo on a non-repo path still exits — GC is not run on miss."""
491 with pytest.raises(SystemExit):
492 require_repo(start=tmp_path)
493
494
495 # ---------------------------------------------------------------------------
496 # 6. Multiple consecutive SIGKILLs — accumulated stale files all swept
497 # ---------------------------------------------------------------------------
498
499
500 class TestMultipleSigkills:
501 """Simulate N crashes: stale files from each accumulate and are all swept."""
502
503 def test_three_crash_generations_all_swept(self, tmp_path: pathlib.Path) -> None:
504 repo = _repo(tmp_path)
505 muse = repo / ".muse"
506 shard = repo / ".muse" / "objects" / "cc"
507 shard.mkdir(parents=True, exist_ok=True)
508
509 # Simulate 3 separate crashes leaving stale files from each family.
510 stales: list[pathlib.Path] = []
511 for _ in range(3):
512 stales.append(_plant_stale_obj_tmp(shard))
513 stales.append(_plant_stale_restore_tmp(shard))
514 stales.append(_plant_stale_muse_tmp(muse))
515 stales.append(_plant_stale_muse_tmp(muse, "commits"))
516 stales.append(_plant_stale_stat_cache_tmp(muse))
517
518 assert len(stales) == 15
519 _startup_gc(repo)
520 for f in stales:
521 assert not f.exists(), f"Stale file survived: {f.name}"
522
523 def test_gc_count_is_accurate(self, tmp_path: pathlib.Path) -> None:
524 repo = _repo(tmp_path)
525 muse = repo / ".muse"
526 (muse / "tags").mkdir()
527 for _ in range(4):
528 _plant_stale_muse_tmp(muse)
529 for _ in range(3):
530 _plant_stale_stat_cache_tmp(muse)
531 # 7 total stale files in muse dir
532 removed = _cleanup_muse_dir_temps(muse)
533 assert removed == 7
534 assert _count_stale_files(repo) == 0
535
536
537 # ---------------------------------------------------------------------------
538 # 7. SIGKILL at T+50ms / T+100ms / T+200ms — object-store write sequence
539 # ---------------------------------------------------------------------------
540
541
542 class TestSigkillAtTimingWindows:
543 """Subprocess SIGKILL at precise timing windows: store stays consistent."""
544
545 @pytest.mark.slow
546 @pytest.mark.parametrize("delay_ms", [50, 100, 200])
547 def test_object_store_consistent_after_sigkill(
548 self, tmp_path: pathlib.Path, delay_ms: int
549 ) -> None:
550 repo = _repo(tmp_path)
551
552 # Pre-write 10 known objects before the crashable process starts.
553 pre_data: list[tuple[str, bytes]] = []
554 for i in range(10):
555 payload = f"pre-kill-{i:03d}".encode()
556 oid = _oid(payload)
557 write_object(repo, oid, payload)
558 pre_data.append((oid, payload))
559
560 # Spawn a fresh process that writes objects in a tight loop.
561 ctx = multiprocessing.get_context("spawn")
562 proc = ctx.Process(target=_write_objects_worker, args=(repo, 2000))
563 proc.start()
564
565 # Kill it at the specified timing window.
566 time.sleep(delay_ms / 1000.0)
567 if proc.is_alive():
568 assert proc.pid is not None
569 os.kill(proc.pid, signal.SIGKILL)
570 proc.join(timeout=5)
571
572 # Startup GC: simulates the next command after the crash.
573 _startup_gc(repo)
574
575 # No stale temp files must remain anywhere in .muse/.
576 assert _count_stale_files(repo) == 0, (
577 f"Stale temp files survived SIGKILL at T+{delay_ms}ms"
578 )
579
580 # Every pre-kill object must still be readable and hash-verified.
581 for oid, payload in pre_data:
582 assert read_object(repo, oid) == payload, (
583 f"Pre-kill object {oid[:8]} corrupted after SIGKILL at T+{delay_ms}ms"
584 )
585
586 @pytest.mark.slow
587 @pytest.mark.parametrize("delay_ms", [50, 100, 200])
588 def test_store_write_consistent_after_sigkill(
589 self, tmp_path: pathlib.Path, delay_ms: int
590 ) -> None:
591 """SIGKILL during write_text_atomic loop leaves no stale .muse-tmp-* files."""
592 repo = _repo(tmp_path)
593
594 ctx = multiprocessing.get_context("spawn")
595 proc = ctx.Process(target=_write_store_worker, args=(repo, 2000))
596 proc.start()
597
598 time.sleep(delay_ms / 1000.0)
599 if proc.is_alive():
600 assert proc.pid is not None
601 os.kill(proc.pid, signal.SIGKILL)
602 proc.join(timeout=5)
603
604 # Startup GC sweep.
605 _startup_gc(repo)
606
607 # No .muse-tmp-* files may survive.
608 muse = repo / ".muse"
609 leftovers = list(muse.rglob(".muse-tmp-*"))
610 assert leftovers == [], (
611 f"Stale .muse-tmp-* survived SIGKILL at T+{delay_ms}ms: {leftovers}"
612 )
613
614
615 # ---------------------------------------------------------------------------
616 # 8. Full CLI commit survives SIGKILL
617 # ---------------------------------------------------------------------------
618
619
620 class TestSigkillDuringCommit:
621 """End-to-end: SIGKILL during `muse commit` leaves repo in a recoverable state."""
622
623 def _init_real_repo(self, tmp_path: pathlib.Path) -> pathlib.Path:
624 """Create a minimal real muse repo with a committed file."""
625 import subprocess
626
627 env = os.environ.copy()
628 env["MUSE_REPO_ROOT"] = str(tmp_path)
629
630 subprocess.run(["muse", "init"], cwd=str(tmp_path), env=env, check=True,
631 capture_output=True)
632 # Stage and commit a file so there is a valid HEAD.
633 (tmp_path / "file.txt").write_text("hello", encoding="utf-8")
634 subprocess.run(["muse", "code", "add", "."], cwd=str(tmp_path), env=env,
635 check=True, capture_output=True)
636 subprocess.run(["muse", "commit", "-m", "init"], cwd=str(tmp_path), env=env,
637 check=True, capture_output=True)
638 return tmp_path
639
640 @pytest.mark.slow
641 def test_muse_status_runs_after_sigkill(self, tmp_path: pathlib.Path) -> None:
642 """`muse status` must exit cleanly (exit 0) after a SIGKILL'd commit."""
643 import subprocess
644
645 repo = self._init_real_repo(tmp_path)
646
647 # Modify a file so there is something to commit.
648 (repo / "file.txt").write_text("changed", encoding="utf-8")
649 subprocess.run(["muse", "code", "add", "."], cwd=str(repo),
650 capture_output=True)
651
652 # Spawn commit subprocess and kill it immediately.
653 ctx = multiprocessing.get_context("spawn")
654 proc = ctx.Process(target=_full_commit_worker, args=(repo, "crash-me"))
655 proc.start()
656 time.sleep(0.05)
657 if proc.is_alive():
658 assert proc.pid is not None
659 os.kill(proc.pid, signal.SIGKILL)
660 proc.join(timeout=5)
661
662 # Startup GC runs on next require_repo invocation (muse status triggers it).
663 result = subprocess.run(
664 ["muse", "status"],
665 cwd=str(repo),
666 capture_output=True,
667 text=True,
668 )
669 # status must exit 0; any non-zero means repo is corrupt.
670 assert result.returncode == 0, (
671 f"muse status failed after SIGKILL:\n{result.stdout}\n{result.stderr}"
672 )
673
674 @pytest.mark.slow
675 def test_no_stale_temps_after_sigkill_and_next_command(
676 self, tmp_path: pathlib.Path
677 ) -> None:
678 """After SIGKILL + muse status, zero stale temps remain in .muse/."""
679 import subprocess
680
681 repo = self._init_real_repo(tmp_path)
682 (repo / "file.txt").write_text("changed again", encoding="utf-8")
683 subprocess.run(["muse", "code", "add", "."], cwd=str(repo), capture_output=True)
684
685 ctx = multiprocessing.get_context("spawn")
686 proc = ctx.Process(target=_full_commit_worker, args=(repo, "crash-me-2"))
687 proc.start()
688 time.sleep(0.05)
689 if proc.is_alive():
690 assert proc.pid is not None
691 os.kill(proc.pid, signal.SIGKILL)
692 proc.join(timeout=5)
693
694 # Trigger startup GC via the next command.
695 subprocess.run(["muse", "status"], cwd=str(repo), capture_output=True)
696
697 assert _count_stale_files(repo) == 0, "Stale temps remain after startup GC"
698
699
700 # ---------------------------------------------------------------------------
701 # 9. Push path idempotency under SIGKILL
702 # ---------------------------------------------------------------------------
703
704
705 class TestSigkillDuringPush:
706 """SIGKILL during push writes leaves no corruption: write_object is atomic."""
707
708 @pytest.mark.slow
709 def test_push_objects_atomic_under_sigkill(self, tmp_path: pathlib.Path) -> None:
710 """Objects pushed before kill are readable; partial objects are absent."""
711 (tmp_path / "local").mkdir()
712 (tmp_path / "remote").mkdir()
713 local = _repo(tmp_path / "local")
714 remote = _repo(tmp_path / "remote")
715
716 # Write 10 objects to local and also push them to remote before kill.
717 pre_data: list[tuple[str, bytes]] = []
718 for i in range(10):
719 payload = f"push-pre-kill-{i}".encode()
720 oid = _oid(payload)
721 write_object(local, oid, payload)
722 write_object(remote, oid, payload) # simulate push of pre-kill objects
723 pre_data.append((oid, payload))
724
725 # Spawn a process that keeps writing objects to the remote store.
726 ctx = multiprocessing.get_context("spawn")
727 proc = ctx.Process(target=_write_objects_worker, args=(remote, 2000))
728 proc.start()
729 time.sleep(0.08) # T+80ms kill
730 if proc.is_alive():
731 assert proc.pid is not None
732 os.kill(proc.pid, signal.SIGKILL)
733 proc.join(timeout=5)
734
735 # Backdate all temp files left by the killed process — the 60-second
736 # age gate in cleanup_stale_object_temps skips fresh files to protect
737 # concurrent writers; in tests we fast-forward mtime to simulate aging.
738 for f in (remote / ".muse").rglob("*"):
739 if f.is_file() and any(
740 f.name.startswith(p)
741 for p in (".obj-tmp-", ".restore-tmp-", ".muse-tmp-", ".stat_cache_")
742 ):
743 os.utime(f, (0, 0))
744
745 # Simulate remote-side startup GC (next muse command on the remote).
746 _startup_gc(remote)
747
748 # Remote must have no stale temps.
749 assert _count_stale_files(remote) == 0
750
751 # All pre-kill objects on remote must be intact.
752 for oid, payload in pre_data:
753 assert read_object(remote, oid) == payload, (
754 f"Remote object {oid[:8]} corrupted after push SIGKILL"
755 )
756
757 def test_push_write_object_is_idempotent(self, tmp_path: pathlib.Path) -> None:
758 """write_object called twice for same OID returns False (skip) both times."""
759 repo = _repo(tmp_path)
760 payload = b"idempotent object"
761 oid = _oid(payload)
762 first = write_object(repo, oid, payload)
763 second = write_object(repo, oid, payload)
764 assert first is True
765 assert second is False
766 assert read_object(repo, oid) == payload
767
768 def test_partial_write_interrupted_at_os_level_leaves_no_dest(
769 self, tmp_path: pathlib.Path
770 ) -> None:
771 """The mkstemp→replace contract: if replace never happens, dest is absent."""
772 repo = _repo(tmp_path)
773 data = b"will be interrupted"
774 oid = _oid(data)
775
776 # Manually plant the stale temp (simulates SIGKILL after mkstemp, before replace)
777 shard = repo / ".muse" / "objects" / oid[:2]
778 shard.mkdir(parents=True, exist_ok=True)
779 stale = _plant_stale_obj_tmp(shard)
780
781 # The destination object must NOT exist (replace never happened).
782 from muse.core.object_store import has_object
783 assert not has_object(repo, oid)
784
785 # Cleanup removes the stale temp.
786 cleanup_stale_object_temps(repo)
787 assert not stale.exists()
788
789 # A fresh write_object succeeds normally.
790 write_object(repo, oid, data)
791 assert has_object(repo, oid)
792
793
794 # ---------------------------------------------------------------------------
795 # 10. GC preserves all real stored objects
796 # ---------------------------------------------------------------------------
797
798
799 class TestGcPreservesRealObjects:
800 """_startup_gc must never delete a valid stored object."""
801
802 def test_100_objects_all_survive_gc(self, tmp_path: pathlib.Path) -> None:
803 repo = _repo(tmp_path)
804 written: list[tuple[str, bytes]] = []
805 for i in range(100):
806 payload = f"object-{i:04d}".encode()
807 oid = _oid(payload)
808 write_object(repo, oid, payload)
809 written.append((oid, payload))
810
811 _startup_gc(repo)
812
813 for oid, payload in written:
814 assert read_object(repo, oid) == payload, f"Object {oid[:8]} deleted by GC"
815
816 def test_real_head_and_config_survive_gc(self, tmp_path: pathlib.Path) -> None:
817 repo = _repo(tmp_path)
818 muse = repo / ".muse"
819 head = muse / "HEAD"
820 head.write_text("ref: refs/heads/main\n", encoding="utf-8")
821 config = muse / "config.toml"
822 config.write_text("[core]\n name = \"test\"\n", encoding="utf-8")
823
824 _startup_gc(repo)
825
826 assert head.read_text(encoding="utf-8") == "ref: refs/heads/main\n"
827 assert "test" in config.read_text(encoding="utf-8")
828
829 def test_gc_on_empty_repo_is_noop(self, tmp_path: pathlib.Path) -> None:
830 repo = _repo(tmp_path)
831 # No objects, no stale files
832 _startup_gc(repo)
833 assert _count_stale_files(repo) == 0
834
835
836 # ---------------------------------------------------------------------------
837 # 11. Performance: GC sweep ≤ 10 ms with 1 000 stale files
838 # ---------------------------------------------------------------------------
839
840
841 class TestGcSweeperPerformance:
842 """Startup GC is fast enough to run on every require_repo invocation."""
843
844 @pytest.mark.slow
845 def test_sweep_1000_stale_files_under_500ms(self, tmp_path: pathlib.Path) -> None:
846 """GC sweeps 1 000 stale files in < 500 ms (wall clock including logging).
847
848 The budget is generous because macOS APFS + pytest log capture add
849 overhead (~50–100 μs per unlink + warning emission). On a real crash
850 scenario a repo will have at most 1–3 stale files, so steady-state
851 latency is < 1 ms. This test validates the worst-case bound.
852 """
853 repo = _repo(tmp_path)
854 muse = repo / ".muse"
855 (muse / "commits").mkdir(exist_ok=True)
856
857 # Plant 1 000 stale .muse-tmp-* files across two subdirs.
858 for _ in range(500):
859 _plant_stale_muse_tmp(muse)
860 for _ in range(500):
861 _plant_stale_muse_tmp(muse, "commits")
862
863 start = time.perf_counter()
864 removed = _cleanup_muse_dir_temps(muse)
865 elapsed_ms = (time.perf_counter() - start) * 1000
866
867 assert removed == 1000
868 assert elapsed_ms < 500.0, (
869 f"_cleanup_muse_dir_temps took {elapsed_ms:.1f} ms for 1 000 files "
870 f"(budget: 500 ms)"
871 )
872
873 @pytest.mark.slow
874 def test_full_startup_gc_under_200ms(self, tmp_path: pathlib.Path) -> None:
875 """Full _startup_gc (both sweeps) is < 200 ms with 200 stale temps.
876
877 In production a crash leaves 1–3 stale files at most; this tests the
878 adversarial bound of 200 simultaneous stale temps across both the
879 object store and .muse/ directories.
880 """
881 repo = _repo(tmp_path)
882 muse = repo / ".muse"
883 (muse / "commits").mkdir(exist_ok=True)
884
885 # 100 stale object temps across 10 shards.
886 for i in range(10):
887 shard = repo / ".muse" / "objects" / f"{i:02x}"
888 shard.mkdir(parents=True, exist_ok=True)
889 for _ in range(10):
890 _plant_stale_obj_tmp(shard)
891
892 # 100 stale muse temps.
893 for _ in range(50):
894 _plant_stale_muse_tmp(muse)
895 for _ in range(50):
896 _plant_stale_muse_tmp(muse, "commits")
897
898 start = time.perf_counter()
899 _startup_gc(repo)
900 elapsed_ms = (time.perf_counter() - start) * 1000
901
902 assert elapsed_ms < 200.0, (
903 f"_startup_gc took {elapsed_ms:.1f} ms (budget: 200 ms)"
904 )
905 assert _count_stale_files(repo) == 0
906
907
908 # ---------------------------------------------------------------------------
909 # 12. .restore-tmp-* in working-tree: scope documentation test
910 # ---------------------------------------------------------------------------
911
912
913 class TestRestoreTempWorkdirBound:
914 """.restore-tmp-* files in the working tree (not .muse/) are outside GC scope.
915
916 restore_object writes to a user-provided destination directory, not inside
917 .muse/. If SIGKILL occurs between mkstemp and os.replace in restore_object,
918 the stale temp stays in the working tree.
919
920 The documented guarantee: the .muse/ repo state is never corrupted, and
921 the stale restore temp in the working tree is inert (it does not block the
922 next checkout/merge, which simply overwrites the destination path atomically).
923 """
924
925 def test_restore_tmp_in_workdir_not_swept_by_gc(self, tmp_path: pathlib.Path) -> None:
926 """GC sweeps .muse/ only — stale restore temps in workdir are out of scope."""
927 repo = _repo(tmp_path)
928 # Plant a stale restore temp in the working tree (outside .muse/)
929 fd, stale_str = tempfile.mkstemp(dir=tmp_path, prefix=".restore-tmp-")
930 os.close(fd)
931 stale = pathlib.Path(stale_str)
932 stale.write_bytes(b"stale workdir restore temp")
933
934 # GC sweeps only .muse/; workdir stale is untouched.
935 _startup_gc(repo)
936
937 # Stale in workdir persists — this is the documented limitation.
938 assert stale.exists(), (
939 "GC must not touch working-tree files outside .muse/"
940 )
941 # But .muse/ is clean.
942 assert _count_stale_files(repo) == 0
943
944 stale.unlink() # cleanup
945
946 def test_restore_tmp_in_obj_shard_IS_swept(self, tmp_path: pathlib.Path) -> None:
947 """.restore-tmp-* inside .muse/objects/ shard dirs ARE swept (object store scope)."""
948 repo = _repo(tmp_path)
949 shard = repo / ".muse" / "objects" / "dd"
950 shard.mkdir(parents=True, exist_ok=True)
951 stale = _plant_stale_restore_tmp(shard)
952 _startup_gc(repo)
953 assert not stale.exists(), ".restore-tmp-* in object shard must be swept"
File History 5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 26 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e Merge branch 'dev' into main Human 29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce Merge branch 'dev' into main Human 48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa feat: Muse — version control for the agent era Human 100 days ago