test_gc_corrupt_commit_object_retention.py
python
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
| 1 | """Tests for Bug 10: GC deletes objects reachable from a corrupt commit. |
| 2 | |
| 3 | When a commit file has a corrupt snapshot_id field (bit-flip, tampered, or |
| 4 | previously written by the now-fixed from_dict timestamp substitution bug), |
| 5 | _collect_reachable_objects returns an empty set for that commit's snapshot. |
| 6 | muse gc then deletes those objects, permanently destroying file content. |
| 7 | |
| 8 | The sequence of doom: |
| 9 | 1. Commit A on disk: snapshot_id = "f"*64 (corrupt — should be "S1") |
| 10 | 2. `get_all_commits` returns this commit (no hash verification) |
| 11 | 3. `read_snapshot(root, "f"*64)` → None (file for "f"*64 doesn't exist) |
| 12 | 4. Objects from the REAL snapshot S1 are NOT added to reachable |
| 13 | 5. `muse gc` deletes those objects (no other commit references them) |
| 14 | 6. The working tree cannot be reconstructed from commit A |
| 15 | |
| 16 | Fix: `_collect_reachable_objects` must also scan snapshot files directly |
| 17 | (without hash verification) when read_snapshot returns None for a commit's |
| 18 | snapshot_id. This conservatively retains any object referenced in any |
| 19 | snapshot manifest on disk, regardless of whether the commit's snapshot_id |
| 20 | field is correct. |
| 21 | |
| 22 | Scope of tests |
| 23 | -------------- |
| 24 | Unit (_collect_reachable_objects): |
| 25 | - Objects reachable from a valid commit are retained |
| 26 | - Objects reachable from a commit with corrupt snapshot_id (field points |
| 27 | to non-existent snapshot) are still retained via raw snapshot scan |
| 28 | - Objects reachable from a commit with corrupt snapshot_id (field points |
| 29 | to a WRONG existing snapshot) are still retained via raw snapshot scan |
| 30 | - GC does NOT delete objects that are in ANY snapshot on disk, even orphaned |
| 31 | - Empty store returns empty reachable set (no crash) |
| 32 | - Multiple commits, one corrupt: all objects from all snapshots retained |
| 33 | |
| 34 | Integration (run_gc with corrupt commit): |
| 35 | - run_gc dry_run=True does not delete anything from a store with corrupt commit |
| 36 | - run_gc does not delete objects from corrupt commit's snapshot (default mode) |
| 37 | - run_gc --full does not delete objects from corrupt commit's snapshot |
| 38 | - Objects from a legitimately unreachable commit ARE deleted (GC still works) |
| 39 | |
| 40 | Stress: |
| 41 | - 50 commits, 5 with corrupt snapshot_ids: all objects from all 50 snapshots retained |
| 42 | """ |
| 43 | from __future__ import annotations |
| 44 | |
| 45 | type _FileStore = dict[str, bytes] |
| 46 | |
| 47 | import datetime |
| 48 | import hashlib |
| 49 | import pathlib |
| 50 | |
| 51 | import msgpack |
| 52 | import pytest |
| 53 | |
| 54 | from muse.core.gc import _collect_reachable_objects, run_gc |
| 55 | from muse.core.object_store import write_object |
| 56 | from muse.core.snapshot import compute_commit_id, compute_snapshot_id |
| 57 | from muse.core.store import ( |
| 58 | CommitRecord, |
| 59 | SnapshotRecord, |
| 60 | read_snapshot, |
| 61 | write_commit, |
| 62 | write_snapshot, |
| 63 | ) |
| 64 | |
| 65 | _TS = datetime.datetime(2024, 6, 15, 10, 0, 0, tzinfo=datetime.timezone.utc) |
| 66 | |
| 67 | |
| 68 | def _make_repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 69 | repo = tmp_path / "repo" |
| 70 | repo.mkdir() |
| 71 | (repo / ".muse").mkdir() |
| 72 | return repo |
| 73 | |
| 74 | |
| 75 | def _write_object(repo: pathlib.Path, content: bytes) -> str: |
| 76 | oid = hashlib.sha256(content).hexdigest() |
| 77 | write_object(repo, oid, content) |
| 78 | return oid |
| 79 | |
| 80 | |
| 81 | def _make_snapshot(repo: pathlib.Path, files: _FileStore) -> SnapshotRecord: |
| 82 | manifest = {} |
| 83 | for path, content in files.items(): |
| 84 | oid = _write_object(repo, content) |
| 85 | manifest[path] = oid |
| 86 | snap_id = compute_snapshot_id(manifest) |
| 87 | snap = SnapshotRecord( |
| 88 | snapshot_id=snap_id, |
| 89 | manifest=manifest, |
| 90 | directories=[], |
| 91 | created_at=_TS, |
| 92 | note="", |
| 93 | ) |
| 94 | write_snapshot(repo, snap) |
| 95 | return snap |
| 96 | |
| 97 | |
| 98 | def _make_commit( |
| 99 | repo: pathlib.Path, |
| 100 | snap_id: str, |
| 101 | *, |
| 102 | message: str = "test", |
| 103 | ts: datetime.datetime = _TS, |
| 104 | parent: str | None = None, |
| 105 | ) -> CommitRecord: |
| 106 | parent_ids = [parent] if parent else [] |
| 107 | commit_id = compute_commit_id(parent_ids, snap_id, message, ts.isoformat()) |
| 108 | record = CommitRecord( |
| 109 | commit_id=commit_id, |
| 110 | repo_id="r" * 64, |
| 111 | branch="main", |
| 112 | snapshot_id=snap_id, |
| 113 | message=message, |
| 114 | committed_at=ts, |
| 115 | parent_commit_id=parent, |
| 116 | parent2_commit_id=None, |
| 117 | author="gabriel", |
| 118 | metadata={}, |
| 119 | structured_delta=None, |
| 120 | sem_ver_bump="none", |
| 121 | breaking_changes=[], |
| 122 | agent_id="", |
| 123 | model_id="", |
| 124 | toolchain_id="", |
| 125 | prompt_hash="", |
| 126 | signature="", |
| 127 | signer_key_id="", |
| 128 | format_version=1, |
| 129 | reviewed_by=[], |
| 130 | test_runs=0, |
| 131 | ) |
| 132 | write_commit(repo, record) |
| 133 | return record |
| 134 | |
| 135 | |
| 136 | def _corrupt_commit_snapshot_id( |
| 137 | repo: pathlib.Path, commit_id: str, bad_snapshot_id: str = "f" * 64 |
| 138 | ) -> None: |
| 139 | """Directly corrupt the snapshot_id field in a commit file on disk.""" |
| 140 | path = repo / ".muse" / "commits" / f"{commit_id}.msgpack" |
| 141 | data = msgpack.unpackb(path.read_bytes(), raw=False) |
| 142 | data["snapshot_id"] = bad_snapshot_id |
| 143 | path.write_bytes(msgpack.packb(data, use_bin_type=True)) |
| 144 | |
| 145 | |
| 146 | # ────────────────────────────────────────────────────────────────────────────── |
| 147 | # Unit: _collect_reachable_objects |
| 148 | # ────────────────────────────────────────────────────────────────────────────── |
| 149 | |
| 150 | class TestCollectReachableObjects: |
| 151 | |
| 152 | def test_valid_commit_objects_retained(self, tmp_path: pathlib.Path) -> None: |
| 153 | repo = _make_repo(tmp_path) |
| 154 | snap = _make_snapshot(repo, {"src/a.py": b"content_a"}) |
| 155 | _make_commit(repo, snap.snapshot_id) |
| 156 | reachable = _collect_reachable_objects(repo) |
| 157 | for oid in snap.manifest.values(): |
| 158 | assert oid in reachable, f"Object {oid[:8]} should be reachable" |
| 159 | |
| 160 | def test_corrupt_snapshot_id_objects_still_retained(self, tmp_path: pathlib.Path) -> None: |
| 161 | """BUG: When commit's snapshot_id is corrupt, objects are not retained.""" |
| 162 | repo = _make_repo(tmp_path) |
| 163 | snap = _make_snapshot(repo, {"src/main.py": b"important data"}) |
| 164 | commit = _make_commit(repo, snap.snapshot_id) |
| 165 | |
| 166 | # Corrupt the snapshot_id to a non-existent value |
| 167 | _corrupt_commit_snapshot_id(repo, commit.commit_id, "f" * 64) |
| 168 | |
| 169 | # Verify that read_snapshot now fails for the commit |
| 170 | stored = _collect_reachable_objects.__module__ # just to check we're testing the right thing |
| 171 | |
| 172 | reachable = _collect_reachable_objects(repo) |
| 173 | for oid in snap.manifest.values(): |
| 174 | assert oid in reachable, ( |
| 175 | f"DATA LOSS: Object {oid[:8]} from snapshot {snap.snapshot_id[:8]} " |
| 176 | f"was NOT retained by GC after the commit's snapshot_id was corrupted. " |
| 177 | f"Running muse gc would delete this object permanently." |
| 178 | ) |
| 179 | |
| 180 | def test_corrupt_snapshot_id_points_to_wrong_existing_snapshot(self, tmp_path: pathlib.Path) -> None: |
| 181 | """Even worse: corrupt snapshot_id points to a DIFFERENT existing snapshot. |
| 182 | The objects from the ORIGINAL snapshot are still not retained. |
| 183 | """ |
| 184 | repo = _make_repo(tmp_path) |
| 185 | snap1 = _make_snapshot(repo, {"src/file1.py": b"content 1"}) |
| 186 | snap2 = _make_snapshot(repo, {"src/file2.py": b"content 2"}) |
| 187 | commit = _make_commit(repo, snap1.snapshot_id) |
| 188 | |
| 189 | # Corrupt: now points to snap2's ID instead of snap1's |
| 190 | _corrupt_commit_snapshot_id(repo, commit.commit_id, snap2.snapshot_id) |
| 191 | |
| 192 | reachable = _collect_reachable_objects(repo) |
| 193 | for oid in snap1.manifest.values(): |
| 194 | assert oid in reachable, ( |
| 195 | f"DATA LOSS: Object from snap1 ({oid[:8]}) not retained — " |
| 196 | f"corrupt snapshot_id pointed to snap2 instead, and snap1's " |
| 197 | f"objects were not retained. GC would delete them." |
| 198 | ) |
| 199 | |
| 200 | def test_two_commits_one_corrupt_all_objects_retained(self, tmp_path: pathlib.Path) -> None: |
| 201 | """One corrupt commit must not prevent the other commit's objects from being retained.""" |
| 202 | repo = _make_repo(tmp_path) |
| 203 | snap1 = _make_snapshot(repo, {"a.py": b"aaa"}) |
| 204 | snap2 = _make_snapshot(repo, {"b.py": b"bbb"}) |
| 205 | commit1 = _make_commit(repo, snap1.snapshot_id, message="c1") |
| 206 | _make_commit(repo, snap2.snapshot_id, message="c2") |
| 207 | |
| 208 | _corrupt_commit_snapshot_id(repo, commit1.commit_id, "0" * 64) |
| 209 | |
| 210 | reachable = _collect_reachable_objects(repo) |
| 211 | # Both snapshots' objects must be retained |
| 212 | for oid in snap1.manifest.values(): |
| 213 | assert oid in reachable, f"snap1 object {oid[:8]} not retained after corruption" |
| 214 | for oid in snap2.manifest.values(): |
| 215 | assert oid in reachable, f"snap2 object {oid[:8]} not retained" |
| 216 | |
| 217 | def test_empty_store_no_crash(self, tmp_path: pathlib.Path) -> None: |
| 218 | repo = _make_repo(tmp_path) |
| 219 | reachable = _collect_reachable_objects(repo) |
| 220 | assert reachable == set() |
| 221 | |
| 222 | def test_valid_commit_chain_all_retained(self, tmp_path: pathlib.Path) -> None: |
| 223 | repo = _make_repo(tmp_path) |
| 224 | snap1 = _make_snapshot(repo, {"a.py": b"v1"}) |
| 225 | snap2 = _make_snapshot(repo, {"a.py": b"v2"}) |
| 226 | c1 = _make_commit(repo, snap1.snapshot_id, message="v1") |
| 227 | _make_commit(repo, snap2.snapshot_id, message="v2", parent=c1.commit_id) |
| 228 | |
| 229 | reachable = _collect_reachable_objects(repo) |
| 230 | for oid in snap1.manifest.values(): |
| 231 | assert oid in reachable |
| 232 | for oid in snap2.manifest.values(): |
| 233 | assert oid in reachable |
| 234 | |
| 235 | |
| 236 | # ────────────────────────────────────────────────────────────────────────────── |
| 237 | # Integration: run_gc with corrupt commit |
| 238 | # ────────────────────────────────────────────────────────────────────────────── |
| 239 | |
| 240 | class TestRunGcCorruptCommit: |
| 241 | |
| 242 | def test_gc_dry_run_reports_no_collected_objects_for_corrupt_commit(self, tmp_path: pathlib.Path) -> None: |
| 243 | """Dry run must show 0 objects to collect when all objects are reachable.""" |
| 244 | repo = _make_repo(tmp_path) |
| 245 | snap = _make_snapshot(repo, {"main.py": b"code"}) |
| 246 | commit = _make_commit(repo, snap.snapshot_id) |
| 247 | _corrupt_commit_snapshot_id(repo, commit.commit_id) |
| 248 | |
| 249 | result = run_gc(repo, dry_run=True, grace_period_seconds=0) |
| 250 | assert result.collected_count == 0, ( |
| 251 | f"BUG: dry_run GC reports {result.collected_count} objects to collect, " |
| 252 | f"but all objects are reachable (just via a corrupt commit). " |
| 253 | f"Running without dry_run would permanently delete these objects." |
| 254 | ) |
| 255 | |
| 256 | def test_gc_does_not_delete_objects_from_corrupt_commit(self, tmp_path: pathlib.Path) -> None: |
| 257 | """Objects from a corrupt commit's snapshot must survive GC.""" |
| 258 | repo = _make_repo(tmp_path) |
| 259 | snap = _make_snapshot(repo, {"main.py": b"valuable content"}) |
| 260 | commit = _make_commit(repo, snap.snapshot_id) |
| 261 | _corrupt_commit_snapshot_id(repo, commit.commit_id) |
| 262 | |
| 263 | result = run_gc(repo, dry_run=False, grace_period_seconds=0) |
| 264 | assert result.collected_count == 0, ( |
| 265 | f"DATA LOSS: GC deleted {result.collected_count} object(s) that were " |
| 266 | f"reachable from a commit with a corrupt snapshot_id. Those objects " |
| 267 | f"are now permanently gone." |
| 268 | ) |
| 269 | # Verify the object is still on disk |
| 270 | oid = list(snap.manifest.values())[0] |
| 271 | from muse.core.object_store import read_object |
| 272 | assert read_object(repo, oid) is not None, ( |
| 273 | f"CONFIRMED DATA LOSS: Object {oid[:8]} was deleted by GC." |
| 274 | ) |
| 275 | |
| 276 | def test_gc_full_retains_objects_when_corrupt_snapshot_file_exists(self, tmp_path: pathlib.Path) -> None: |
| 277 | """GC --full must retain objects when a snapshot FILE exists at the correct |
| 278 | path but its stored snapshot_id field doesn't match the computed hash. |
| 279 | The commit references snap_id S1; the file at S1 has a corrupt snapshot_id |
| 280 | field (not the manifest) so _verify_snapshot_id fails. Our raw-fallback |
| 281 | path must read the manifest directly and retain its object IDs. |
| 282 | """ |
| 283 | repo = _make_repo(tmp_path) |
| 284 | snap = _make_snapshot(repo, {"main.py": b"important"}) |
| 285 | commit = _make_commit(repo, snap.snapshot_id) |
| 286 | |
| 287 | # Point the branch ref at the commit (making it reachable) |
| 288 | heads_dir = repo / ".muse" / "refs" / "heads" |
| 289 | heads_dir.mkdir(parents=True, exist_ok=True) |
| 290 | (heads_dir / "main").write_text(commit.commit_id) |
| 291 | |
| 292 | # Corrupt the SNAPSHOT FILE's stored snapshot_id field (NOT the manifest). |
| 293 | # The manifest still has the correct object IDs. |
| 294 | # _verify_snapshot_id will fail (recomputed hash != stored snapshot_id field), |
| 295 | # causing read_snapshot to return None — but the object IDs in the manifest |
| 296 | # are still valid and should be retained by our raw-fallback path. |
| 297 | snap_path = repo / ".muse" / "snapshots" / f"{snap.snapshot_id}.msgpack" |
| 298 | snap_data = msgpack.unpackb(snap_path.read_bytes(), raw=False) |
| 299 | oid = list(snap_data["manifest"].values())[0] # save the real object ID |
| 300 | snap_data["snapshot_id"] = "corrupt_id_" + "0" * 53 # corrupt the stored ID field |
| 301 | snap_path.write_bytes(msgpack.packb(snap_data, use_bin_type=True)) |
| 302 | |
| 303 | result = run_gc(repo, dry_run=False, grace_period_seconds=0, full=True) |
| 304 | from muse.core.object_store import read_object |
| 305 | obj = read_object(repo, oid) |
| 306 | assert obj is not None, ( |
| 307 | f"DATA LOSS: GC --full deleted object {oid[:8]} that was referenced " |
| 308 | f"in a corrupt snapshot file (corrupt stored snapshot_id field, " |
| 309 | f"valid manifest). The raw-fallback path must have retained it." |
| 310 | ) |
| 311 | |
| 312 | def test_gc_full_corrupt_commit_snapshot_id_no_file_no_crash(self, tmp_path: pathlib.Path) -> None: |
| 313 | """Document the known edge case: if a commit's snapshot_id is corrupt AND |
| 314 | the referenced file doesn't exist, GC --full cannot retain those objects. |
| 315 | Users must run `muse verify-pack` before `muse gc --full` in this scenario. |
| 316 | This test verifies no crash occurs (the behavior is a known limitation). |
| 317 | """ |
| 318 | repo = _make_repo(tmp_path) |
| 319 | snap = _make_snapshot(repo, {"main.py": b"important"}) |
| 320 | commit = _make_commit(repo, snap.snapshot_id) |
| 321 | |
| 322 | heads_dir = repo / ".muse" / "refs" / "heads" |
| 323 | heads_dir.mkdir(parents=True, exist_ok=True) |
| 324 | (heads_dir / "main").write_text(commit.commit_id) |
| 325 | |
| 326 | # Corrupt the commit so its snapshot_id points to a non-existent file |
| 327 | _corrupt_commit_snapshot_id(repo, commit.commit_id, "f" * 64) |
| 328 | |
| 329 | # Known limitation: when commit.snapshot_id points to a non-existent file, |
| 330 | # GC --full cannot determine which objects to retain. No crash must occur. |
| 331 | result = run_gc(repo, dry_run=False, grace_period_seconds=0, full=True) |
| 332 | # No assertion about objects — this is the documented limitation. |
| 333 | |
| 334 | def test_gc_still_collects_truly_orphaned_objects(self, tmp_path: pathlib.Path) -> None: |
| 335 | """Regression: GC must still delete truly unreachable objects.""" |
| 336 | repo = _make_repo(tmp_path) |
| 337 | # Write an object that is NOT in any snapshot |
| 338 | orphan_content = b"orphaned content - no snapshot references this" |
| 339 | orphan_oid = _write_object(repo, orphan_content) |
| 340 | |
| 341 | # Write a valid commit with a snapshot that does NOT reference the orphan |
| 342 | snap = _make_snapshot(repo, {"other.py": b"other content"}) |
| 343 | _make_commit(repo, snap.snapshot_id) |
| 344 | |
| 345 | result = run_gc(repo, dry_run=False, grace_period_seconds=0) |
| 346 | assert orphan_oid in result.collected_ids, ( |
| 347 | f"Orphaned object {orphan_oid[:8]} was not collected by GC. " |
| 348 | f"GC is too conservative." |
| 349 | ) |
| 350 | |
| 351 | |
| 352 | # ────────────────────────────────────────────────────────────────────────────── |
| 353 | # Stress |
| 354 | # ────────────────────────────────────────────────────────────────────────────── |
| 355 | |
| 356 | class TestGcCorruptStress: |
| 357 | |
| 358 | def test_50_commits_5_corrupt_all_objects_retained(self, tmp_path: pathlib.Path) -> None: |
| 359 | """50 commits, 5 with corrupt snapshot_ids: all objects retained, no crash.""" |
| 360 | repo = _make_repo(tmp_path) |
| 361 | commit_records = [] |
| 362 | all_oids: set[str] = set() |
| 363 | |
| 364 | for i in range(50): |
| 365 | content = f"content_{i}".encode() |
| 366 | snap = _make_snapshot(repo, {f"f{i}.py": content}) |
| 367 | ts = _TS + datetime.timedelta(seconds=i) |
| 368 | commit = _make_commit(repo, snap.snapshot_id, message=f"commit {i}", ts=ts) |
| 369 | commit_records.append(commit) |
| 370 | all_oids.update(snap.manifest.values()) |
| 371 | |
| 372 | # Corrupt 5 commits (indices 10, 20, 30, 40, 49) |
| 373 | corrupt_indices = {10, 20, 30, 40, 49} |
| 374 | for idx in corrupt_indices: |
| 375 | _corrupt_commit_snapshot_id( |
| 376 | repo, commit_records[idx].commit_id, "9" * 64 |
| 377 | ) |
| 378 | |
| 379 | reachable = _collect_reachable_objects(repo) |
| 380 | |
| 381 | missing = [oid for oid in all_oids if oid not in reachable] |
| 382 | assert not missing, ( |
| 383 | f"DATA LOSS: {len(missing)} object(s) not retained by GC despite " |
| 384 | f"being reachable from snapshots on disk. " |
| 385 | f"Missing: {[o[:8] for o in missing[:5]]}" |
| 386 | ) |
File History
5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62
baseline: rc14 re-baseline after rc3 store corruption recovery
Human
patch
26 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e
Merge branch 'dev' into main
Human
29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce
Merge branch 'dev' into main
Human
48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa
feat: Muse — version control for the agent era
Human
100 days ago