test_integrity_I6_snapshot_scale.py
python
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
| 1 | """Phase 1.6 — Linux-kernel scale snapshot manifest. |
| 2 | |
| 3 | Tests cover: |
| 4 | - 75 000-file cold walk correctness and timing (< 30 s on tmpfs) |
| 5 | - Warm walk (cache hit, no changes) < 500 ms |
| 6 | - Partial change: only modified files re-hashed |
| 7 | - Memory ceiling: RSS stays flat during large-file streaming |
| 8 | - Deep directory nesting (100 levels) — no recursion errors |
| 9 | - Inode-based cache invalidation: atomic file replacement detected |
| 10 | - Concurrent walks: two processes walking simultaneously |
| 11 | - Cache size limit (MAX_CACHE_BYTES) enforced on load |
| 12 | - Cache format: msgpack v2 on disk |
| 13 | - Cache fsync + mkstemp atomicity: no fixed .tmp collision |
| 14 | - Deleted-file prune: stale cache entries removed after walk |
| 15 | - Empty repo: no crash, empty manifest |
| 16 | - Path safety: symlinks excluded, non-regular files excluded |
| 17 | """ |
| 18 | |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import hashlib |
| 22 | import json |
| 23 | import os |
| 24 | import pathlib |
| 25 | import resource |
| 26 | import stat |
| 27 | import tempfile |
| 28 | import threading |
| 29 | import time |
| 30 | |
| 31 | import msgpack |
| 32 | import pytest |
| 33 | |
| 34 | from muse.core.snapshot import build_snapshot_manifest, walk_workdir |
| 35 | from muse.core.stat_cache import ( |
| 36 | MAX_CACHE_BYTES, |
| 37 | FileCacheEntry, |
| 38 | StatCache, |
| 39 | _CACHE_FILENAME, |
| 40 | _CACHE_VERSION, |
| 41 | load_cache, |
| 42 | ) |
| 43 | |
| 44 | # --------------------------------------------------------------------------- |
| 45 | # Helpers |
| 46 | # --------------------------------------------------------------------------- |
| 47 | |
| 48 | def _make_muse_dir(root: pathlib.Path) -> pathlib.Path: |
| 49 | d = root / ".muse" |
| 50 | d.mkdir(exist_ok=True) |
| 51 | return d |
| 52 | |
| 53 | |
| 54 | def _write(path: pathlib.Path, content: str = "x") -> pathlib.Path: |
| 55 | path.parent.mkdir(parents=True, exist_ok=True) |
| 56 | path.write_text(content, encoding="utf-8") |
| 57 | return path |
| 58 | |
| 59 | |
| 60 | def _sha(text: str) -> str: |
| 61 | return hashlib.sha256(text.encode()).hexdigest() |
| 62 | |
| 63 | |
| 64 | def _create_files_flat(root: pathlib.Path, n: int, size_bytes: int = 256) -> None: |
| 65 | """Create *n* files under *root* with *size_bytes* of content each.""" |
| 66 | content = b"x" * size_bytes |
| 67 | for i in range(n): |
| 68 | p = root / f"file_{i:06d}.txt" |
| 69 | p.write_bytes(content) |
| 70 | |
| 71 | |
| 72 | def _create_files_deep(root: pathlib.Path, depth: int, files_per_level: int = 2) -> None: |
| 73 | """Create a directory tree *depth* levels deep with files at each level.""" |
| 74 | cur = root |
| 75 | for level in range(depth): |
| 76 | cur = cur / f"d{level:03d}" |
| 77 | cur.mkdir(exist_ok=True) |
| 78 | for f in range(files_per_level): |
| 79 | (cur / f"f{f}.txt").write_bytes(b"level" + str(level).encode()) |
| 80 | |
| 81 | |
| 82 | # --------------------------------------------------------------------------- |
| 83 | # 1. Format — msgpack v2 on disk |
| 84 | # --------------------------------------------------------------------------- |
| 85 | |
| 86 | class TestCacheFormat: |
| 87 | def test_cache_file_is_msgpack(self, tmp_path: pathlib.Path) -> None: |
| 88 | muse_dir = _make_muse_dir(tmp_path) |
| 89 | f = _write(tmp_path / "a.py", "hello") |
| 90 | cache = StatCache.load(muse_dir) |
| 91 | cache.get_object_hash(tmp_path, f) |
| 92 | cache.save() |
| 93 | |
| 94 | cache_path = muse_dir / _CACHE_FILENAME |
| 95 | assert cache_path.is_file() |
| 96 | assert cache_path.suffix == ".msgpack" |
| 97 | raw = msgpack.unpackb(cache_path.read_bytes(), raw=False) |
| 98 | assert raw["version"] == _CACHE_VERSION |
| 99 | assert _CACHE_VERSION == 2 |
| 100 | |
| 101 | def test_cache_entry_has_ino_field(self, tmp_path: pathlib.Path) -> None: |
| 102 | muse_dir = _make_muse_dir(tmp_path) |
| 103 | f = _write(tmp_path / "b.py", "world") |
| 104 | cache = StatCache.load(muse_dir) |
| 105 | cache.get_object_hash(tmp_path, f) |
| 106 | cache.save() |
| 107 | |
| 108 | raw = msgpack.unpackb((muse_dir / _CACHE_FILENAME).read_bytes(), raw=False) |
| 109 | entry = raw["entries"]["b.py"] |
| 110 | assert "ino" in entry |
| 111 | assert isinstance(entry["ino"], int) |
| 112 | assert entry["ino"] > 0 |
| 113 | |
| 114 | def test_version_mismatch_returns_empty(self, tmp_path: pathlib.Path) -> None: |
| 115 | muse_dir = _make_muse_dir(tmp_path) |
| 116 | # Write a v99 cache — should be discarded |
| 117 | bad = msgpack.packb({"version": 99, "entries": {}}) |
| 118 | (muse_dir / _CACHE_FILENAME).write_bytes(bad) |
| 119 | cache = StatCache.load(muse_dir) |
| 120 | assert len(cache._entries) == 0 |
| 121 | |
| 122 | def test_corrupt_cache_returns_empty(self, tmp_path: pathlib.Path) -> None: |
| 123 | muse_dir = _make_muse_dir(tmp_path) |
| 124 | (muse_dir / _CACHE_FILENAME).write_bytes(b"\xff\x00garbage\xde\xad") |
| 125 | cache = StatCache.load(muse_dir) |
| 126 | assert len(cache._entries) == 0 |
| 127 | |
| 128 | def test_absent_cache_returns_empty(self, tmp_path: pathlib.Path) -> None: |
| 129 | muse_dir = _make_muse_dir(tmp_path) |
| 130 | cache = StatCache.load(muse_dir) |
| 131 | assert len(cache._entries) == 0 |
| 132 | |
| 133 | |
| 134 | # --------------------------------------------------------------------------- |
| 135 | # 2. Inode-based invalidation |
| 136 | # --------------------------------------------------------------------------- |
| 137 | |
| 138 | class TestInodeInvalidation: |
| 139 | def test_atomic_replace_invalidates_cache(self, tmp_path: pathlib.Path) -> None: |
| 140 | """Atomically replacing a file (same mtime/size) is detected via new inode.""" |
| 141 | muse_dir = _make_muse_dir(tmp_path) |
| 142 | f = tmp_path / "data.bin" |
| 143 | content_a = b"A" * 64 |
| 144 | content_b = b"B" * 64 # same size as content_a |
| 145 | f.write_bytes(content_a) |
| 146 | |
| 147 | cache = StatCache.load(muse_dir) |
| 148 | hash_a = cache.get_object_hash(tmp_path, f) |
| 149 | cache.save() |
| 150 | |
| 151 | # Atomically replace with different content (same size). |
| 152 | # Force mtime to be identical (same second) by setting it manually |
| 153 | # after the write — this simulates the NFS/tmpfs scenario. |
| 154 | with tempfile.NamedTemporaryFile(dir=tmp_path, delete=False) as tmp: |
| 155 | tmp.write(content_b) |
| 156 | tmp_name = tmp.name |
| 157 | orig_stat = f.stat() |
| 158 | os.replace(tmp_name, str(f)) |
| 159 | # Restore original mtime to simulate a racy replacement |
| 160 | os.utime(str(f), (orig_stat.st_atime, orig_stat.st_mtime)) |
| 161 | |
| 162 | # The new file has a different inode — cache must miss |
| 163 | cache2 = StatCache.load(muse_dir) |
| 164 | hash_b = cache2.get_object_hash(tmp_path, f) |
| 165 | |
| 166 | assert hash_a != hash_b, ( |
| 167 | "Cache returned stale hash after atomic replacement — " |
| 168 | "inode invalidation is not working" |
| 169 | ) |
| 170 | |
| 171 | def test_mtime_size_same_but_ino_changed_is_miss( |
| 172 | self, tmp_path: pathlib.Path |
| 173 | ) -> None: |
| 174 | """If ino changes, cache must invalidate even with same mtime+size.""" |
| 175 | muse_dir = _make_muse_dir(tmp_path) |
| 176 | f = tmp_path / "tricky.py" |
| 177 | f.write_bytes(b"old_content_pad") # 15 bytes |
| 178 | |
| 179 | cache = StatCache.load(muse_dir) |
| 180 | st = f.stat() |
| 181 | old_hash = cache.get_cached("tricky.py", str(f), st.st_mtime, st.st_size, st.st_ino) |
| 182 | cache.save() |
| 183 | |
| 184 | # Simulate a different inode by using a fresh inode value |
| 185 | fake_ino = st.st_ino + 999_999 |
| 186 | cache2 = StatCache.load(muse_dir) |
| 187 | # Should miss because ino doesn't match — forces re-hash |
| 188 | entry = cache2._entries.get("tricky.py") |
| 189 | assert entry is not None |
| 190 | assert entry["ino"] != fake_ino |
| 191 | # Direct get_cached with wrong ino triggers miss |
| 192 | new_hash = cache2.get_cached("tricky.py", str(f), st.st_mtime, st.st_size, fake_ino) |
| 193 | # The content hash is still the same (same file contents) |
| 194 | assert new_hash == old_hash # same file content |
| 195 | assert cache2._dirty # but it was a miss (dirty) |
| 196 | |
| 197 | def test_unchanged_file_is_cache_hit(self, tmp_path: pathlib.Path) -> None: |
| 198 | muse_dir = _make_muse_dir(tmp_path) |
| 199 | f = _write(tmp_path / "stable.py", "unchanged") |
| 200 | cache = StatCache.load(muse_dir) |
| 201 | h1 = cache.get_object_hash(tmp_path, f) |
| 202 | cache.save() |
| 203 | |
| 204 | cache2 = StatCache.load(muse_dir) |
| 205 | cache2._dirty = False |
| 206 | h2 = cache2.get_object_hash(tmp_path, f) |
| 207 | |
| 208 | assert h1 == h2 |
| 209 | assert not cache2._dirty # genuine cache hit |
| 210 | |
| 211 | |
| 212 | # --------------------------------------------------------------------------- |
| 213 | # 3. Concurrent write safety (mkstemp — no .tmp collision) |
| 214 | # --------------------------------------------------------------------------- |
| 215 | |
| 216 | class TestConcurrentSaveSafety: |
| 217 | def test_concurrent_saves_no_collision(self, tmp_path: pathlib.Path) -> None: |
| 218 | """Two threads saving simultaneously must not corrupt each other.""" |
| 219 | muse_dir = _make_muse_dir(tmp_path) |
| 220 | errors: list[Exception] = [] |
| 221 | |
| 222 | def save_cache(i: int) -> None: |
| 223 | f = _write(tmp_path / f"thread_{i}.py", f"content {i}") |
| 224 | cache = StatCache.load(muse_dir) |
| 225 | cache.get_object_hash(tmp_path, f) |
| 226 | try: |
| 227 | cache.save() |
| 228 | except Exception as exc: |
| 229 | errors.append(exc) |
| 230 | |
| 231 | threads = [threading.Thread(target=save_cache, args=(i,)) for i in range(20)] |
| 232 | for t in threads: |
| 233 | t.start() |
| 234 | for t in threads: |
| 235 | t.join() |
| 236 | |
| 237 | assert not errors, f"Concurrent saves failed: {errors[:3]}" |
| 238 | # No stray .tmp files left behind |
| 239 | tmp_files = list(muse_dir.glob(".stat_cache_*.tmp")) |
| 240 | assert not tmp_files, f"Stray temp files: {tmp_files}" |
| 241 | |
| 242 | def test_no_fixed_tmp_suffix(self, tmp_path: pathlib.Path) -> None: |
| 243 | """save() must NOT create a file named stat_cache.msgpack.tmp.""" |
| 244 | muse_dir = _make_muse_dir(tmp_path) |
| 245 | f = _write(tmp_path / "x.py", "hi") |
| 246 | cache = StatCache.load(muse_dir) |
| 247 | cache.get_object_hash(tmp_path, f) |
| 248 | cache.save() |
| 249 | |
| 250 | fixed_tmp = muse_dir / (str(_CACHE_FILENAME) + ".tmp") |
| 251 | assert not fixed_tmp.exists(), "Fixed .tmp name — concurrent save race possible" |
| 252 | |
| 253 | |
| 254 | # --------------------------------------------------------------------------- |
| 255 | # 4. Cache size limit (MAX_CACHE_BYTES) |
| 256 | # --------------------------------------------------------------------------- |
| 257 | |
| 258 | class TestCacheSizeLimit: |
| 259 | def test_max_cache_bytes_is_exported(self) -> None: |
| 260 | assert isinstance(MAX_CACHE_BYTES, int) |
| 261 | assert MAX_CACHE_BYTES >= 64 * 1024 * 1024 |
| 262 | |
| 263 | def test_oversized_cache_returns_empty( |
| 264 | self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture |
| 265 | ) -> None: |
| 266 | import logging |
| 267 | muse_dir = _make_muse_dir(tmp_path) |
| 268 | # Write a valid cache but patch the size limit to reject it |
| 269 | f = _write(tmp_path / "z.py", "z" * 1000) |
| 270 | cache = StatCache.load(muse_dir) |
| 271 | cache.get_object_hash(tmp_path, f) |
| 272 | cache.save() |
| 273 | |
| 274 | real_size = (muse_dir / _CACHE_FILENAME).stat().st_size |
| 275 | import unittest.mock as _mock |
| 276 | with _mock.patch("muse.core.stat_cache.MAX_CACHE_BYTES", real_size - 1): |
| 277 | with caplog.at_level(logging.CRITICAL, logger="muse.core.stat_cache"): |
| 278 | loaded = StatCache.load(muse_dir) |
| 279 | assert len(loaded._entries) == 0 |
| 280 | assert any(r.levelno >= logging.CRITICAL for r in caplog.records) |
| 281 | |
| 282 | def test_exactly_at_limit_is_accepted(self, tmp_path: pathlib.Path) -> None: |
| 283 | muse_dir = _make_muse_dir(tmp_path) |
| 284 | f = _write(tmp_path / "y.py", "y") |
| 285 | cache = StatCache.load(muse_dir) |
| 286 | cache.get_object_hash(tmp_path, f) |
| 287 | cache.save() |
| 288 | |
| 289 | real_size = (muse_dir / _CACHE_FILENAME).stat().st_size |
| 290 | import unittest.mock as _mock |
| 291 | with _mock.patch("muse.core.stat_cache.MAX_CACHE_BYTES", real_size): |
| 292 | loaded = StatCache.load(muse_dir) |
| 293 | assert len(loaded._entries) == 1 |
| 294 | |
| 295 | |
| 296 | # --------------------------------------------------------------------------- |
| 297 | # 5. Prune: deleted files evicted from cache |
| 298 | # --------------------------------------------------------------------------- |
| 299 | |
| 300 | class TestPruneAfterWalk: |
| 301 | def test_deleted_file_pruned_on_walk(self, tmp_path: pathlib.Path) -> None: |
| 302 | muse_dir = _make_muse_dir(tmp_path) |
| 303 | fa = _write(tmp_path / "keep.py", "keep") |
| 304 | fb = _write(tmp_path / "delete_me.py", "bye") |
| 305 | |
| 306 | # First walk — both files cached |
| 307 | walk_workdir(tmp_path) |
| 308 | cache = StatCache.load(muse_dir) |
| 309 | assert "keep.py" in cache._entries |
| 310 | assert "delete_me.py" in cache._entries |
| 311 | |
| 312 | # Delete one file then re-walk |
| 313 | fb.unlink() |
| 314 | walk_workdir(tmp_path) |
| 315 | |
| 316 | cache2 = StatCache.load(muse_dir) |
| 317 | assert "keep.py" in cache2._entries |
| 318 | assert "delete_me.py" not in cache2._entries, "Stale cache entry not pruned" |
| 319 | |
| 320 | |
| 321 | # --------------------------------------------------------------------------- |
| 322 | # 6. Path safety |
| 323 | # --------------------------------------------------------------------------- |
| 324 | |
| 325 | class TestPathSafety: |
| 326 | def test_symlinks_excluded_from_manifest(self, tmp_path: pathlib.Path) -> None: |
| 327 | _make_muse_dir(tmp_path) |
| 328 | real = _write(tmp_path / "real.py", "real") |
| 329 | link = tmp_path / "link.py" |
| 330 | link.symlink_to(real) |
| 331 | |
| 332 | manifest = build_snapshot_manifest(tmp_path) |
| 333 | assert "real.py" in manifest |
| 334 | assert "link.py" not in manifest, "Symlinks must be excluded" |
| 335 | |
| 336 | def test_non_regular_files_excluded(self, tmp_path: pathlib.Path) -> None: |
| 337 | _make_muse_dir(tmp_path) |
| 338 | _write(tmp_path / "normal.py", "ok") |
| 339 | # Create a FIFO (named pipe) — non-regular file |
| 340 | fifo = tmp_path / "pipe.fifo" |
| 341 | os.mkfifo(str(fifo)) |
| 342 | |
| 343 | manifest = build_snapshot_manifest(tmp_path) |
| 344 | assert "normal.py" in manifest |
| 345 | assert "pipe.fifo" not in manifest |
| 346 | |
| 347 | def test_muse_dir_excluded_from_manifest(self, tmp_path: pathlib.Path) -> None: |
| 348 | _make_muse_dir(tmp_path) |
| 349 | _write(tmp_path / "real.py", "code") |
| 350 | |
| 351 | manifest = build_snapshot_manifest(tmp_path) |
| 352 | assert not any(k.startswith(".muse/") for k in manifest) |
| 353 | |
| 354 | |
| 355 | # --------------------------------------------------------------------------- |
| 356 | # 7. Warm-walk no-op: second commit on unchanged tree is fast |
| 357 | # --------------------------------------------------------------------------- |
| 358 | |
| 359 | class TestWarmWalkPerformance: |
| 360 | def test_warm_walk_uses_cache_no_rehash(self, tmp_path: pathlib.Path) -> None: |
| 361 | """Unchanged files must not be re-hashed on the second walk.""" |
| 362 | _make_muse_dir(tmp_path) |
| 363 | for i in range(100): |
| 364 | _write(tmp_path / f"f{i}.py", f"content {i}") |
| 365 | |
| 366 | # Cold walk — populates cache |
| 367 | walk_workdir(tmp_path) |
| 368 | |
| 369 | # Warm walk — should be all cache hits |
| 370 | cache_before = StatCache.load(tmp_path / ".muse") |
| 371 | count_before = len(cache_before._entries) |
| 372 | |
| 373 | walk_workdir(tmp_path) |
| 374 | |
| 375 | # After warm walk, cache entries are unchanged (same count, not dirty) |
| 376 | cache_after = StatCache.load(tmp_path / ".muse") |
| 377 | assert len(cache_after._entries) == count_before |
| 378 | |
| 379 | @pytest.mark.slow |
| 380 | def test_warm_walk_500ms_on_1000_files(self, tmp_path: pathlib.Path) -> None: |
| 381 | """1000-file warm walk must complete in < 500 ms.""" |
| 382 | _make_muse_dir(tmp_path) |
| 383 | for i in range(1000): |
| 384 | _write(tmp_path / f"warm_{i:04d}.py", f"content {i} " * 10) |
| 385 | |
| 386 | walk_workdir(tmp_path) # cold |
| 387 | |
| 388 | t0 = time.perf_counter() |
| 389 | walk_workdir(tmp_path) # warm |
| 390 | elapsed = time.perf_counter() - t0 |
| 391 | |
| 392 | assert elapsed < 0.5, ( |
| 393 | f"Warm walk over 1000 files took {elapsed:.3f}s — must be < 500 ms. " |
| 394 | "Cache is not being consulted." |
| 395 | ) |
| 396 | |
| 397 | |
| 398 | # --------------------------------------------------------------------------- |
| 399 | # 8. Deep directory nesting — no recursion errors |
| 400 | # --------------------------------------------------------------------------- |
| 401 | |
| 402 | class TestDeepNesting: |
| 403 | def test_100_level_nesting_no_error(self, tmp_path: pathlib.Path) -> None: |
| 404 | """os.walk must not hit recursion limit at 100 directory levels.""" |
| 405 | _make_muse_dir(tmp_path) |
| 406 | _create_files_deep(tmp_path, depth=100, files_per_level=1) |
| 407 | |
| 408 | manifest = build_snapshot_manifest(tmp_path) |
| 409 | assert len(manifest) == 100, f"Expected 100 files, got {len(manifest)}" |
| 410 | |
| 411 | def test_50_level_nesting_correct_paths(self, tmp_path: pathlib.Path) -> None: |
| 412 | """Paths at deep levels must be POSIX-relative with correct separators.""" |
| 413 | _make_muse_dir(tmp_path) |
| 414 | cur = tmp_path |
| 415 | for i in range(50): |
| 416 | cur = cur / f"l{i}" |
| 417 | cur.mkdir() |
| 418 | leaf = cur / "deep.py" |
| 419 | leaf.write_bytes(b"deep") |
| 420 | |
| 421 | manifest = build_snapshot_manifest(tmp_path) |
| 422 | # Path uses forward slashes regardless of OS |
| 423 | assert any("deep.py" in k and "/" in k for k in manifest) |
| 424 | assert not any("\\" in k for k in manifest) |
| 425 | |
| 426 | |
| 427 | # --------------------------------------------------------------------------- |
| 428 | # 9. Large-file streaming — memory stays flat |
| 429 | # --------------------------------------------------------------------------- |
| 430 | |
| 431 | class TestLargeFileStreaming: |
| 432 | @pytest.mark.slow |
| 433 | def test_large_file_memory_flat(self, tmp_path: pathlib.Path) -> None: |
| 434 | """Hashing a 50 MiB file must not load it all into memory at once.""" |
| 435 | import sys as _sys |
| 436 | _make_muse_dir(tmp_path) |
| 437 | large = tmp_path / "large.bin" |
| 438 | chunk = b"Z" * 65_536 |
| 439 | with large.open("wb") as fh: |
| 440 | for _ in range(800): # 800 × 64 KiB = 50 MiB |
| 441 | fh.write(chunk) |
| 442 | |
| 443 | rss_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss |
| 444 | manifest = build_snapshot_manifest(tmp_path) |
| 445 | rss_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss |
| 446 | |
| 447 | assert "large.bin" in manifest |
| 448 | # ru_maxrss is bytes on macOS, KiB on Linux. |
| 449 | _scale = 1024 * 1024 if _sys.platform == "darwin" else 1024 |
| 450 | rss_delta_mib = (rss_after - rss_before) / _scale |
| 451 | # RSS growth must be < 10 MiB (file is 50 MiB — proves streaming) |
| 452 | assert rss_delta_mib < 10, ( |
| 453 | f"RSS grew {rss_delta_mib:.1f} MiB while hashing a 50 MiB file " |
| 454 | "— file is being loaded entirely into memory." |
| 455 | ) |
| 456 | |
| 457 | def test_zero_byte_file_hashes_correctly(self, tmp_path: pathlib.Path) -> None: |
| 458 | _make_muse_dir(tmp_path) |
| 459 | empty = tmp_path / "empty.py" |
| 460 | empty.write_bytes(b"") |
| 461 | |
| 462 | manifest = build_snapshot_manifest(tmp_path) |
| 463 | expected = hashlib.sha256(b"").hexdigest() |
| 464 | assert manifest["empty.py"] == expected |
| 465 | |
| 466 | |
| 467 | # --------------------------------------------------------------------------- |
| 468 | # 10. 75 000-file scale — cold walk correctness + warm walk timing |
| 469 | # --------------------------------------------------------------------------- |
| 470 | |
| 471 | @pytest.mark.slow |
| 472 | class TestLinuxKernelScale: |
| 473 | def test_75k_cold_walk_correct_and_bounded(self, tmp_path: pathlib.Path) -> None: |
| 474 | """75k files: cold walk must be correct and complete in < 30 s.""" |
| 475 | _make_muse_dir(tmp_path) |
| 476 | n = 75_000 |
| 477 | content = b"k" * 1024 # 1 KiB per file |
| 478 | |
| 479 | t_create = time.perf_counter() |
| 480 | for i in range(n): |
| 481 | subdir = tmp_path / f"d{i // 1000:03d}" |
| 482 | subdir.mkdir(exist_ok=True) |
| 483 | (subdir / f"f{i:06d}.c").write_bytes(content) |
| 484 | t_create_done = time.perf_counter() |
| 485 | |
| 486 | t_walk = time.perf_counter() |
| 487 | manifest = build_snapshot_manifest(tmp_path) |
| 488 | t_walk_done = time.perf_counter() |
| 489 | |
| 490 | walk_seconds = t_walk_done - t_walk |
| 491 | assert len(manifest) == n, f"Expected {n} files, got {len(manifest)}" |
| 492 | # All hashes must be the 1 KiB content hash (same content → same hash) |
| 493 | expected_hash = hashlib.sha256(content).hexdigest() |
| 494 | assert all(v == expected_hash for v in manifest.values()), ( |
| 495 | "Hash mismatch — some files were hashed incorrectly" |
| 496 | ) |
| 497 | assert walk_seconds < 30, ( |
| 498 | f"75k cold walk took {walk_seconds:.1f}s — must be < 30 s" |
| 499 | ) |
| 500 | |
| 501 | def test_75k_warm_walk_zero_misses(self, tmp_path: pathlib.Path) -> None: |
| 502 | """75k files: warm walk must produce zero cache misses. |
| 503 | |
| 504 | Uses ``_dirty`` as the oracle — if the cache has no misses after a |
| 505 | warm walk, ``_dirty`` remains False (no new hashes were computed). |
| 506 | This is platform-independent: it proves correctness regardless of |
| 507 | whether lstat or file-read latency dominates. |
| 508 | """ |
| 509 | _make_muse_dir(tmp_path) |
| 510 | n = 75_000 |
| 511 | content = b"w" * 512 |
| 512 | |
| 513 | for i in range(n): |
| 514 | subdir = tmp_path / f"d{i // 1000:03d}" |
| 515 | subdir.mkdir(exist_ok=True) |
| 516 | (subdir / f"f{i:06d}.go").write_bytes(content) |
| 517 | |
| 518 | build_snapshot_manifest(tmp_path) # cold — populates cache |
| 519 | |
| 520 | # Load the persisted cache and walk again; _dirty should stay False |
| 521 | # because every file matches its cached (ino, mtime, size). |
| 522 | cache = load_cache(tmp_path) |
| 523 | cache._dirty = False # reset to ensure we detect any miss |
| 524 | manifest2 = build_snapshot_manifest(tmp_path) |
| 525 | |
| 526 | assert len(manifest2) == n |
| 527 | # Reload the cache from disk — if anything was re-hashed during the |
| 528 | # warm walk, the on-disk cache will have grown entries (or been re-written). |
| 529 | # More directly: run walk_workdir manually with our cache instance. |
| 530 | from muse.core.snapshot import walk_workdir as _walk |
| 531 | muse_dir = tmp_path / ".muse" |
| 532 | cache2 = StatCache.load(muse_dir) |
| 533 | cache2._dirty = False |
| 534 | _ = _walk(tmp_path) # warm walk |
| 535 | # The on-disk cache must not have been re-written with any new entries |
| 536 | # (same inode, mtime, size → all cache hits → not dirty → no save) |
| 537 | assert not cache2._dirty, ( |
| 538 | "Warm walk marked cache dirty — at least one file was re-hashed. " |
| 539 | "The stat cache is not being consulted correctly at 75k-file scale." |
| 540 | ) |
| 541 | |
| 542 | def test_cache_speedup_with_large_files(self, tmp_path: pathlib.Path) -> None: |
| 543 | """Cache delivers ≥ 5× speedup when file-content I/O dominates. |
| 544 | |
| 545 | Uses 100 × 1 MiB files so cold-walk hashing clearly dominates lstat |
| 546 | overhead. Warm walk skips all file reads → dramatic speedup. |
| 547 | Platform-independent: cold is I/O bound; warm is stat-bound. |
| 548 | """ |
| 549 | _make_muse_dir(tmp_path) |
| 550 | content = b"L" * (1024 * 1024) # 1 MiB per file |
| 551 | for i in range(100): |
| 552 | (tmp_path / f"large_{i:03d}.bin").write_bytes(content) |
| 553 | |
| 554 | t_cold0 = time.perf_counter() |
| 555 | build_snapshot_manifest(tmp_path) # cold |
| 556 | t_cold = time.perf_counter() - t_cold0 |
| 557 | |
| 558 | t_warm0 = time.perf_counter() |
| 559 | build_snapshot_manifest(tmp_path) # warm |
| 560 | t_warm = time.perf_counter() - t_warm0 |
| 561 | |
| 562 | speedup = t_cold / t_warm if t_warm > 0 else float("inf") |
| 563 | assert speedup >= 5, ( |
| 564 | f"Warm walk ({t_warm:.3f}s) is only {speedup:.1f}× faster than " |
| 565 | f"cold ({t_cold:.3f}s) — expected ≥ 5×. " |
| 566 | "Cache must skip all file-content reads on warm walk." |
| 567 | ) |
| 568 | |
| 569 | def test_75k_partial_change_only_rehashes_changed( |
| 570 | self, tmp_path: pathlib.Path |
| 571 | ) -> None: |
| 572 | """After changing 10 files, only those 10 should trigger a cache miss.""" |
| 573 | _make_muse_dir(tmp_path) |
| 574 | n = 75_000 |
| 575 | content = b"p" * 256 |
| 576 | |
| 577 | paths: list[pathlib.Path] = [] |
| 578 | for i in range(n): |
| 579 | subdir = tmp_path / f"d{i // 1000:03d}" |
| 580 | subdir.mkdir(exist_ok=True) |
| 581 | p = subdir / f"f{i:06d}.rs" |
| 582 | p.write_bytes(content) |
| 583 | paths.append(p) |
| 584 | |
| 585 | build_snapshot_manifest(tmp_path) # cold |
| 586 | |
| 587 | # Modify 10 files |
| 588 | changed = paths[:10] |
| 589 | new_content = b"CHANGED" * 37 # different size ensures definite miss |
| 590 | for p in changed: |
| 591 | p.write_bytes(new_content) |
| 592 | |
| 593 | manifest2 = build_snapshot_manifest(tmp_path) |
| 594 | |
| 595 | new_hash = hashlib.sha256(new_content).hexdigest() |
| 596 | old_hash = hashlib.sha256(content).hexdigest() |
| 597 | changed_rels = {str(p.relative_to(tmp_path)).replace(os.sep, "/") for p in changed} |
| 598 | |
| 599 | for rel, h in manifest2.items(): |
| 600 | if rel in changed_rels: |
| 601 | assert h == new_hash, f"{rel} should have new hash" |
| 602 | else: |
| 603 | assert h == old_hash, f"{rel} should still have old hash" |
File History
5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62
baseline: rc14 re-baseline after rc3 store corruption recovery
Human
patch
26 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e
Merge branch 'dev' into main
Human
29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce
Merge branch 'dev' into main
Human
48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa
feat: Muse — version control for the agent era
Human
100 days ago