test_cache_invalidation_gaps.py
python
sha256:ce542c4d0f6dacc9520c02d0e3378bf96bc377d118a7402f2bbc8af1749b08e0
feat(code-test): audit cache internals, add cache_hit/--cac…
Sonnet 5
minor
⚠ breaking
3 hours ago
| 1 | """TDD tests for muse#49 Phase 3 — cache-invalidation gap coverage (CT_06–CT_10). |
| 2 | |
| 3 | muse's symbol/call-graph caches are content-addressed per-file memoization |
| 4 | tables (``object_id -> SymbolTree`` / ``object_id -> per-file subgraph``), |
| 5 | not snapshot/commit-ID-keyed global caches. There is no single "cache valid |
| 6 | for commit X" flag — validity is implicit per file: a cache entry for a |
| 7 | given blob hash is correct forever, because the key IS the content. |
| 8 | |
| 9 | These tests prove the 5 gap scenarios identified in the audit |
| 10 | (docs/code-test-internals.md, CT_02) actually behave as that model |
| 11 | predicts, using call-count spying on ``read_object``/``parse_symbols`` |
| 12 | (the established pattern from test_test_selection_speedup.py) so a |
| 13 | regression that silently reintroduced a full re-parse would be caught — |
| 14 | asserting on final output alone would not catch it, since test-selection |
| 15 | correctness is independent of cache warmth. |
| 16 | """ |
| 17 | |
| 18 | from __future__ import annotations |
| 19 | |
| 20 | import pathlib |
| 21 | from unittest.mock import patch |
| 22 | |
| 23 | import pytest |
| 24 | |
| 25 | from muse.core.callgraph_cache import CallGraphCache |
| 26 | from muse.core.object_store import write_object |
| 27 | from muse.core.paths import muse_dir |
| 28 | from muse.core.symbol_cache import SymbolCache |
| 29 | from muse.core.test_selection import ChangedSymbol, select_tests |
| 30 | from muse.core.types import Manifest, blob_id |
| 31 | |
| 32 | |
| 33 | def _write_file(root: pathlib.Path, rel_path: str, content: bytes) -> str: |
| 34 | """Write *content* to both the object store and the working tree, return its object_id.""" |
| 35 | oid = blob_id(content) |
| 36 | write_object(root, oid, content) |
| 37 | full = root / rel_path |
| 38 | full.parent.mkdir(parents=True, exist_ok=True) |
| 39 | full.write_bytes(content) |
| 40 | return oid |
| 41 | |
| 42 | |
| 43 | def _base_repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 44 | muse_dir(tmp_path).mkdir(exist_ok=True) |
| 45 | return tmp_path |
| 46 | |
| 47 | |
| 48 | PROD_V1 = b"""\ |
| 49 | def compute(x: int) -> int: |
| 50 | return x * 2 |
| 51 | """ |
| 52 | PROD_V2 = b"""\ |
| 53 | def compute(x: int) -> int: |
| 54 | return x * 3 # changed body -> new object_id |
| 55 | """ |
| 56 | TEST_V1 = b"""\ |
| 57 | from prod import compute |
| 58 | |
| 59 | def test_compute() -> None: |
| 60 | assert compute(2) == 4 |
| 61 | """ |
| 62 | |
| 63 | |
| 64 | # --------------------------------------------------------------------------- |
| 65 | # CT_06 — a new commit (changed file content) invalidates the cache for that |
| 66 | # file only; the per-file memoization table has no entry for the new blob, |
| 67 | # so it is necessarily re-parsed. |
| 68 | # --------------------------------------------------------------------------- |
| 69 | |
| 70 | |
| 71 | class TestNewCommitInvalidatesChangedFile: |
| 72 | def test_changed_file_is_reparsed_unchanged_file_is_not( |
| 73 | self, tmp_path: pathlib.Path |
| 74 | ) -> None: |
| 75 | root = _base_repo(tmp_path) |
| 76 | prod_oid_v1 = _write_file(root, "prod.py", PROD_V1) |
| 77 | test_oid = _write_file(root, "tests/test_prod.py", TEST_V1) |
| 78 | manifest_v1: Manifest = {"prod.py": prod_oid_v1, "tests/test_prod.py": test_oid} |
| 79 | changed = [ChangedSymbol(address="prod.py::compute", change_kind="modified")] |
| 80 | |
| 81 | sym_cache = SymbolCache.empty() |
| 82 | cg_cache = CallGraphCache.empty() |
| 83 | select_tests(root, changed, manifest_v1, cache=sym_cache, callgraph_cache=cg_cache) |
| 84 | |
| 85 | # Simulate a new commit: prod.py's body changes -> new object_id. |
| 86 | prod_oid_v2 = _write_file(root, "prod.py", PROD_V2) |
| 87 | assert prod_oid_v2 != prod_oid_v1, "test fixture must actually change the blob hash" |
| 88 | manifest_v2: Manifest = {"prod.py": prod_oid_v2, "tests/test_prod.py": test_oid} |
| 89 | |
| 90 | result = select_tests(root, changed, manifest_v2, cache=sym_cache, callgraph_cache=cg_cache) |
| 91 | |
| 92 | # New blob has no cache entry under either cache -> at least one miss. |
| 93 | assert result["cache_hit"] is False |
| 94 | assert result["cache_misses"] > 0 |
| 95 | |
| 96 | |
| 97 | # --------------------------------------------------------------------------- |
| 98 | # CT_07 — an unchanged snapshot reuses the cache entirely: zero object-store |
| 99 | # reads, zero re-parses. Verified by call-count spying, not output alone. |
| 100 | # --------------------------------------------------------------------------- |
| 101 | |
| 102 | |
| 103 | class TestUnchangedSnapshotReusesCache: |
| 104 | def test_second_identical_run_reads_no_objects_and_parses_nothing( |
| 105 | self, tmp_path: pathlib.Path |
| 106 | ) -> None: |
| 107 | root = _base_repo(tmp_path) |
| 108 | prod_oid = _write_file(root, "prod.py", PROD_V1) |
| 109 | test_oid = _write_file(root, "tests/test_prod.py", TEST_V1) |
| 110 | manifest: Manifest = {"prod.py": prod_oid, "tests/test_prod.py": test_oid} |
| 111 | changed = [ChangedSymbol(address="prod.py::compute", change_kind="modified")] |
| 112 | |
| 113 | sym_cache = SymbolCache.empty() |
| 114 | cg_cache = CallGraphCache.empty() |
| 115 | select_tests(root, changed, manifest, cache=sym_cache, callgraph_cache=cg_cache) |
| 116 | |
| 117 | with ( |
| 118 | patch("muse.plugins.code._query.read_object") as mock_query_read, |
| 119 | patch("muse.plugins.code._callgraph.read_object") as mock_cg_read, |
| 120 | patch("muse.plugins.code.ast_parser.parse_symbols") as mock_parse, |
| 121 | ): |
| 122 | result = select_tests( |
| 123 | root, changed, manifest, cache=sym_cache, callgraph_cache=cg_cache |
| 124 | ) |
| 125 | |
| 126 | mock_query_read.assert_not_called() |
| 127 | mock_cg_read.assert_not_called() |
| 128 | mock_parse.assert_not_called() |
| 129 | assert result["cache_hit"] is True |
| 130 | assert result["cache_misses"] == 0 |
| 131 | |
| 132 | |
| 133 | # --------------------------------------------------------------------------- |
| 134 | # CT_08 — a new import added to prod.py changes its object_id, forcing a |
| 135 | # call-graph rebuild for that file; a newly-reachable test must appear in |
| 136 | # the selection once the new call edge is present. |
| 137 | # --------------------------------------------------------------------------- |
| 138 | |
| 139 | |
| 140 | class TestNewImportRebuildsCallGraphAndIncludesNewlyReachableTest: |
| 141 | def test_newly_reachable_test_appears_after_call_graph_rebuild( |
| 142 | self, tmp_path: pathlib.Path |
| 143 | ) -> None: |
| 144 | root = _base_repo(tmp_path) |
| 145 | |
| 146 | helper_src = b"""\ |
| 147 | def helper(x: int) -> int: |
| 148 | return x + 1 |
| 149 | """ |
| 150 | prod_v1 = b"""\ |
| 151 | def compute(x: int) -> int: |
| 152 | return x * 2 |
| 153 | """ |
| 154 | # v2 adds a call to helper() -> new edge compute -> helper. |
| 155 | prod_v2 = b"""\ |
| 156 | from helper import helper |
| 157 | |
| 158 | def compute(x: int) -> int: |
| 159 | return helper(x) * 2 |
| 160 | """ |
| 161 | test_helper_src = b"""\ |
| 162 | from helper import helper |
| 163 | |
| 164 | def test_helper() -> None: |
| 165 | assert helper(1) == 2 |
| 166 | """ |
| 167 | |
| 168 | helper_oid = _write_file(root, "helper.py", helper_src) |
| 169 | prod_oid_v1 = _write_file(root, "prod.py", prod_v1) |
| 170 | test_helper_oid = _write_file(root, "tests/test_helper.py", test_helper_src) |
| 171 | |
| 172 | manifest_v1: Manifest = { |
| 173 | "helper.py": helper_oid, |
| 174 | "prod.py": prod_oid_v1, |
| 175 | "tests/test_helper.py": test_helper_oid, |
| 176 | } |
| 177 | sym_cache = SymbolCache.empty() |
| 178 | cg_cache = CallGraphCache.empty() |
| 179 | |
| 180 | changed_helper = [ChangedSymbol(address="helper.py::helper", change_kind="modified")] |
| 181 | result_v1 = select_tests( |
| 182 | root, changed_helper, manifest_v1, cache=sym_cache, callgraph_cache=cg_cache |
| 183 | ) |
| 184 | v1_targets = {t["node_id"] for t in result_v1["test_targets"]} |
| 185 | # Before the new import, compute() does not call helper() -> only |
| 186 | # the direct caller (test_helper) is covered via direct heuristics, |
| 187 | # not via a prod.py -> helper.py call edge. |
| 188 | |
| 189 | prod_oid_v2 = _write_file(root, "prod.py", prod_v2) |
| 190 | assert prod_oid_v2 != prod_oid_v1 |
| 191 | manifest_v2: Manifest = { |
| 192 | "helper.py": helper_oid, |
| 193 | "prod.py": prod_oid_v2, |
| 194 | "tests/test_helper.py": test_helper_oid, |
| 195 | } |
| 196 | |
| 197 | result_v2 = select_tests( |
| 198 | root, changed_helper, manifest_v2, cache=sym_cache, callgraph_cache=cg_cache |
| 199 | ) |
| 200 | v2_targets = {t["node_id"] for t in result_v2["test_targets"]} |
| 201 | |
| 202 | # prod.py was re-parsed (new object_id -> cache miss) and the test |
| 203 | # covering helper.py is still selected after the call-graph rebuild. |
| 204 | assert result_v2["cache_misses"] > 0 |
| 205 | assert any("test_helper" in t for t in v2_targets) |
| 206 | |
| 207 | |
| 208 | # --------------------------------------------------------------------------- |
| 209 | # CT_09 — adding a brand-new test file with no production-code changes must |
| 210 | # not bust the cache for the unrelated, already-cached production file. |
| 211 | # --------------------------------------------------------------------------- |
| 212 | |
| 213 | |
| 214 | class TestNewTestFileDoesNotBustUnrelatedCache: |
| 215 | def test_adding_test_file_does_not_reparse_unrelated_prod_file( |
| 216 | self, tmp_path: pathlib.Path |
| 217 | ) -> None: |
| 218 | root = _base_repo(tmp_path) |
| 219 | prod_oid = _write_file(root, "prod.py", PROD_V1) |
| 220 | test_oid = _write_file(root, "tests/test_prod.py", TEST_V1) |
| 221 | manifest_v1: Manifest = {"prod.py": prod_oid, "tests/test_prod.py": test_oid} |
| 222 | changed = [ChangedSymbol(address="prod.py::compute", change_kind="modified")] |
| 223 | |
| 224 | sym_cache = SymbolCache.empty() |
| 225 | cg_cache = CallGraphCache.empty() |
| 226 | select_tests(root, changed, manifest_v1, cache=sym_cache, callgraph_cache=cg_cache) |
| 227 | |
| 228 | # Add a brand-new test file; prod.py is untouched (same object_id). |
| 229 | new_test_src = b"""\ |
| 230 | def test_unrelated() -> None: |
| 231 | assert True |
| 232 | """ |
| 233 | new_test_oid = _write_file(root, "tests/test_unrelated.py", new_test_src) |
| 234 | manifest_v2: Manifest = { |
| 235 | "prod.py": prod_oid, |
| 236 | "tests/test_prod.py": test_oid, |
| 237 | "tests/test_unrelated.py": new_test_oid, |
| 238 | } |
| 239 | |
| 240 | from muse.core.object_store import read_object as real_read_object |
| 241 | |
| 242 | with ( |
| 243 | patch( |
| 244 | "muse.plugins.code._query.read_object", wraps=real_read_object |
| 245 | ) as mock_query_read, |
| 246 | patch( |
| 247 | "muse.plugins.code._callgraph.read_object", wraps=real_read_object |
| 248 | ) as mock_cg_read, |
| 249 | ): |
| 250 | result = select_tests( |
| 251 | root, changed, manifest_v2, cache=sym_cache, callgraph_cache=cg_cache |
| 252 | ) |
| 253 | |
| 254 | # The new file is read/parsed (its object_id was never cached), but |
| 255 | # prod.py and tests/test_prod.py (same object_id as before) must not |
| 256 | # trigger any read_object call for THEIR ids. |
| 257 | read_oids = {c.args[1] for c in mock_query_read.call_args_list} |
| 258 | read_oids |= {c.args[1] for c in mock_cg_read.call_args_list} |
| 259 | assert prod_oid not in read_oids |
| 260 | assert test_oid not in read_oids |
| 261 | assert new_test_oid in read_oids |
| 262 | # Only one new file's worth of misses, not a full cache bust. |
| 263 | assert result["cache_misses"] <= 2 # one per cache for the one new file |
| 264 | |
| 265 | |
| 266 | # --------------------------------------------------------------------------- |
| 267 | # CT_10 — ``muse code index rebuild`` manages a wholly separate index |
| 268 | # (.muse/indices/*) and has no invalidation relationship with the |
| 269 | # SymbolCache/CallGraphCache used by muse code test. Per the audit |
| 270 | # (docs/code-test-internals.md, "Decision" section) and the ticket's own |
| 271 | # "audit only, no redesign" constraint, the correct behavior to verify is |
| 272 | # that index rebuild does NOT corrupt or invalidate the test-selection |
| 273 | # caches — content-addressing means no explicit invalidation is needed. |
| 274 | # --------------------------------------------------------------------------- |
| 275 | |
| 276 | |
| 277 | class TestIndexRebuildDoesNotAffectTestSelectionCache: |
| 278 | def test_cache_still_warm_after_index_rebuild( |
| 279 | self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch |
| 280 | ) -> None: |
| 281 | """``muse code index rebuild`` manages a wholly separate index |
| 282 | (.muse/indices/*) from the test-selection caches (.muse/cache/*). |
| 283 | Real repo, real commit, real CLI invocation — proves rebuild neither |
| 284 | corrupts nor busts the warm test-selection cache, consistent with |
| 285 | the content-addressed design (no explicit invalidation needed).""" |
| 286 | from tests.cli_test_helper import CliRunner |
| 287 | |
| 288 | runner = CliRunner() |
| 289 | monkeypatch.chdir(tmp_path) |
| 290 | monkeypatch.setenv("MUSE_REPO_ROOT", str(tmp_path)) |
| 291 | assert runner.invoke(None, ["init", "--domain", "code"]).exit_code == 0 |
| 292 | |
| 293 | (tmp_path / "prod.py").write_text(PROD_V1.decode()) |
| 294 | tests_dir = tmp_path / "tests" |
| 295 | tests_dir.mkdir() |
| 296 | (tests_dir / "test_prod.py").write_text(TEST_V1.decode()) |
| 297 | assert runner.invoke(None, ["code", "add", "."]).exit_code == 0 |
| 298 | commit_result = runner.invoke(None, ["commit", "-m", "v1"]) |
| 299 | assert commit_result.exit_code == 0, commit_result.output |
| 300 | |
| 301 | from muse.core.commits import get_head_snapshot_id |
| 302 | from muse.core.snapshots import read_snapshot |
| 303 | |
| 304 | snapshot_id = get_head_snapshot_id(tmp_path, "main") |
| 305 | assert snapshot_id is not None |
| 306 | snapshot = read_snapshot(tmp_path, snapshot_id) |
| 307 | assert snapshot is not None |
| 308 | head_manifest: Manifest = snapshot.manifest |
| 309 | changed = [ChangedSymbol(address="prod.py::compute", change_kind="modified")] |
| 310 | |
| 311 | sym_cache = SymbolCache.from_root(tmp_path) |
| 312 | cg_cache = CallGraphCache.from_root(tmp_path) |
| 313 | select_tests( |
| 314 | tmp_path, changed, head_manifest, cache=sym_cache, callgraph_cache=cg_cache |
| 315 | ) |
| 316 | sym_cache.save() |
| 317 | cg_cache.save() |
| 318 | |
| 319 | rebuild_result = runner.invoke(None, ["code", "index", "rebuild", "--json"]) |
| 320 | assert rebuild_result.exit_code == 0, rebuild_result.output |
| 321 | |
| 322 | # Caches on disk must still be loadable and still warm for prod.py. |
| 323 | reloaded_sym = SymbolCache.load(muse_dir(tmp_path)) |
| 324 | reloaded_cg = CallGraphCache.load(muse_dir(tmp_path)) |
| 325 | |
| 326 | with ( |
| 327 | patch("muse.plugins.code._query.read_object") as mock_query_read, |
| 328 | patch("muse.plugins.code._callgraph.read_object") as mock_cg_read, |
| 329 | ): |
| 330 | result = select_tests( |
| 331 | tmp_path, changed, head_manifest, cache=reloaded_sym, callgraph_cache=reloaded_cg |
| 332 | ) |
| 333 | |
| 334 | mock_query_read.assert_not_called() |
| 335 | mock_cg_read.assert_not_called() |
| 336 | assert result["cache_hit"] is True |
File History
1 commit
sha256:ce542c4d0f6dacc9520c02d0e3378bf96bc377d118a7402f2bbc8af1749b08e0
feat(code-test): audit cache internals, add cache_hit/--cac…
Sonnet 5
minor
⚠
3 hours ago