aaronrene / gabriel-muse public
stat_cache.py python
433 lines 15.6 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 32 days ago
1 """Stat-based file hash cache — fast snapshot computation for all domains.
2
3 Architecture
4 ------------
5 Every ``plugin.snapshot()`` call must hash every tracked file to detect
6 changes. On a repository with hundreds of files this is the dominant cost of
7 ``muse status``, ``muse diff``, and any command that calls ``snapshot()``.
8
9 ``StatCache`` eliminates redundant I/O by persisting two classes of hash per
10 file between invocations:
11
12 Object hash
13 SHA-256 of raw bytes. Used by the content-addressed object store.
14 Recomputed only when ``(ino, mtime, size)`` changes.
15
16 Dimension hashes
17 Domain-specific semantic hashes. For the code domain these might be the
18 SHA-256 of the AST symbol set, the import set, and so on. For the MIDI
19 domain they might be the hash of parsed note events, tempo map, and
20 harmony analysis. Populated by domain plugins after parsing; consumed by
21 ``diff()`` and ``merge()`` to skip re-parsing unchanged files entirely.
22
23 An empty ``dimensions`` dict means no semantic hashes are cached yet —
24 this is the baseline state and is always safe.
25
26 Cache validity
27 --------------
28 A cache entry is valid when the file's current ``(st_ino, st_mtime, st_size)``
29 exactly match the stored values. Including the inode number eliminates the
30 "racy Muse" false-cache-hit that the previous ``(mtime, size)``-only check
31 was vulnerable to: an atomically replaced file (e.g. from a build system)
32 gets a fresh inode number even when its mtime and size are unchanged.
33
34 The cache is **self-healing**: a cache miss always triggers a fresh hash and
35 updates the stored entry.
36
37 Storage
38 -------
39 ``.muse/stat_cache.msgpack`` — a versioned msgpack document (version 2).
40 Format:
41
42 .. code-block:: text
43
44 {
45 "version": 2,
46 "entries": {
47 "muse/core/snapshot.py": {
48 "mtime": 1710000000.123456,
49 "size": 4321,
50 "ino": 12345678,
51 "object_hash": "<sha256-of-raw-bytes>",
52 "dimensions": {
53 "symbols": "<sha256-of-ast-symbol-set>",
54 "imports": "<sha256-of-import-set>"
55 }
56 }
57 }
58 }
59
60 msgpack is used instead of JSON because at 75 000-file scale the JSON
61 serialisation alone costs ~200 ms per commit; msgpack cuts that to ~50 ms
62 while also reducing disk footprint by ~30 %.
63
64 Writes are atomic: data is flushed via ``os.fdopen(mkstemp(...))`` then
65 ``os.replace``-d over the target, so a crash mid-write never corrupts the
66 cache. Each concurrent ``muse commit`` process gets its own unique temp file
67 (via ``mkstemp``) so parallel saves do not collide.
68 """
69
70 from __future__ import annotations
71
72 import hashlib
73 import logging
74 import os
75 import pathlib
76 import tempfile
77 from typing import TypedDict
78
79 import msgpack
80
81 logger = logging.getLogger(__name__)
82
83 type _DimensionMap = dict[str, str]
84 type _EntryMap = dict[str, FileCacheEntry]
85
86 _CACHE_VERSION = 2
87 _CACHE_FILENAME = "stat_cache.msgpack"
88 _CHUNK = 65_536
89
90 # Defense in depth: refuse to load a cache file larger than this.
91 # A 75k-file repo produces ~7 MiB of msgpack; 256 MiB is a generous ceiling.
92 MAX_CACHE_BYTES: int = 256 * 1024 * 1024
93
94 # Per-value msgpack unpack limits — prevent malformed cache from allocating
95 # unbounded memory even before the size check above fires.
96 _MAX_STR_LEN: int = 32 * 1024 # 32 KiB — longest plausible rel path
97 _MAX_ARRAY_LEN: int = 10_000_000 # generous for large repos
98 _MAX_MAP_LEN: int = 10_000_000 # generous for large repos
99
100
101 class FileCacheEntry(TypedDict):
102 """Persisted metadata for a single workspace file."""
103
104 mtime: float
105 size: int
106 ino: int # inode number — disambiguates atomically replaced files
107 object_hash: str
108 # Domain plugins write semantic hashes here after parsing.
109 # Keys are dimension names ("symbols", "imports", "notes", …).
110 # Empty dict == no dimension hashes cached yet; always safe to return None.
111 dimensions: _DimensionMap
112
113
114 class _CacheDoc(TypedDict):
115 """On-disk msgpack document shape."""
116
117 version: int
118 entries: _EntryMap
119
120
121 def _hash_bytes(path: pathlib.Path) -> str:
122 """Return the SHA-256 hex digest of *path*'s raw bytes.
123
124 Reads in 64 KiB chunks so memory usage is constant regardless of file size.
125 This is the single canonical implementation shared by the cache and all
126 domain plugins — no more duplicated ``_hash_file`` helpers.
127 """
128 return _hash_str(str(path))
129
130
131 def _hash_str(path_str: str) -> str:
132 """String-path variant of ``_hash_bytes`` — avoids constructing a Path object.
133
134 Used in the hot inner loop of ``walk_workdir`` and plugin snapshot methods
135 where the file path is already a plain string from ``os.walk``.
136 """
137 h = hashlib.sha256()
138 with open(path_str, "rb") as fh:
139 for chunk in iter(lambda: fh.read(_CHUNK), b""):
140 h.update(chunk)
141 return h.hexdigest()
142
143
144 class StatCache:
145 """Shared stat-based hash cache for all domain plugin ``snapshot()`` calls.
146
147 Typical lifecycle inside a plugin's ``snapshot()``::
148
149 cache = StatCache.load(root / ".muse")
150 for file_path in walk(...):
151 files[rel] = cache.get_object_hash(root, file_path)
152 cache.prune(set(files))
153 cache.save()
154
155 The same instance can be passed to ``diff()`` or ``merge()`` logic to
156 retrieve already-computed dimension hashes without re-parsing files.
157 """
158
159 def __init__(
160 self, muse_dir: pathlib.Path | None, entries: _EntryMap
161 ) -> None:
162 self._muse_dir = muse_dir
163 self._entries = entries
164 self._dirty = False
165
166 # ------------------------------------------------------------------
167 # Construction
168 # ------------------------------------------------------------------
169
170 @classmethod
171 def load(cls, muse_dir: pathlib.Path) -> StatCache:
172 """Load the cache from *muse_dir*/stat_cache.msgpack.
173
174 Validates the version field and every entry's field types on load so
175 a corrupt or future-format file never poisons the cache. Returns a
176 fresh empty cache if the file is absent, unreadable, exceeds
177 ``MAX_CACHE_BYTES``, or version mismatches — never raises.
178 """
179 cache_file = muse_dir / _CACHE_FILENAME
180 if not cache_file.is_file():
181 return cls(muse_dir, {})
182 try:
183 file_size = cache_file.stat().st_size
184 if file_size > MAX_CACHE_BYTES:
185 logger.critical(
186 "❌ stat_cache %s is %d bytes — exceeds %d MiB limit; "
187 "starting fresh.",
188 cache_file.name,
189 file_size,
190 MAX_CACHE_BYTES // (1024 * 1024),
191 )
192 return cls(muse_dir, {})
193 raw = msgpack.unpackb(
194 cache_file.read_bytes(),
195 raw=False,
196 max_str_len=_MAX_STR_LEN,
197 max_bin_len=0,
198 max_array_len=_MAX_ARRAY_LEN,
199 max_map_len=_MAX_MAP_LEN,
200 )
201 if not (isinstance(raw, dict) and raw.get("version") == _CACHE_VERSION):
202 return cls(muse_dir, {})
203 raw_entries = raw.get("entries")
204 if not isinstance(raw_entries, dict):
205 return cls(muse_dir, {})
206 entries: _EntryMap = {}
207 for rel, ev in raw_entries.items():
208 if not isinstance(rel, str) or not isinstance(ev, dict):
209 continue
210 mtime = ev.get("mtime")
211 size = ev.get("size")
212 ino = ev.get("ino")
213 obj_hash = ev.get("object_hash")
214 dims = ev.get("dimensions")
215 if not (
216 isinstance(mtime, (int, float))
217 and isinstance(size, int)
218 and isinstance(ino, int)
219 and isinstance(obj_hash, str)
220 and isinstance(dims, dict)
221 ):
222 continue
223 entries[rel] = FileCacheEntry(
224 mtime=float(mtime),
225 size=size,
226 ino=ino,
227 object_hash=obj_hash,
228 dimensions={str(k): str(v) for k, v in dims.items()},
229 )
230 return cls(muse_dir, entries)
231 except Exception:
232 logger.debug("⚠️ stat_cache unreadable — starting fresh")
233 return cls(muse_dir, {})
234
235 @classmethod
236 def empty(cls) -> StatCache:
237 """Return a no-op cache for contexts without a ``.muse`` directory."""
238 return cls(None, {})
239
240 # ------------------------------------------------------------------
241 # Object hash — raw-bytes SHA-256
242 # ------------------------------------------------------------------
243
244 def get_cached(
245 self, rel: str, abs_path_str: str, mtime: float, size: int, ino: int
246 ) -> str:
247 """Fast inner-loop hash lookup with pre-computed stat values.
248
249 Callers that already have ``(mtime, size, ino)`` from an ``os.stat``
250 or ``os.walk`` call should use this method to avoid a redundant
251 ``stat()`` syscall inside :meth:`get_object_hash`.
252
253 The inode number (``ino``) is included in the cache key in addition to
254 ``mtime`` and ``size`` to eliminate false cache hits when a file is
255 atomically replaced (e.g. by a build system or ``muse checkout``) with
256 content of identical size. An atomically replaced file always gets a
257 new inode, so the cache correctly invalidates.
258
259 Args:
260 rel: Workspace-relative POSIX path (cache key).
261 abs_path_str: Absolute file path as a plain string — avoids
262 constructing a ``pathlib.Path`` in the hot loop.
263 mtime: ``st_mtime`` from the caller's stat result.
264 size: ``st_size`` from the caller's stat result.
265 ino: ``st_ino`` from the caller's stat result.
266
267 Returns:
268 64-character lowercase hex SHA-256 digest.
269 """
270 entry = self._entries.get(rel)
271 if (
272 entry is not None
273 and entry["ino"] == ino
274 and entry["mtime"] == mtime
275 and entry["size"] == size
276 ):
277 return entry["object_hash"]
278
279 obj_hash = _hash_str(abs_path_str)
280 self._entries[rel] = FileCacheEntry(
281 mtime=mtime,
282 size=size,
283 ino=ino,
284 object_hash=obj_hash,
285 dimensions={},
286 )
287 self._dirty = True
288 return obj_hash
289
290 def get_object_hash(self, root: pathlib.Path, file_path: pathlib.Path) -> str:
291 """Return the SHA-256 of *file_path*, using the cache when valid.
292
293 Convenience wrapper around :meth:`get_cached` for callers that work
294 with ``pathlib.Path`` objects. The hot inner loops of ``walk_workdir``
295 and plugin snapshot methods call :meth:`get_cached` directly to skip
296 pathlib overhead.
297
298 Args:
299 root: Repository root — used to compute the workspace-relative
300 POSIX key.
301 file_path: Absolute path to the file.
302
303 Returns:
304 64-character lowercase hex SHA-256 digest.
305 """
306 rel = file_path.relative_to(root).as_posix()
307 st = file_path.stat()
308 return self.get_cached(rel, str(file_path), st.st_mtime, st.st_size, st.st_ino)
309
310 # ------------------------------------------------------------------
311 # Dimension hashes — domain-specific semantic hashes
312 # ------------------------------------------------------------------
313
314 def get_dimension(
315 self,
316 root: pathlib.Path,
317 file_path: pathlib.Path,
318 dimension: str,
319 ) -> str | None:
320 """Return a cached dimension hash, or ``None`` if not yet computed.
321
322 Callers must verify that the entry is still valid by checking that
323 the object hash hasn't changed (i.e. call ``get_object_hash`` first
324 to ensure the entry is fresh).
325
326 Args:
327 root: Repository root.
328 file_path: Absolute path to the file.
329 dimension: Dimension name, e.g. ``"symbols"`` or ``"notes"``.
330
331 Returns:
332 Cached hash string, or ``None`` if absent.
333 """
334 rel = file_path.relative_to(root).as_posix()
335 entry = self._entries.get(rel)
336 if entry is None:
337 return None
338 return entry["dimensions"].get(dimension)
339
340 def set_dimension(
341 self,
342 root: pathlib.Path,
343 file_path: pathlib.Path,
344 dimension: str,
345 hash_value: str,
346 ) -> None:
347 """Store a semantic hash for a specific dimension of *file_path*.
348
349 Should be called by domain plugins after parsing a file whose object
350 hash triggered a cache miss. Silently ignored if the file has no
351 entry (which should not happen in normal operation).
352
353 Args:
354 root: Repository root.
355 file_path: Absolute path to the file.
356 dimension: Dimension name, e.g. ``"symbols"``.
357 hash_value: Hash string to store.
358 """
359 rel = file_path.relative_to(root).as_posix()
360 entry = self._entries.get(rel)
361 if entry is None:
362 return
363 entry["dimensions"][dimension] = hash_value
364 self._dirty = True
365
366 # ------------------------------------------------------------------
367 # Lifecycle helpers
368 # ------------------------------------------------------------------
369
370 def prune(self, known_paths: set[str]) -> None:
371 """Remove entries for paths no longer present in the working tree.
372
373 Call this after a full directory walk, passing the set of
374 workspace-relative POSIX paths that were found. Keeps the cache
375 lean by evicting stale entries for deleted files.
376
377 Args:
378 known_paths: Set of rel-posix paths observed during the walk.
379 """
380 stale = set(self._entries) - known_paths
381 if stale:
382 for k in stale:
383 del self._entries[k]
384 self._dirty = True
385
386 def save(self) -> None:
387 """Atomically persist the cache to disk if it has changed.
388
389 Uses ``mkstemp`` for a process-unique temp file (no concurrent-save
390 collisions), ``fsync`` for durability, then ``os.replace`` for
391 atomicity. Silently skips when there is no ``.muse`` directory
392 (e.g. in-memory unit tests).
393 """
394 if not self._dirty or self._muse_dir is None:
395 return
396 doc = _CacheDoc(version=_CACHE_VERSION, entries=self._entries)
397 payload = msgpack.packb(doc, use_bin_type=True)
398 cache_file = self._muse_dir / _CACHE_FILENAME
399 fd, tmp_path = tempfile.mkstemp(
400 dir=self._muse_dir, prefix=".stat_cache_", suffix=".tmp"
401 )
402 try:
403 with os.fdopen(fd, "wb") as fh:
404 fh.write(payload)
405 fh.flush()
406 os.fsync(fh.fileno())
407 except Exception:
408 try:
409 os.unlink(tmp_path)
410 except OSError:
411 pass
412 raise
413 os.replace(tmp_path, cache_file)
414 self._dirty = False
415 logger.debug("✅ stat_cache saved (%d entries)", len(self._entries))
416
417
418 def load_cache(root: pathlib.Path) -> StatCache:
419 """Convenience loader: return a ``StatCache`` for a repository root.
420
421 Returns ``StatCache.empty()`` when *root* has no ``.muse`` directory
422 so callers never need to guard against a missing repo.
423
424 Args:
425 root: Repository root (the directory that contains ``.muse/``).
426
427 Returns:
428 A ``StatCache`` instance ready for use.
429 """
430 muse_dir = root / ".muse"
431 if muse_dir.is_dir():
432 return StatCache.load(muse_dir)
433 return StatCache.empty()
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 32 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 32 days ago