stat_cache.py
python
sha256:37af4ab271a64f11d7bf14b4871ebaaf5dd68fb0247521007c6b034d35aa01be
Merge branch 'fix/hub-user-read-update-path' into dev
Human
10 days ago
| 1 | """Stat-based file hash cache — fast snapshot computation for all domains. |
| 2 | |
| 3 | Architecture |
| 4 | ------------ |
| 5 | Every ``plugin.snapshot()`` call must hash every tracked file to detect |
| 6 | changes. On a repository with hundreds of files this is the dominant cost of |
| 7 | ``muse status``, ``muse diff``, and any command that calls ``snapshot()``. |
| 8 | |
| 9 | ``StatCache`` eliminates redundant I/O by persisting two classes of hash per |
| 10 | file between invocations: |
| 11 | |
| 12 | Object hash |
| 13 | SHA-256 of raw bytes. Used by the content-addressed object store. |
| 14 | Recomputed only when ``(ino, mtime, size)`` changes. |
| 15 | |
| 16 | Dimension hashes |
| 17 | Domain-specific semantic hashes. For the code domain these might be the |
| 18 | SHA-256 of the AST symbol set, the import set, and so on. For the MIDI |
| 19 | domain they might be the hash of parsed note events, tempo map, and |
| 20 | harmony analysis. Populated by domain plugins after parsing; consumed by |
| 21 | ``diff()`` and ``merge()`` to skip re-parsing unchanged files entirely. |
| 22 | |
| 23 | An empty ``dimensions`` dict means no semantic hashes are cached yet — |
| 24 | this is the baseline state and is always safe. |
| 25 | |
| 26 | Cache validity |
| 27 | -------------- |
| 28 | A cache entry is valid when the file's current ``(st_ino, st_mtime, st_size)`` |
| 29 | exactly match the stored values. Including the inode number eliminates the |
| 30 | "racy Muse" false-cache-hit that the previous ``(mtime, size)``-only check |
| 31 | was vulnerable to: an atomically replaced file (e.g. from a build system) |
| 32 | gets a fresh inode number even when its mtime and size are unchanged. |
| 33 | |
| 34 | The cache is **self-healing**: a cache miss always triggers a fresh hash and |
| 35 | updates the stored entry. |
| 36 | |
| 37 | Storage |
| 38 | ------- |
| 39 | ``.muse/cache/stat.json`` — a versioned JSON document (version 4). |
| 40 | Format: |
| 41 | |
| 42 | .. code-block:: text |
| 43 | |
| 44 | { |
| 45 | "version": 4, |
| 46 | "entries": { |
| 47 | "muse/core/snapshot.py": { |
| 48 | "mtime": 1710000000.123456, |
| 49 | "size": 4321, |
| 50 | "ino": 12345678, |
| 51 | "object_hash": "<sha256-of-raw-bytes>", |
| 52 | "dimensions": { |
| 53 | "symbols": "<sha256-of-ast-symbol-set>", |
| 54 | "imports": "<sha256-of-import-set>" |
| 55 | } |
| 56 | } |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | Writes are atomic: data is flushed via ``os.fdopen(mkstemp(...))`` then |
| 61 | ``os.replace``-d over the target, so a crash mid-write never corrupts the |
| 62 | cache. Each concurrent ``muse commit`` process gets its own unique temp file |
| 63 | (via ``mkstemp``) so parallel saves do not collide. |
| 64 | """ |
| 65 | |
| 66 | import logging |
| 67 | import os |
| 68 | import pathlib |
| 69 | import tempfile |
| 70 | from typing import TypedDict |
| 71 | |
| 72 | import json as _json |
| 73 | |
| 74 | from muse.core.types import MUSE_DIR, hash_file, long_id |
| 75 | from muse.core.paths import cache_dir as _cache_dir_path, muse_dir as _muse_dir |
| 76 | |
| 77 | logger = logging.getLogger(__name__) |
| 78 | |
| 79 | type _DimensionMap = dict[str, str] |
| 80 | type _EntryMap = dict[str, FileCacheEntry] |
| 81 | |
| 82 | _CACHE_VERSION = 4 |
| 83 | _CACHE_FILENAME = "stat.json" |
| 84 | |
| 85 | # Defense in depth: refuse to load a cache file larger than this. |
| 86 | # A 75k-file repo produces ~15 MiB of JSON; 256 MiB is a generous ceiling. |
| 87 | MAX_CACHE_BYTES: int = 256 * 1024 * 1024 |
| 88 | |
| 89 | class FileCacheEntry(TypedDict): |
| 90 | """Persisted metadata for a single workspace file.""" |
| 91 | |
| 92 | mtime: float |
| 93 | size: int |
| 94 | ino: int # inode number — disambiguates atomically replaced files |
| 95 | object_hash: str |
| 96 | # Domain plugins write semantic hashes here after parsing. |
| 97 | # Keys are dimension names ("symbols", "imports", "notes", …). |
| 98 | # Empty dict == no dimension hashes cached yet; always safe to return None. |
| 99 | dimensions: _DimensionMap |
| 100 | |
| 101 | class _CacheDoc(TypedDict): |
| 102 | """On-disk JSON document shape.""" |
| 103 | |
| 104 | version: int |
| 105 | entries: _EntryMap |
| 106 | |
| 107 | def _hash_bytes(path: pathlib.Path) -> str: |
| 108 | """Return the ``sha256:``-prefixed content ID of *path*'s raw bytes.""" |
| 109 | return hash_file(path) |
| 110 | |
| 111 | def _hash_str(path_str: str) -> str: |
| 112 | """String-path convenience wrapper around :func:`~muse.core.types.hash_file`. |
| 113 | |
| 114 | Used in the hot inner loop of ``walk_workdir`` and plugin snapshot methods |
| 115 | where the file path is already a plain string from ``os.walk``. |
| 116 | """ |
| 117 | return hash_file(pathlib.Path(path_str)) |
| 118 | |
| 119 | class StatCache: |
| 120 | """Shared stat-based hash cache for all domain plugin ``snapshot()`` calls. |
| 121 | |
| 122 | Typical lifecycle inside a plugin's ``snapshot()``:: |
| 123 | |
| 124 | cache = StatCache.load(root / MUSE_DIR) |
| 125 | for file_path in walk(...): |
| 126 | files[rel] = cache.get_object_hash(root, file_path) |
| 127 | cache.prune(set(files)) |
| 128 | cache.save() |
| 129 | |
| 130 | The same instance can be passed to ``diff()`` or ``merge()`` logic to |
| 131 | retrieve already-computed dimension hashes without re-parsing files. |
| 132 | """ |
| 133 | |
| 134 | def __init__( |
| 135 | self, cache_dir: pathlib.Path | None, entries: _EntryMap |
| 136 | ) -> None: |
| 137 | self._cache_dir = cache_dir |
| 138 | self._entries = entries |
| 139 | self._dirty = False |
| 140 | |
| 141 | # ------------------------------------------------------------------ |
| 142 | # Construction |
| 143 | # ------------------------------------------------------------------ |
| 144 | |
| 145 | @classmethod |
| 146 | def load(cls, muse_dir: pathlib.Path) -> StatCache: |
| 147 | """Load the cache from *muse_dir*/cache/stat.json. |
| 148 | |
| 149 | Validates the version field and every entry's field types on load so |
| 150 | a corrupt or future-format file never poisons the cache. Returns a |
| 151 | fresh empty cache if the file is absent, unreadable, exceeds |
| 152 | ``MAX_CACHE_BYTES``, or version mismatches — never raises. |
| 153 | """ |
| 154 | cache_dir = muse_dir / "cache" |
| 155 | cache_file = cache_dir / _CACHE_FILENAME |
| 156 | if not cache_file.is_file(): |
| 157 | return cls(cache_dir, {}) |
| 158 | try: |
| 159 | file_size = cache_file.stat().st_size |
| 160 | if file_size > MAX_CACHE_BYTES: |
| 161 | logger.critical( |
| 162 | "❌ stat_cache %s is %d bytes — exceeds %d MiB limit; " |
| 163 | "starting fresh.", |
| 164 | cache_file.name, |
| 165 | file_size, |
| 166 | MAX_CACHE_BYTES // (1024 * 1024), |
| 167 | ) |
| 168 | return cls(cache_dir, {}) |
| 169 | file_bytes = cache_file.read_bytes() |
| 170 | # Old binary msgpack files start with a byte > 0x7F — treat as stale. |
| 171 | if file_bytes and file_bytes[0] > 0x7F: |
| 172 | logger.debug("⚠️ stat_cache is old binary format — starting fresh") |
| 173 | return cls(cache_dir, {}) |
| 174 | raw = _json.loads(file_bytes.decode("utf-8")) |
| 175 | if not (isinstance(raw, dict) and raw.get("version") == _CACHE_VERSION): |
| 176 | return cls(cache_dir, {}) |
| 177 | raw_entries = raw.get("entries") |
| 178 | if not isinstance(raw_entries, dict): |
| 179 | return cls(cache_dir, {}) |
| 180 | entries: _EntryMap = {} |
| 181 | for rel, ev in raw_entries.items(): |
| 182 | if not isinstance(rel, str) or not isinstance(ev, dict): |
| 183 | continue |
| 184 | mtime = ev.get("mtime") |
| 185 | size = ev.get("size") |
| 186 | ino = ev.get("ino") |
| 187 | obj_hash = ev.get("object_hash") |
| 188 | dims = ev.get("dimensions") |
| 189 | if not ( |
| 190 | isinstance(mtime, (int, float)) |
| 191 | and isinstance(size, int) |
| 192 | and isinstance(ino, int) |
| 193 | and isinstance(obj_hash, str) |
| 194 | and isinstance(dims, dict) |
| 195 | ): |
| 196 | continue |
| 197 | entries[rel] = FileCacheEntry( |
| 198 | mtime=float(mtime), |
| 199 | size=size, |
| 200 | ino=ino, |
| 201 | object_hash=obj_hash, |
| 202 | dimensions={str(k): str(v) for k, v in dims.items()}, |
| 203 | ) |
| 204 | return cls(cache_dir, entries) |
| 205 | except Exception: |
| 206 | logger.debug("⚠️ stat_cache unreadable — starting fresh") |
| 207 | return cls(cache_dir, {}) |
| 208 | |
| 209 | @classmethod |
| 210 | def empty(cls) -> StatCache: |
| 211 | """Return a no-op cache for contexts without a ``.muse`` directory.""" |
| 212 | return cls(None, {}) |
| 213 | |
| 214 | # ------------------------------------------------------------------ |
| 215 | # Object hash — raw-bytes SHA-256 |
| 216 | # ------------------------------------------------------------------ |
| 217 | |
| 218 | def get_cached( |
| 219 | self, rel: str, abs_path_str: str, mtime: float, size: int, ino: int |
| 220 | ) -> str: |
| 221 | """Fast inner-loop hash lookup with pre-computed stat values. |
| 222 | |
| 223 | Callers that already have ``(mtime, size, ino)`` from an ``os.stat`` |
| 224 | or ``os.walk`` call should use this method to avoid a redundant |
| 225 | ``stat()`` syscall inside :meth:`get_object_hash`. |
| 226 | |
| 227 | The inode number (``ino``) is included in the cache key in addition to |
| 228 | ``mtime`` and ``size`` to eliminate false cache hits when a file is |
| 229 | atomically replaced (e.g. by a build system or ``muse checkout``) with |
| 230 | content of identical size. An atomically replaced file always gets a |
| 231 | new inode, so the cache correctly invalidates. |
| 232 | |
| 233 | Args: |
| 234 | rel: Workspace-relative POSIX path (cache key). |
| 235 | abs_path_str: Absolute file path as a plain string — avoids |
| 236 | constructing a ``pathlib.Path`` in the hot loop. |
| 237 | mtime: ``st_mtime`` from the caller's stat result. |
| 238 | size: ``st_size`` from the caller's stat result. |
| 239 | ino: ``st_ino`` from the caller's stat result. |
| 240 | |
| 241 | Returns: |
| 242 | 64-character lowercase hex SHA-256 digest. |
| 243 | """ |
| 244 | entry = self._entries.get(rel) |
| 245 | if ( |
| 246 | entry is not None |
| 247 | and entry["ino"] == ino |
| 248 | and entry["mtime"] == mtime |
| 249 | and entry["size"] == size |
| 250 | ): |
| 251 | cached = entry["object_hash"] |
| 252 | # Normalize legacy bare-hex cache entries to the canonical format. |
| 253 | return long_id(cached) |
| 254 | |
| 255 | obj_hash = _hash_str(abs_path_str) |
| 256 | self._entries[rel] = FileCacheEntry( |
| 257 | mtime=mtime, |
| 258 | size=size, |
| 259 | ino=ino, |
| 260 | object_hash=obj_hash, |
| 261 | dimensions={}, |
| 262 | ) |
| 263 | self._dirty = True |
| 264 | return obj_hash |
| 265 | |
| 266 | def get_object_hash(self, root: pathlib.Path, file_path: pathlib.Path) -> str: |
| 267 | """Return the SHA-256 of *file_path*, using the cache when valid. |
| 268 | |
| 269 | Convenience wrapper around :meth:`get_cached` for callers that work |
| 270 | with ``pathlib.Path`` objects. The hot inner loops of ``walk_workdir`` |
| 271 | and plugin snapshot methods call :meth:`get_cached` directly to skip |
| 272 | pathlib overhead. |
| 273 | |
| 274 | Args: |
| 275 | root: Repository root — used to compute the workspace-relative |
| 276 | POSIX key. |
| 277 | file_path: Absolute path to the file. |
| 278 | |
| 279 | Returns: |
| 280 | 64-character lowercase hex SHA-256 digest. |
| 281 | """ |
| 282 | rel = file_path.relative_to(root).as_posix() |
| 283 | st = file_path.stat() |
| 284 | return self.get_cached(rel, str(file_path), st.st_mtime, st.st_size, st.st_ino) |
| 285 | |
| 286 | # ------------------------------------------------------------------ |
| 287 | # Dimension hashes — domain-specific semantic hashes |
| 288 | # ------------------------------------------------------------------ |
| 289 | |
| 290 | def get_dimension( |
| 291 | self, |
| 292 | root: pathlib.Path, |
| 293 | file_path: pathlib.Path, |
| 294 | dimension: str, |
| 295 | ) -> str | None: |
| 296 | """Return a cached dimension hash, or ``None`` if not yet computed. |
| 297 | |
| 298 | Callers must verify that the entry is still valid by checking that |
| 299 | the object hash hasn't changed (i.e. call ``get_object_hash`` first |
| 300 | to ensure the entry is fresh). |
| 301 | |
| 302 | Args: |
| 303 | root: Repository root. |
| 304 | file_path: Absolute path to the file. |
| 305 | dimension: Dimension name, e.g. ``"symbols"`` or ``"notes"``. |
| 306 | |
| 307 | Returns: |
| 308 | Cached hash string, or ``None`` if absent. |
| 309 | """ |
| 310 | rel = file_path.relative_to(root).as_posix() |
| 311 | entry = self._entries.get(rel) |
| 312 | if entry is None: |
| 313 | return None |
| 314 | return entry["dimensions"].get(dimension) |
| 315 | |
| 316 | def set_dimension( |
| 317 | self, |
| 318 | root: pathlib.Path, |
| 319 | file_path: pathlib.Path, |
| 320 | dimension: str, |
| 321 | hash_value: str, |
| 322 | ) -> None: |
| 323 | """Store a semantic hash for a specific dimension of *file_path*. |
| 324 | |
| 325 | Should be called by domain plugins after parsing a file whose object |
| 326 | hash triggered a cache miss. Silently ignored if the file has no |
| 327 | entry (which should not happen in normal operation). |
| 328 | |
| 329 | Args: |
| 330 | root: Repository root. |
| 331 | file_path: Absolute path to the file. |
| 332 | dimension: Dimension name, e.g. ``"symbols"``. |
| 333 | hash_value: Hash string to store. |
| 334 | """ |
| 335 | rel = file_path.relative_to(root).as_posix() |
| 336 | entry = self._entries.get(rel) |
| 337 | if entry is None: |
| 338 | return |
| 339 | entry["dimensions"][dimension] = hash_value |
| 340 | self._dirty = True |
| 341 | |
| 342 | # ------------------------------------------------------------------ |
| 343 | # Lifecycle helpers |
| 344 | # ------------------------------------------------------------------ |
| 345 | |
| 346 | def prune(self, known_paths: set[str]) -> None: |
| 347 | """Remove entries for paths no longer present in the working tree. |
| 348 | |
| 349 | Call this after a full directory walk, passing the set of |
| 350 | workspace-relative POSIX paths that were found. Keeps the cache |
| 351 | lean by evicting stale entries for deleted files. |
| 352 | |
| 353 | Args: |
| 354 | known_paths: Set of rel-posix paths observed during the walk. |
| 355 | """ |
| 356 | stale = set(self._entries) - known_paths |
| 357 | if stale: |
| 358 | for k in stale: |
| 359 | del self._entries[k] |
| 360 | self._dirty = True |
| 361 | |
| 362 | def save(self) -> None: |
| 363 | """Atomically persist the cache to disk if it has changed. |
| 364 | |
| 365 | Uses ``mkstemp`` for a process-unique temp file (no concurrent-save |
| 366 | collisions), ``fsync`` for durability, then ``os.replace`` for |
| 367 | atomicity. Silently skips when there is no ``.muse`` directory |
| 368 | (e.g. in-memory unit tests). |
| 369 | """ |
| 370 | if not self._dirty or self._cache_dir is None: |
| 371 | return |
| 372 | self._cache_dir.mkdir(parents=True, exist_ok=True) |
| 373 | doc = _CacheDoc(version=_CACHE_VERSION, entries=self._entries) |
| 374 | payload = _json.dumps(doc, ensure_ascii=False, separators=(",", ":")).encode("utf-8") |
| 375 | cache_file = self._cache_dir / _CACHE_FILENAME |
| 376 | fd, tmp_path = tempfile.mkstemp( |
| 377 | dir=self._cache_dir, prefix=".stat_cache_", suffix=".tmp" |
| 378 | ) |
| 379 | try: |
| 380 | with os.fdopen(fd, "wb") as fh: |
| 381 | fh.write(payload) |
| 382 | fh.flush() |
| 383 | os.fsync(fh.fileno()) |
| 384 | except Exception: |
| 385 | try: |
| 386 | os.unlink(tmp_path) |
| 387 | except OSError: |
| 388 | pass |
| 389 | raise |
| 390 | os.replace(tmp_path, cache_file) |
| 391 | self._dirty = False |
| 392 | logger.debug("✅ stat_cache saved (%d entries)", len(self._entries)) |
| 393 | |
| 394 | def load_cache(root: pathlib.Path) -> StatCache: |
| 395 | """Convenience loader: return a ``StatCache`` for a repository root. |
| 396 | |
| 397 | Returns ``StatCache.empty()`` when *root* has no ``.muse`` directory |
| 398 | so callers never need to guard against a missing repo. |
| 399 | |
| 400 | Args: |
| 401 | root: Repository root (the directory that contains ``.muse/``). |
| 402 | |
| 403 | Returns: |
| 404 | A ``StatCache`` instance ready for use. |
| 405 | """ |
| 406 | muse_dir = root / MUSE_DIR |
| 407 | if muse_dir.is_dir(): |
| 408 | return StatCache.load(muse_dir) |
| 409 | return StatCache.empty() |
File History
11 commits
sha256:37af4ab271a64f11d7bf14b4871ebaaf5dd68fb0247521007c6b034d35aa01be
Merge branch 'fix/hub-user-read-update-path' into dev
Human
10 days ago
sha256:b7be56ec091919a612cffe7f3c8b600209d5155517443fdac0e16954c8c7d81f
Merge 'task/git-export-ignored-file-deletion' into 'dev' — …
Human
13 days ago
sha256:8de4334a98c945aace420969d389ad678aa926d4ab4e886b2ac4c4241cb3bf2b
revert: keep pyproject.toml in canonical PEP 440 form
Sonnet 4.6
patch
16 days ago
sha256:50bb615c573aa4b928fca75b60f82ba2a659910066507cec6a95c412ae22ccb9
docs: add issue docs for push have-negotiation bug (#55) an…
Sonnet 4.6
19 days ago
sha256:d90d175cded68aae1d4ffcf4858917854195d0cd8ce1fe73cee4dbc02541cb74
chore: trigger push to surface null-OID paths
Sonnet 4.6
25 days ago
sha256:e452ad9a6ace6ccc6d875a35e06caf9da5576a970c1c36133b69a891ce5fefa8
chore: prebuild timing test
Sonnet 4.6
35 days ago
sha256:0008ab6695e3e064b3e236b24fd19e538fef6a588eb0d211622f4466d919c0b1
merge: pull staging/dev — advance to 0.2.0rc12
Sonnet 4.6
patch
37 days ago
sha256:9c33d61749fff814c5226d5386aa2af7064c2c02788594a25fdd709358132eea
fix: _PROPOSAL_PREFIX_RESOLVE_LIMIT 200 → 100 to match hub …
Sonnet 4.6
48 days ago
sha256:36c3cb3e76619d4c30a6d9bf81b5ec4ff148e30dcfed913e3114ca7b43b81c7e
fix: rename objects→blobs in push client and all stale test…
Sonnet 4.6
patch
51 days ago
sha256:c06a9b9b9fee26c68ea725b44d54b2c0a171301ce9de746d5b656617b4463a9a
fix: repair four test failures from post-migration audit
Sonnet 4.6
patch
57 days ago
sha256:1900655993c83c4107067375548a7be823e471d2515830842f1a12cba4bd3cdf
fix: unified object store migration — idempotent writes, JS…
Sonnet 4.6
minor
⚠
57 days ago