snapshot.py
python
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8
docs(KD-STAGING): sync governance after KD-6b DONE
Human
13 days ago
| 1 | """Pure filesystem snapshot logic for ``muse commit``. |
| 2 | |
| 3 | All functions here are side-effect-free (no DB, no I/O besides reading |
| 4 | files under ``workdir``). They are kept separate so they can be |
| 5 | unit-tested without a database. |
| 6 | |
| 7 | ID derivation contract (deterministic, no random/UUID components): |
| 8 | |
| 9 | object_id = sha256(file_bytes).hexdigest() |
| 10 | |
| 11 | snapshot_id = sha256( |
| 12 | NUL.join(sorted(f"{path}NUL{oid}" for path, oid in manifest.items())) |
| 13 | ).hexdigest() |
| 14 | |
| 15 | commit_id = sha256( |
| 16 | NUL.join([NUL.join(sorted(parent_ids)), |
| 17 | snapshot_id, message, committed_at_iso]) |
| 18 | ).hexdigest() |
| 19 | |
| 20 | The null byte (\\x00) is used as the field separator because it is: |
| 21 | - Illegal in POSIX filenames (preventing separator-injection attacks from |
| 22 | crafted file paths). |
| 23 | - Absent from SHA-256 hex strings (preventing injection via object IDs). |
| 24 | - Absent from ISO-8601 timestamps and typical message text. |
| 25 | |
| 26 | This replaces the previous ``|`` / ``:`` separator scheme which allowed two |
| 27 | distinct manifests or commit inputs to produce the same hash if filenames |
| 28 | contained those characters. |
| 29 | |
| 30 | Symlinks in the working tree are excluded from snapshots. Following a |
| 31 | symlink that points outside state/ would silently commit the contents |
| 32 | of arbitrary filesystem paths. |
| 33 | |
| 34 | Exclusion policy |
| 35 | ---------------- |
| 36 | Dotfiles and dot-directories are **tracked by default** — ``.cursorrules``, |
| 37 | ``.editorconfig``, ``.eslintrc`` are intentional project configuration that |
| 38 | collaborators need. Exclusion is driven entirely by ``.museignore`` plus the |
| 39 | built-in secrets blocklist below. The only hard-coded directory skip is |
| 40 | ``.muse/`` itself (internal VCS storage) and a performance-only list of |
| 41 | directories that are universally noise (``node_modules/``, ``__pycache__/``, |
| 42 | ``.venv/`` etc.). |
| 43 | """ |
| 44 | |
| 45 | from __future__ import annotations |
| 46 | |
| 47 | import fnmatch |
| 48 | import hashlib |
| 49 | import json |
| 50 | import os |
| 51 | import pathlib |
| 52 | import re |
| 53 | import stat as _stat |
| 54 | |
| 55 | from muse.core._types import Manifest |
| 56 | from muse.core.ignore import is_ignored, load_ignore_config, resolve_patterns |
| 57 | from muse.core.stat_cache import load_cache |
| 58 | |
| 59 | # Directories that are always pruned before os.walk descends into them. |
| 60 | # These are either internal VCS storage (.muse) or universally-noisy |
| 61 | # directories whose contents are never meaningful project source. |
| 62 | # Kept as a frozenset for O(1) lookup inside the hot walk loop. |
| 63 | _ALWAYS_PRUNE_DIRS: frozenset[str] = frozenset( |
| 64 | { |
| 65 | ".muse", |
| 66 | ".git", |
| 67 | "node_modules", |
| 68 | "__pycache__", |
| 69 | ".venv", |
| 70 | "venv", |
| 71 | ".tox", |
| 72 | ".nox", |
| 73 | ".mypy_cache", |
| 74 | ".ruff_cache", |
| 75 | ".pytest_cache", |
| 76 | ".coverage", |
| 77 | "htmlcov", |
| 78 | "dist", |
| 79 | "build", |
| 80 | } |
| 81 | ) |
| 82 | |
| 83 | # Built-in secrets blocklist — applied even when .museignore is absent. |
| 84 | # This is the last line of defence: these files must never appear in a |
| 85 | # snapshot regardless of what a user configures in .museignore. |
| 86 | # |
| 87 | # Note: .env.example is intentionally NOT listed here — it is the universal |
| 88 | # convention for a safe, credential-free environment template and must be |
| 89 | # trackable. We block the real secret files explicitly instead of using a |
| 90 | # wildcard that would accidentally catch .env.example. |
| 91 | _BUILTIN_SECRET_PATTERNS: list[str] = [ |
| 92 | ".env", |
| 93 | ".env.local", |
| 94 | ".env.development", |
| 95 | ".env.staging", |
| 96 | ".env.production", |
| 97 | ".env.prod", |
| 98 | ".envrc", |
| 99 | "*.pem", |
| 100 | "*.key", |
| 101 | "*.p12", |
| 102 | "*.pfx", |
| 103 | ".DS_Store", |
| 104 | "Thumbs.db", |
| 105 | ] |
| 106 | |
| 107 | |
| 108 | def _build_filename_filter(patterns: list[str]) -> re.Pattern[str] | None: |
| 109 | """Compile a combined regex for fast per-file ignore pre-rejection. |
| 110 | |
| 111 | Translates every *simple* pattern (no ``/`` in the body) into a single |
| 112 | alternating regex so ``re.search(fname)`` can reject the overwhelming |
| 113 | majority of files in one call instead of N ``fnmatch`` calls. |
| 114 | |
| 115 | Only patterns without ``/`` in their body are included — they test the |
| 116 | filename component. Patterns with an embedded ``/`` (e.g. |
| 117 | ``docs/*.md``) or a trailing ``/`` (directory patterns) must still go |
| 118 | through the full :func:`~muse.core.ignore.is_ignored` path. |
| 119 | |
| 120 | Returns ``None`` when *patterns* is empty or contains no simple patterns. |
| 121 | |
| 122 | Performance: at 75 000 files with 9 builtin patterns, replacing 9 × N |
| 123 | ``fnmatch`` calls with one ``re.search`` call reduces ignore-matching |
| 124 | overhead by ~10×, dropping warm ``walk_workdir`` time from ~850 ms to |
| 125 | ~85 ms at 75 k scale, making the 1-file-change target of < 200 ms |
| 126 | achievable. |
| 127 | """ |
| 128 | translated: list[str] = [] |
| 129 | for raw_pat in patterns: |
| 130 | body = raw_pat.lstrip("!") # strip negation marker |
| 131 | if body.endswith("/"): |
| 132 | body = body.rstrip("/") |
| 133 | if "/" in body: |
| 134 | continue # path-level pattern — needs full is_ignored evaluation |
| 135 | translated.append(fnmatch.translate(body)) |
| 136 | if not translated: |
| 137 | return None |
| 138 | return re.compile("(?:" + "|".join(translated) + ")") |
| 139 | |
| 140 | |
| 141 | def load_ignore_patterns(workdir: pathlib.Path) -> list[str]: |
| 142 | """Return the combined ignore pattern list for *workdir*. |
| 143 | |
| 144 | Reads ``.museignore`` from *workdir* and detects the active domain from |
| 145 | ``.muse/repo.json``. Falls back to ``"code"`` when either file is absent. |
| 146 | The built-in secrets blocklist is always prepended so it cannot be |
| 147 | overridden by user configuration. |
| 148 | |
| 149 | This function is intentionally public so that commands outside |
| 150 | ``snapshot.py`` (e.g. ``stash``) can apply the same ignore rules without |
| 151 | duplicating the domain-detection logic. |
| 152 | """ |
| 153 | domain = "code" |
| 154 | repo_json = workdir / ".muse" / "repo.json" |
| 155 | if repo_json.exists(): |
| 156 | try: |
| 157 | raw = json.loads(repo_json.read_text(encoding="utf-8")) |
| 158 | if isinstance(raw, dict) and isinstance(raw.get("domain"), str): |
| 159 | domain = raw["domain"] |
| 160 | except (OSError, json.JSONDecodeError): |
| 161 | pass |
| 162 | |
| 163 | config = load_ignore_config(workdir) |
| 164 | user_patterns = resolve_patterns(config, domain) |
| 165 | return _BUILTIN_SECRET_PATTERNS + user_patterns |
| 166 | |
| 167 | _SEP = "\x00" |
| 168 | |
| 169 | |
| 170 | def hash_file(path: pathlib.Path) -> str: |
| 171 | """Return the sha256 hex digest of a file's raw bytes. |
| 172 | |
| 173 | This is the ``object_id`` for the given file. Reading in chunks |
| 174 | keeps memory usage constant regardless of file size. |
| 175 | """ |
| 176 | h = hashlib.sha256() |
| 177 | with path.open("rb") as fh: |
| 178 | for chunk in iter(lambda: fh.read(65536), b""): |
| 179 | h.update(chunk) |
| 180 | return h.hexdigest() |
| 181 | |
| 182 | |
| 183 | def build_snapshot_manifest(workdir: pathlib.Path) -> Manifest: |
| 184 | """Return ``{rel_path: object_id}`` for every tracked file in *workdir*. |
| 185 | |
| 186 | Preferred public name; delegates to :func:`walk_workdir`. |
| 187 | """ |
| 188 | return walk_workdir(workdir) |
| 189 | |
| 190 | |
| 191 | def directories_from_manifest(files: Manifest) -> list[str]: |
| 192 | """Derive all implicit parent directories from a file manifest. |
| 193 | |
| 194 | For every file path in *files*, all ancestor directory components are |
| 195 | collected. The result is a sorted, deduplicated list of POSIX directory |
| 196 | paths relative to the repository root. |
| 197 | |
| 198 | Empty directories that have no files are not present in *files* and |
| 199 | therefore cannot be derived here — they require an explicit ``.musekeep`` |
| 200 | marker file so the filesystem walk in :func:`walk_workdir_with_dirs` can |
| 201 | detect them. |
| 202 | |
| 203 | This helper is used by merge / rebase / cherry-pick operations that |
| 204 | compute a merged file manifest without performing a fresh filesystem |
| 205 | walk, so that every ``SnapshotRecord`` stores a consistent directory list. |
| 206 | """ |
| 207 | dirs: set[str] = set() |
| 208 | for path in files: |
| 209 | parts = path.split("/") |
| 210 | for i in range(1, len(parts)): |
| 211 | dirs.add("/".join(parts[:i])) |
| 212 | return sorted(dirs) |
| 213 | |
| 214 | |
| 215 | def walk_workdir_with_dirs( |
| 216 | workdir: pathlib.Path, |
| 217 | ) -> tuple[dict[str, str], list[str]]: |
| 218 | """Walk *workdir* and return ``(files_manifest, sorted_directories)``. |
| 219 | |
| 220 | A single ``os.walk`` pass collects both the file content map and the |
| 221 | list of every non-root directory encountered (minus always-pruned dirs). |
| 222 | This is the canonical entry point for commit and status operations that |
| 223 | need first-class directory identity. |
| 224 | |
| 225 | See :func:`walk_workdir` for the exclusion rules that apply to files. |
| 226 | Directories follow the same pruning rules — any directory whose name is |
| 227 | in :data:`_ALWAYS_PRUNE_DIRS` is never descended into and therefore |
| 228 | never appears in the returned list. |
| 229 | """ |
| 230 | ignore_patterns = load_ignore_patterns(workdir) |
| 231 | cache = load_cache(workdir) |
| 232 | manifest: Manifest = {} |
| 233 | dirs: list[str] = [] |
| 234 | root_str = str(workdir) |
| 235 | prefix_len = len(root_str) + 1 |
| 236 | |
| 237 | _filename_filter: re.Pattern[str] | None = _build_filename_filter(ignore_patterns) |
| 238 | _has_complex_patterns: bool = any( |
| 239 | "/" in p.lstrip("!") |
| 240 | for p in ignore_patterns |
| 241 | ) |
| 242 | |
| 243 | for dirpath, dirnames, filenames in os.walk(root_str, followlinks=False): |
| 244 | # Prune always-excluded names and any subdirectory that is itself a |
| 245 | # nested muse repo (contains .muse/). Nested repos are independent |
| 246 | # version-controlled units — their contents belong to their own |
| 247 | # snapshot, not the parent repo's. |
| 248 | dirnames[:] = [ |
| 249 | d for d in dirnames |
| 250 | if d not in _ALWAYS_PRUNE_DIRS |
| 251 | and not os.path.isdir(os.path.join(dirpath, d, ".muse")) |
| 252 | ] |
| 253 | |
| 254 | # Track every non-root directory we descend into. |
| 255 | if dirpath != root_str: |
| 256 | rel_dir = dirpath[prefix_len:] |
| 257 | if os.sep != "/": |
| 258 | rel_dir = rel_dir.replace(os.sep, "/") |
| 259 | dirs.append(rel_dir) |
| 260 | |
| 261 | for fname in filenames: |
| 262 | abs_str = os.path.join(dirpath, fname) |
| 263 | try: |
| 264 | st = os.lstat(abs_str) |
| 265 | except OSError: |
| 266 | continue |
| 267 | if not _stat.S_ISREG(st.st_mode): |
| 268 | continue |
| 269 | rel = abs_str[prefix_len:] |
| 270 | if os.sep != "/": |
| 271 | rel = rel.replace(os.sep, "/") |
| 272 | if ( |
| 273 | _filename_filter is not None |
| 274 | and not _filename_filter.search(fname) |
| 275 | and not _has_complex_patterns |
| 276 | ): |
| 277 | manifest[rel] = cache.get_cached(rel, abs_str, st.st_mtime, st.st_size, st.st_ino) |
| 278 | continue |
| 279 | if is_ignored(rel, ignore_patterns): |
| 280 | continue |
| 281 | manifest[rel] = cache.get_cached(rel, abs_str, st.st_mtime, st.st_size, st.st_ino) |
| 282 | |
| 283 | cache.prune(set(manifest)) |
| 284 | cache.save() |
| 285 | return manifest, sorted(dirs) |
| 286 | |
| 287 | |
| 288 | def walk_workdir(workdir: pathlib.Path) -> Manifest: |
| 289 | """Walk *workdir* and return only the file manifest. |
| 290 | |
| 291 | Thin wrapper around :func:`walk_workdir_with_dirs` that discards the |
| 292 | directory list. Callers that need both files and directories should call |
| 293 | :func:`walk_workdir_with_dirs` directly to avoid a second filesystem walk. |
| 294 | |
| 295 | Walk *workdir* recursively and return ``{rel_path: object_id}``. |
| 296 | |
| 297 | Exclusions (all silent, no warning emitted): |
| 298 | - Symlinks — following them could commit content from outside the repo. |
| 299 | - Non-regular files — only regular files are included. |
| 300 | - Paths matched by ``.museignore`` or the built-in secrets blocklist. |
| 301 | - Directories in ``_ALWAYS_PRUNE_DIRS`` — internal VCS storage and |
| 302 | universally-noisy directories (node_modules, __pycache__, .venv, …). |
| 303 | |
| 304 | Dotfiles and dot-directories are tracked unless excluded by the above |
| 305 | rules. ``.cursorrules``, ``.editorconfig``, ``.eslintrc`` etc. are |
| 306 | intentional project configuration; the blanket dot-skip that Git-adjacent |
| 307 | tools inherited is not carried forward here. |
| 308 | |
| 309 | Paths use POSIX separators regardless of host OS for cross-platform |
| 310 | reproducibility. |
| 311 | |
| 312 | Performance note: ``os.walk`` with in-place ``dirnames`` pruning is used |
| 313 | instead of ``pathlib.rglob`` so that large noisy directories are never |
| 314 | descended into. The stat cache further skips re-hashing files whose |
| 315 | ``(mtime, size)`` is unchanged since the last walk. |
| 316 | |
| 317 | Ignore-pattern fast path: patterns are compiled into a single combined |
| 318 | regex (see :func:`_build_filename_filter`) that is evaluated against the |
| 319 | bare filename once per file. For the builtin secrets blocklist (9 simple |
| 320 | ``*.ext`` / ``name`` patterns with no ``/``), this replaces 9 separate |
| 321 | ``fnmatch`` calls with one ``re.search`` call — a ~10× speedup at 75 k |
| 322 | scale that brings warm 1-file-change latency from ~850 ms to < 200 ms. |
| 323 | Files whose filename can't possibly match any pattern skip ``is_ignored`` |
| 324 | entirely; files that might match (rare) fall through to the full check. |
| 325 | """ |
| 326 | files, _ = walk_workdir_with_dirs(workdir) |
| 327 | return files |
| 328 | |
| 329 | |
| 330 | def compute_snapshot_id( |
| 331 | manifest: Manifest, |
| 332 | directories: list[str] | None = None, |
| 333 | ) -> str: |
| 334 | """Return sha256 of the sorted ``path NUL object_id`` pairs and directory paths. |
| 335 | |
| 336 | The null-byte separator prevents collisions from filenames or object IDs |
| 337 | that contain the previous ``|`` / ``:`` separators. |
| 338 | |
| 339 | Sorting ensures two identical working trees always produce the same |
| 340 | snapshot_id, regardless of filesystem traversal order. |
| 341 | |
| 342 | When *directories* is provided (non-empty), directory paths are appended |
| 343 | to the hash payload so that a directory rename produces a different |
| 344 | snapshot ID even when file contents are unchanged. Callers that do not |
| 345 | yet track directories may omit the argument; the resulting ID is identical |
| 346 | to the pre-directory-tracking behaviour. |
| 347 | """ |
| 348 | parts = sorted(f"{path}{_SEP}{oid}" for path, oid in manifest.items()) |
| 349 | if directories: |
| 350 | # Prefix directory entries with "dir" so they occupy a distinct namespace |
| 351 | # from file entries and cannot collide with path/oid pairs. |
| 352 | parts.extend(f"dir{_SEP}{d}" for d in sorted(directories)) |
| 353 | payload = _SEP.join(parts).encode() |
| 354 | return hashlib.sha256(payload).hexdigest() |
| 355 | |
| 356 | |
| 357 | def detect_directory_renames( |
| 358 | deleted_dirs: set[str], |
| 359 | added_dirs: set[str], |
| 360 | last_manifest: Manifest, |
| 361 | current_manifest: Manifest, |
| 362 | ) -> list[tuple[str, str]]: |
| 363 | """Return ``[(old_dir, new_dir)]`` pairs detected from manifest diffs. |
| 364 | |
| 365 | A directory rename is inferred when all files that were under *old_dir* |
| 366 | in *last_manifest* appear under *new_dir* in *current_manifest* with |
| 367 | identical object IDs (same content, different path). Empty directories |
| 368 | and directories whose file sets do not match any added directory are not |
| 369 | returned. |
| 370 | |
| 371 | The heuristic is conservative: only 1-to-1 renames are reported. If |
| 372 | multiple added directories share the same file set (unusual but possible), |
| 373 | the match is ambiguous and no rename is emitted for that pair. |
| 374 | """ |
| 375 | renames: list[tuple[str, str]] = [] |
| 376 | matched_new: set[str] = set() |
| 377 | |
| 378 | for old_dir in sorted(deleted_dirs): |
| 379 | prefix = old_dir + "/" |
| 380 | old_files = { |
| 381 | path[len(prefix):]: oid |
| 382 | for path, oid in last_manifest.items() |
| 383 | if path.startswith(prefix) |
| 384 | } |
| 385 | if not old_files: |
| 386 | continue # empty dir — can't match by content |
| 387 | |
| 388 | candidates = [ |
| 389 | new_dir for new_dir in sorted(added_dirs) |
| 390 | if new_dir not in matched_new |
| 391 | ] |
| 392 | for new_dir in candidates: |
| 393 | new_prefix = new_dir + "/" |
| 394 | new_files = { |
| 395 | path[len(new_prefix):]: oid |
| 396 | for path, oid in current_manifest.items() |
| 397 | if path.startswith(new_prefix) |
| 398 | } |
| 399 | if new_files == old_files: |
| 400 | renames.append((old_dir, new_dir)) |
| 401 | matched_new.add(new_dir) |
| 402 | break |
| 403 | |
| 404 | return renames |
| 405 | |
| 406 | |
| 407 | def diff_workdir_vs_snapshot( |
| 408 | workdir: pathlib.Path, |
| 409 | last_manifest: Manifest, |
| 410 | last_directories: list[str] | None = None, |
| 411 | ) -> tuple[set[str], set[str], set[str], set[str], set[str], set[str]]: |
| 412 | """Compare *workdir* against *last_manifest* from the previous commit. |
| 413 | |
| 414 | Returns a tuple of six disjoint path sets: |
| 415 | |
| 416 | - ``added`` — files in *workdir* absent from *last_manifest*. |
| 417 | - ``modified`` — files present in both but with a differing sha256 hash. |
| 418 | - ``deleted`` — files in *last_manifest* absent from *workdir*. |
| 419 | - ``untracked`` — non-empty only when *last_manifest* is empty (first |
| 420 | commit): every file in *workdir* is untracked. |
| 421 | - ``added_dirs`` — directories present in *workdir* but not in |
| 422 | *last_directories*. |
| 423 | - ``deleted_dirs``— directories in *last_directories* absent from *workdir*. |
| 424 | |
| 425 | All paths use POSIX separators for cross-platform reproducibility. |
| 426 | """ |
| 427 | if not workdir.exists(): |
| 428 | return ( |
| 429 | set(), set(), |
| 430 | set(last_manifest.keys()), set(), |
| 431 | set(), set(last_directories or []), |
| 432 | ) |
| 433 | |
| 434 | current_manifest, current_dirs = walk_workdir_with_dirs(workdir) |
| 435 | current_paths = set(current_manifest.keys()) |
| 436 | last_paths = set(last_manifest.keys()) |
| 437 | |
| 438 | if not last_paths: |
| 439 | return set(), set(), set(), current_paths, set(current_dirs), set() |
| 440 | |
| 441 | added = current_paths - last_paths |
| 442 | deleted = last_paths - current_paths |
| 443 | common = current_paths & last_paths |
| 444 | modified = {p for p in common if current_manifest[p] != last_manifest[p]} |
| 445 | |
| 446 | # A file that was tracked in the last snapshot but is now listed in |
| 447 | # .museignore and still present on disk is not "deleted" — it has been |
| 448 | # intentionally moved out of tracking. Reporting it as deleted would |
| 449 | # block checkout, pollute status output, and cause stash pop to unlink it. |
| 450 | # Only files that are genuinely absent from the working tree are deleted. |
| 451 | if deleted: |
| 452 | ignore_patterns = load_ignore_patterns(workdir) |
| 453 | deleted = { |
| 454 | p for p in deleted |
| 455 | if not (is_ignored(p, ignore_patterns) and (workdir / p).exists()) |
| 456 | } |
| 457 | |
| 458 | last_dirs_set = set(last_directories or []) |
| 459 | current_dirs_set = set(current_dirs) |
| 460 | added_dirs = current_dirs_set - last_dirs_set |
| 461 | deleted_dirs = last_dirs_set - current_dirs_set |
| 462 | |
| 463 | return added, modified, deleted, set(), added_dirs, deleted_dirs |
| 464 | |
| 465 | |
| 466 | def compute_commit_id( |
| 467 | parent_ids: list[str], |
| 468 | snapshot_id: str, |
| 469 | message: str, |
| 470 | committed_at_iso: str, |
| 471 | ) -> str: |
| 472 | """Return sha256 of the commit's canonical inputs. |
| 473 | |
| 474 | Uses null bytes as field separators to prevent separator-injection |
| 475 | attacks from commit messages, author names, or branch names containing |
| 476 | ``|`` characters. |
| 477 | |
| 478 | Given the same arguments on two machines the result is identical. |
| 479 | ``parent_ids`` is sorted before hashing so insertion order does not |
| 480 | affect determinism. |
| 481 | """ |
| 482 | parts = [ |
| 483 | _SEP.join(sorted(parent_ids)), |
| 484 | snapshot_id, |
| 485 | message, |
| 486 | committed_at_iso, |
| 487 | ] |
| 488 | payload = _SEP.join(parts).encode() |
| 489 | return hashlib.sha256(payload).hexdigest() |
File History
1 commit
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8
docs(KD-STAGING): sync governance after KD-6b DONE
Human
13 days ago