manifest.py
file-level
1
files
1
commits
0
hotspots
0
🧊 dead
0
💥 blast risk
| 1 | """Hierarchical code snapshot manifests for the Muse code domain. |
| 2 | |
| 3 | A code manifest organises a snapshot's files into a three-level hierarchy:: |
| 4 | |
| 5 | PackageManifest ← top-level package (directory) |
| 6 | └─ ModuleManifest ← one source file |
| 7 | └─ FileEntry ← per-file metadata + hashes |
| 8 | |
| 9 | This structure enables *partial re-parsing*: when merging, only files whose |
| 10 | ``content_hash`` changed need to be re-parsed by the AST engine. Files in |
| 11 | unchanged modules are reused from the cached manifest, making three-way |
| 12 | merges on large codebases significantly faster. |
| 13 | |
| 14 | The ``ast_hash`` in :class:`FileEntry` is a SHA-256 of the file's *symbol |
| 15 | tree* rather than its raw bytes. Two files that differ only in whitespace or |
| 16 | comments will have the same ``ast_hash``, meaning the AST engine reports |
| 17 | "no semantic change" for that file. |
| 18 | |
| 19 | Public API |
| 20 | ---------- |
| 21 | - :class:`FileEntry` — per-file metadata. |
| 22 | - :class:`ModuleManifest` — one source file's manifest entry. |
| 23 | - :class:`PackageManifest` — directory-level grouping. |
| 24 | - :class:`CodeManifest` — complete hierarchical snapshot manifest. |
| 25 | - :func:`build_code_manifest` — build from a flat snapshot manifest. |
| 26 | - :func:`diff_manifests` — find added/removed/modified files between two. |
| 27 | - :func:`write_code_manifest` — persist to ``.muse/code_manifests/<id>.json``. |
| 28 | - :func:`read_code_manifest` — load from disk. |
| 29 | """ |
| 30 | |
| 31 | from __future__ import annotations |
| 32 | |
| 33 | import hashlib |
| 34 | import json |
| 35 | import logging |
| 36 | import pathlib |
| 37 | from typing import TypedDict |
| 38 | |
| 39 | from muse.core._types import Manifest, Metadata |
| 40 | from muse.core.object_store import read_object |
| 41 | |
| 42 | logger = logging.getLogger(__name__) |
| 43 | |
| 44 | type _PkgFileMap = dict[str, list[str]] |
| 45 | type _ModuleMap = dict[str, "ModuleManifest"] |
| 46 | |
| 47 | # --------------------------------------------------------------------------- |
| 48 | # Data types |
| 49 | # --------------------------------------------------------------------------- |
| 50 | |
| 51 | |
| 52 | class FileEntry(TypedDict): |
| 53 | """Metadata for a single source file. |
| 54 | |
| 55 | ``path`` Workspace-relative POSIX path. |
| 56 | ``content_hash`` SHA-256 of the raw file bytes (from the object store). |
| 57 | ``ast_hash`` SHA-256 of the symbol-tree JSON (semantic identity). |
| 58 | Empty string when AST parsing is unavailable. |
| 59 | ``language`` Display language name (``"Python"``, ``"TypeScript"``…). |
| 60 | ``symbol_count`` Number of top-level + nested symbols extracted. |
| 61 | ``size_bytes`` Raw file size in bytes (0 if unavailable). |
| 62 | """ |
| 63 | |
| 64 | path: str |
| 65 | content_hash: str |
| 66 | ast_hash: str |
| 67 | language: str |
| 68 | symbol_count: int |
| 69 | size_bytes: int |
| 70 | |
| 71 | |
| 72 | class ModuleManifest(TypedDict): |
| 73 | """Manifest for one source file (module). |
| 74 | |
| 75 | ``module_path`` Workspace-relative POSIX path. |
| 76 | ``content_hash`` Raw-bytes SHA-256 (same as ``FileEntry.content_hash``). |
| 77 | ``ast_hash`` Symbol-tree SHA-256 for semantic change detection. |
| 78 | ``language`` Language name. |
| 79 | ``symbol_count`` Number of symbols in this file. |
| 80 | """ |
| 81 | |
| 82 | module_path: str |
| 83 | content_hash: str |
| 84 | ast_hash: str |
| 85 | language: str |
| 86 | symbol_count: int |
| 87 | |
| 88 | |
| 89 | class PackageManifest(TypedDict): |
| 90 | """Directory-level grouping of modules. |
| 91 | |
| 92 | ``package`` Workspace-relative POSIX directory path. |
| 93 | ``package_hash`` SHA-256 of all sorted ``content_hash`` values in the |
| 94 | package — stable fingerprint for change detection. |
| 95 | ``modules`` All ``ModuleManifest`` entries in this package. |
| 96 | ``total_files`` Total number of files (all types, not just semantic). |
| 97 | ``semantic_files`` Number of AST-parseable files. |
| 98 | """ |
| 99 | |
| 100 | package: str |
| 101 | package_hash: str |
| 102 | modules: list[ModuleManifest] |
| 103 | total_files: int |
| 104 | semantic_files: int |
| 105 | |
| 106 | |
| 107 | class CodeManifest(TypedDict): |
| 108 | """Complete hierarchical manifest for one code snapshot. |
| 109 | |
| 110 | ``snapshot_id`` The snapshot this manifest was built from. |
| 111 | ``manifest_hash`` SHA-256 of this manifest's JSON — stable cache key. |
| 112 | ``packages`` All :class:`PackageManifest` entries, sorted by path. |
| 113 | ``total_files`` Total files in the snapshot. |
| 114 | ``semantic_files`` AST-parseable files. |
| 115 | ``total_symbols`` Sum of ``symbol_count`` across all modules. |
| 116 | """ |
| 117 | |
| 118 | snapshot_id: str |
| 119 | manifest_hash: str |
| 120 | packages: list[PackageManifest] |
| 121 | total_files: int |
| 122 | semantic_files: int |
| 123 | total_symbols: int |
| 124 | |
| 125 | |
| 126 | # --------------------------------------------------------------------------- |
| 127 | # Language detection |
| 128 | # --------------------------------------------------------------------------- |
| 129 | |
| 130 | _SUFFIX_LANG: Metadata = { |
| 131 | ".py": "Python", ".pyi": "Python", |
| 132 | ".ts": "TypeScript", ".tsx": "TypeScript", |
| 133 | ".js": "JavaScript", ".jsx": "JavaScript", |
| 134 | ".mjs": "JavaScript", ".cjs": "JavaScript", |
| 135 | ".go": "Go", |
| 136 | ".rs": "Rust", |
| 137 | ".java": "Java", |
| 138 | ".cs": "C#", |
| 139 | ".c": "C", ".h": "C", |
| 140 | ".cpp": "C++", ".cc": "C++", ".cxx": "C++", ".hpp": "C++", |
| 141 | ".rb": "Ruby", |
| 142 | ".kt": "Kotlin", ".kts": "Kotlin", |
| 143 | } |
| 144 | |
| 145 | _SEMANTIC_SUFFIXES = frozenset(_SUFFIX_LANG) |
| 146 | |
| 147 | |
| 148 | def _language_of(file_path: str) -> str: |
| 149 | suffix = pathlib.PurePosixPath(file_path).suffix.lower() |
| 150 | return _SUFFIX_LANG.get(suffix, suffix or "(no ext)") |
| 151 | |
| 152 | |
| 153 | def _is_semantic(file_path: str) -> bool: |
| 154 | suffix = pathlib.PurePosixPath(file_path).suffix.lower() |
| 155 | return suffix in _SEMANTIC_SUFFIXES |
| 156 | |
| 157 | |
| 158 | # --------------------------------------------------------------------------- |
| 159 | # Builder |
| 160 | # --------------------------------------------------------------------------- |
| 161 | |
| 162 | |
| 163 | def build_code_manifest( |
| 164 | snapshot_id: str, |
| 165 | flat_manifest: Manifest, |
| 166 | repo_root: pathlib.Path, |
| 167 | ) -> CodeManifest: |
| 168 | """Build a :class:`CodeManifest` from a flat ``{path: content_hash}`` dict. |
| 169 | |
| 170 | Attempts to parse each semantic file into a symbol tree to compute |
| 171 | ``ast_hash`` and ``symbol_count``. Falls back gracefully for binary files |
| 172 | or parse errors. |
| 173 | |
| 174 | Args: |
| 175 | snapshot_id: The snapshot this manifest represents. |
| 176 | flat_manifest: ``{workspace_path: sha256}`` from the snapshot. |
| 177 | repo_root: Repository root for object store access. |
| 178 | |
| 179 | Returns: |
| 180 | A fully populated :class:`CodeManifest`. |
| 181 | """ |
| 182 | # Late import to avoid circular dependencies. |
| 183 | from muse.plugins.code.ast_parser import parse_symbols |
| 184 | |
| 185 | # Group files by parent directory. |
| 186 | pkg_files: _PkgFileMap = {} |
| 187 | for file_path in sorted(flat_manifest): |
| 188 | pkg = str(pathlib.PurePosixPath(file_path).parent) |
| 189 | pkg_files.setdefault(pkg, []).append(file_path) |
| 190 | |
| 191 | packages: list[PackageManifest] = [] |
| 192 | total_symbols = 0 |
| 193 | total_semantic = 0 |
| 194 | |
| 195 | for pkg_path, file_paths in sorted(pkg_files.items()): |
| 196 | modules: list[ModuleManifest] = [] |
| 197 | pkg_hashes: list[str] = [] |
| 198 | pkg_semantic = 0 |
| 199 | |
| 200 | for file_path in sorted(file_paths): |
| 201 | content_hash = flat_manifest[file_path] |
| 202 | lang = _language_of(file_path) |
| 203 | is_sem = _is_semantic(file_path) |
| 204 | ast_hash = "" |
| 205 | sym_count = 0 |
| 206 | |
| 207 | if is_sem: |
| 208 | source = read_object(repo_root, content_hash) |
| 209 | if source is not None: |
| 210 | try: |
| 211 | symbols = parse_symbols(source, file_path) |
| 212 | sym_count = len(symbols) |
| 213 | # ast_hash = SHA-256 of sorted symbol content IDs. |
| 214 | sig = hashlib.sha256( |
| 215 | "|".join( |
| 216 | sorted(s["content_id"] for s in symbols.values()) |
| 217 | ).encode() |
| 218 | ).hexdigest() |
| 219 | ast_hash = sig |
| 220 | except Exception: |
| 221 | logger.debug("AST parse failed for %s", file_path) |
| 222 | pkg_semantic += 1 |
| 223 | total_semantic += 1 |
| 224 | |
| 225 | total_symbols += sym_count |
| 226 | pkg_hashes.append(content_hash) |
| 227 | modules.append(ModuleManifest( |
| 228 | module_path=file_path, |
| 229 | content_hash=content_hash, |
| 230 | ast_hash=ast_hash, |
| 231 | language=lang, |
| 232 | symbol_count=sym_count, |
| 233 | )) |
| 234 | |
| 235 | pkg_hash = hashlib.sha256("|".join(sorted(pkg_hashes)).encode()).hexdigest() |
| 236 | packages.append(PackageManifest( |
| 237 | package=pkg_path, |
| 238 | package_hash=pkg_hash, |
| 239 | modules=modules, |
| 240 | total_files=len(file_paths), |
| 241 | semantic_files=pkg_semantic, |
| 242 | )) |
| 243 | |
| 244 | manifest_json = json.dumps( |
| 245 | {"snapshot_id": snapshot_id, "packages": packages}, sort_keys=True |
| 246 | ) |
| 247 | manifest_hash = hashlib.sha256(manifest_json.encode()).hexdigest() |
| 248 | |
| 249 | return CodeManifest( |
| 250 | snapshot_id=snapshot_id, |
| 251 | manifest_hash=manifest_hash, |
| 252 | packages=packages, |
| 253 | total_files=len(flat_manifest), |
| 254 | semantic_files=total_semantic, |
| 255 | total_symbols=total_symbols, |
| 256 | ) |
| 257 | |
| 258 | |
| 259 | # --------------------------------------------------------------------------- |
| 260 | # Diff |
| 261 | # --------------------------------------------------------------------------- |
| 262 | |
| 263 | |
| 264 | class ManifestFileDiff(TypedDict): |
| 265 | """Change record from :func:`diff_manifests` for one file.""" |
| 266 | |
| 267 | path: str |
| 268 | change: str # "added" | "removed" | "modified" | "ast_changed" |
| 269 | old_hash: str |
| 270 | new_hash: str |
| 271 | old_ast_hash: str |
| 272 | new_ast_hash: str |
| 273 | semantic_change: bool # True when ast_hash differs (real code change) |
| 274 | |
| 275 | |
| 276 | def diff_manifests( |
| 277 | base: CodeManifest, |
| 278 | target: CodeManifest, |
| 279 | ) -> list[ManifestFileDiff]: |
| 280 | """Produce a per-file change list between two :class:`CodeManifest` objects. |
| 281 | |
| 282 | Files with identical ``content_hash`` values are skipped (no change). |
| 283 | Files where only the ``content_hash`` changed but ``ast_hash`` is the same |
| 284 | are marked ``"modified"`` with ``semantic_change=False`` — e.g. whitespace |
| 285 | or comment-only diffs. Files where ``ast_hash`` changed are ``"ast_changed"`` |
| 286 | with ``semantic_change=True``. |
| 287 | |
| 288 | Args: |
| 289 | base: Manifest for the earlier state (e.g. parent commit). |
| 290 | target: Manifest for the later state (e.g. current commit). |
| 291 | |
| 292 | Returns: |
| 293 | Sorted list of :class:`ManifestFileDiff` records. |
| 294 | """ |
| 295 | base_modules: _ModuleMap = {} |
| 296 | for pkg in base["packages"]: |
| 297 | for mod in pkg["modules"]: |
| 298 | base_modules[mod["module_path"]] = mod |
| 299 | |
| 300 | target_modules: _ModuleMap = {} |
| 301 | for pkg in target["packages"]: |
| 302 | for mod in pkg["modules"]: |
| 303 | target_modules[mod["module_path"]] = mod |
| 304 | |
| 305 | diffs: list[ManifestFileDiff] = [] |
| 306 | |
| 307 | all_paths = sorted(set(base_modules) | set(target_modules)) |
| 308 | for path in all_paths: |
| 309 | bm = base_modules.get(path) |
| 310 | tm = target_modules.get(path) |
| 311 | |
| 312 | if bm is None and tm is not None: |
| 313 | diffs.append(ManifestFileDiff( |
| 314 | path=path, change="added", |
| 315 | old_hash="", new_hash=tm["content_hash"], |
| 316 | old_ast_hash="", new_ast_hash=tm["ast_hash"], |
| 317 | semantic_change=True, |
| 318 | )) |
| 319 | elif bm is not None and tm is None: |
| 320 | diffs.append(ManifestFileDiff( |
| 321 | path=path, change="removed", |
| 322 | old_hash=bm["content_hash"], new_hash="", |
| 323 | old_ast_hash=bm["ast_hash"], new_ast_hash="", |
| 324 | semantic_change=True, |
| 325 | )) |
| 326 | elif bm is not None and tm is not None: |
| 327 | if bm["content_hash"] == tm["content_hash"]: |
| 328 | continue # No change at all. |
| 329 | ast_changed = bm["ast_hash"] != tm["ast_hash"] |
| 330 | diffs.append(ManifestFileDiff( |
| 331 | path=path, |
| 332 | change="ast_changed" if ast_changed else "modified", |
| 333 | old_hash=bm["content_hash"], new_hash=tm["content_hash"], |
| 334 | old_ast_hash=bm["ast_hash"], new_ast_hash=tm["ast_hash"], |
| 335 | semantic_change=ast_changed, |
| 336 | )) |
| 337 | |
| 338 | return diffs |
| 339 | |
| 340 | |
| 341 | # --------------------------------------------------------------------------- |
| 342 | # Persistence |
| 343 | # --------------------------------------------------------------------------- |
| 344 | |
| 345 | |
| 346 | def write_code_manifest(repo_root: pathlib.Path, manifest: CodeManifest) -> None: |
| 347 | """Persist a :class:`CodeManifest` to ``.muse/code_manifests/<hash>.json``. |
| 348 | |
| 349 | Args: |
| 350 | repo_root: Repository root. |
| 351 | manifest: The manifest to write. |
| 352 | """ |
| 353 | store_dir = repo_root / ".muse" / "code_manifests" |
| 354 | store_dir.mkdir(parents=True, exist_ok=True) |
| 355 | path = store_dir / f"{manifest['manifest_hash']}.json" |
| 356 | if not path.exists(): |
| 357 | path.write_text(json.dumps(manifest)) |
| 358 | |
| 359 | |
| 360 | def read_code_manifest( |
| 361 | repo_root: pathlib.Path, manifest_hash: str |
| 362 | ) -> CodeManifest | None: |
| 363 | """Load a :class:`CodeManifest` by its hash. |
| 364 | |
| 365 | Args: |
| 366 | repo_root: Repository root. |
| 367 | manifest_hash: The ``manifest_hash`` of the target manifest. |
| 368 | |
| 369 | Returns: |
| 370 | The deserialized :class:`CodeManifest`, or ``None`` if not found. |
| 371 | """ |
| 372 | path = repo_root / ".muse" / "code_manifests" / f"{manifest_hash}.json" |
| 373 | if not path.exists(): |
| 374 | return None |
| 375 | raw: CodeManifest = json.loads(path.read_text()) |
| 376 | return raw |