gabriel / muse public
manifest.py python
357 lines 12.7 KB
Raw
sha256:5eff0848db1d0748f3a195ca871a9275e7455d447690411ec2d83434e11baa65 feat(#216): persist write-once BranchMeta creation provenance Human minor ⚠ breaking 1 day ago
1 """Hierarchical code snapshot manifests for the Muse code domain.
2
3 A code manifest organises a snapshot's files into a three-level hierarchy::
4
5 PackageManifest ← top-level package (directory)
6 └─ ModuleManifest ← one source file
7 └─ FileEntry ← per-file metadata + hashes
8
9 This structure enables *partial re-parsing*: when merging, only files whose
10 ``content_hash`` changed need to be re-parsed by the AST engine. Files in
11 unchanged modules are reused from the cached manifest, making three-way
12 merges on large codebases significantly faster.
13
14 The ``ast_hash`` in :class:`FileEntry` is a SHA-256 of the file's *symbol
15 tree* rather than its raw bytes. Two files that differ only in whitespace or
16 comments will have the same ``ast_hash``, meaning the AST engine reports
17 "no semantic change" for that file.
18
19 Public API
20 ----------
21 - :class:`FileEntry` — per-file metadata.
22 - :class:`ModuleManifest` — one source file's manifest entry.
23 - :class:`PackageManifest` — directory-level grouping.
24 - :class:`CodeManifest` — complete hierarchical snapshot manifest.
25 - :func:`build_code_manifest` — build from a flat snapshot manifest.
26 - :func:`diff_manifests` — find added/removed/modified files between two.
27 - :func:`write_code_manifest` — persist to ``.muse/code_manifests/<id>.json``.
28 - :func:`read_code_manifest` — load from disk.
29 """
30
31 import json
32 import logging
33 import pathlib
34 from typing import TypedDict
35
36 from muse.core.types import Manifest, Metadata, blob_id, load_json_file, long_id
37 from muse.core.paths import code_manifests_dir as _code_manifests_dir
38 from muse.core.object_store import read_object
39
40 logger = logging.getLogger(__name__)
41
42 type _PkgFileMap = dict[str, list[str]]
43 type _ModuleMap = dict[str, "ModuleManifest"]
44
45 # ---------------------------------------------------------------------------
46 # Data types
47 # ---------------------------------------------------------------------------
48
49 class FileEntry(TypedDict):
50 """Metadata for a single source file.
51
52 ``path`` Workspace-relative POSIX path.
53 ``content_hash`` SHA-256 of the raw file bytes (from the object store).
54 ``ast_hash`` ``sha256:``-prefixed ID of the symbol-tree JSON (semantic identity).
55 Empty string when AST parsing is unavailable.
56 ``language`` Display language name (``"Python"``, ``"TypeScript"``…).
57 ``symbol_count`` Number of top-level + nested symbols extracted.
58 ``size_bytes`` Raw file size in bytes (0 if unavailable).
59 """
60
61 path: str
62 content_hash: str
63 ast_hash: str
64 language: str
65 symbol_count: int
66 size_bytes: int
67
68 class ModuleManifest(TypedDict):
69 """Manifest for one source file (module).
70
71 ``module_path`` Workspace-relative POSIX path.
72 ``content_hash`` Raw-bytes SHA-256 (same as ``FileEntry.content_hash``).
73 ``ast_hash`` ``sha256:``-prefixed symbol-tree ID for semantic change detection.
74 ``language`` Language name.
75 ``symbol_count`` Number of symbols in this file.
76 """
77
78 module_path: str
79 content_hash: str
80 ast_hash: str
81 language: str
82 symbol_count: int
83
84 class PackageManifest(TypedDict):
85 """Directory-level grouping of modules.
86
87 ``package`` Workspace-relative POSIX directory path.
88 ``package_hash`` SHA-256 of all sorted ``content_hash`` values in the
89 package — stable fingerprint for change detection.
90 ``modules`` All ``ModuleManifest`` entries in this package.
91 ``total_files`` Total number of files (all types, not just semantic).
92 ``semantic_files`` Number of AST-parseable files.
93 """
94
95 package: str
96 package_hash: str
97 modules: list[ModuleManifest]
98 total_files: int
99 semantic_files: int
100
101 class CodeManifest(TypedDict):
102 """Complete hierarchical manifest for one code snapshot.
103
104 ``snapshot_id`` The snapshot this manifest was built from.
105 ``manifest_hash`` SHA-256 of this manifest's JSON — stable cache key.
106 ``packages`` All :class:`PackageManifest` entries, sorted by path.
107 ``total_files`` Total files in the snapshot.
108 ``semantic_files`` AST-parseable files.
109 ``total_symbols`` Sum of ``symbol_count`` across all modules.
110 """
111
112 snapshot_id: str
113 manifest_hash: str
114 packages: list[PackageManifest]
115 total_files: int
116 semantic_files: int
117 total_symbols: int
118
119 # ---------------------------------------------------------------------------
120 # Language detection
121 # ---------------------------------------------------------------------------
122
123 _SUFFIX_LANG: Metadata = {
124 ".py": "Python", ".pyi": "Python",
125 ".ts": "TypeScript", ".tsx": "TypeScript",
126 ".js": "JavaScript", ".jsx": "JavaScript",
127 ".mjs": "JavaScript", ".cjs": "JavaScript",
128 ".go": "Go",
129 ".rs": "Rust",
130 ".java": "Java",
131 ".cs": "C#",
132 ".c": "C", ".h": "C",
133 ".cpp": "C++", ".cc": "C++", ".cxx": "C++", ".hpp": "C++",
134 ".rb": "Ruby",
135 ".kt": "Kotlin", ".kts": "Kotlin",
136 }
137
138 _SEMANTIC_SUFFIXES = frozenset(_SUFFIX_LANG)
139
140 def _language_of(file_path: str) -> str:
141 suffix = pathlib.PurePosixPath(file_path).suffix.lower()
142 return _SUFFIX_LANG.get(suffix, suffix or "(no ext)")
143
144 def _is_semantic(file_path: str) -> bool:
145 suffix = pathlib.PurePosixPath(file_path).suffix.lower()
146 return suffix in _SEMANTIC_SUFFIXES
147
148 # ---------------------------------------------------------------------------
149 # Builder
150 # ---------------------------------------------------------------------------
151
152 def build_code_manifest(
153 snapshot_id: str,
154 flat_manifest: Manifest,
155 repo_root: pathlib.Path,
156 ) -> CodeManifest:
157 """Build a :class:`CodeManifest` from a flat ``{path: content_hash}`` dict.
158
159 Attempts to parse each semantic file into a symbol tree to compute
160 ``ast_hash`` and ``symbol_count``. Falls back gracefully for binary files
161 or parse errors.
162
163 Args:
164 snapshot_id: The snapshot this manifest represents.
165 flat_manifest: ``{workspace_path: sha256}`` from the snapshot.
166 repo_root: Repository root for object store access.
167
168 Returns:
169 A fully populated :class:`CodeManifest`.
170 """
171 # Late import to avoid circular dependencies.
172 from muse.plugins.code.ast_parser import parse_symbols
173
174 # Group files by parent directory.
175 pkg_files: _PkgFileMap = {}
176 for file_path in sorted(flat_manifest):
177 pkg = str(pathlib.PurePosixPath(file_path).parent)
178 pkg_files.setdefault(pkg, []).append(file_path)
179
180 packages: list[PackageManifest] = []
181 total_symbols = 0
182 total_semantic = 0
183
184 for pkg_path, file_paths in sorted(pkg_files.items()):
185 modules: list[ModuleManifest] = []
186 pkg_hashes: list[str] = []
187 pkg_semantic = 0
188
189 for file_path in sorted(file_paths):
190 content_hash = flat_manifest[file_path]
191 lang = _language_of(file_path)
192 is_sem = _is_semantic(file_path)
193 ast_hash = ""
194 sym_count = 0
195
196 if is_sem:
197 source = read_object(repo_root, content_hash)
198 if source is not None:
199 try:
200 symbols = parse_symbols(source, file_path)
201 sym_count = len(symbols)
202 # ast_hash = sha256:-prefixed ID of sorted symbol content IDs.
203 ast_hash = blob_id(
204 "|".join(
205 sorted(s["content_id"] for s in symbols.values())
206 ).encode()
207 )
208 except Exception:
209 logger.debug("AST parse failed for %s", file_path)
210 pkg_semantic += 1
211 total_semantic += 1
212
213 total_symbols += sym_count
214 pkg_hashes.append(content_hash)
215 modules.append(ModuleManifest(
216 module_path=file_path,
217 content_hash=content_hash,
218 ast_hash=ast_hash,
219 language=lang,
220 symbol_count=sym_count,
221 ))
222
223 pkg_hash = blob_id("|".join(sorted(pkg_hashes)).encode())
224 packages.append(PackageManifest(
225 package=pkg_path,
226 package_hash=pkg_hash,
227 modules=modules,
228 total_files=len(file_paths),
229 semantic_files=pkg_semantic,
230 ))
231
232 manifest_json = json.dumps(
233 {"snapshot_id": snapshot_id, "packages": packages}, sort_keys=True
234 )
235 manifest_hash = blob_id(manifest_json.encode())
236
237 return CodeManifest(
238 snapshot_id=snapshot_id,
239 manifest_hash=manifest_hash,
240 packages=packages,
241 total_files=len(flat_manifest),
242 semantic_files=total_semantic,
243 total_symbols=total_symbols,
244 )
245
246 # ---------------------------------------------------------------------------
247 # Diff
248 # ---------------------------------------------------------------------------
249
250 class ManifestFileDiff(TypedDict):
251 """Change record from :func:`diff_manifests` for one file."""
252
253 path: str
254 change: str # "added" | "removed" | "modified" | "ast_changed"
255 old_hash: str
256 new_hash: str
257 old_ast_hash: str
258 new_ast_hash: str
259 semantic_change: bool # True when ast_hash differs (real code change)
260
261 def diff_manifests(
262 base: CodeManifest,
263 target: CodeManifest,
264 ) -> list[ManifestFileDiff]:
265 """Produce a per-file change list between two :class:`CodeManifest` objects.
266
267 Files with identical ``content_hash`` values are skipped (no change).
268 Files where only the ``content_hash`` changed but ``ast_hash`` is the same
269 are marked ``"modified"`` with ``semantic_change=False`` — e.g. whitespace
270 or comment-only diffs. Files where ``ast_hash`` changed are ``"ast_changed"``
271 with ``semantic_change=True``.
272
273 Args:
274 base: Manifest for the earlier state (e.g. parent commit).
275 target: Manifest for the later state (e.g. current commit).
276
277 Returns:
278 Sorted list of :class:`ManifestFileDiff` records.
279 """
280 base_modules: _ModuleMap = {}
281 for pkg in base["packages"]:
282 for mod in pkg["modules"]:
283 base_modules[mod["module_path"]] = mod
284
285 target_modules: _ModuleMap = {}
286 for pkg in target["packages"]:
287 for mod in pkg["modules"]:
288 target_modules[mod["module_path"]] = mod
289
290 diffs: list[ManifestFileDiff] = []
291
292 all_paths = sorted(set(base_modules) | set(target_modules))
293 for path in all_paths:
294 bm = base_modules.get(path)
295 tm = target_modules.get(path)
296
297 if bm is None and tm is not None:
298 diffs.append(ManifestFileDiff(
299 path=path, change="added",
300 old_hash="", new_hash=tm["content_hash"],
301 old_ast_hash="", new_ast_hash=tm["ast_hash"],
302 semantic_change=True,
303 ))
304 elif bm is not None and tm is None:
305 diffs.append(ManifestFileDiff(
306 path=path, change="removed",
307 old_hash=bm["content_hash"], new_hash="",
308 old_ast_hash=bm["ast_hash"], new_ast_hash="",
309 semantic_change=True,
310 ))
311 elif bm is not None and tm is not None:
312 if bm["content_hash"] == tm["content_hash"]:
313 continue # No change at all.
314 ast_changed = bm["ast_hash"] != tm["ast_hash"]
315 diffs.append(ManifestFileDiff(
316 path=path,
317 change="ast_changed" if ast_changed else "modified",
318 old_hash=bm["content_hash"], new_hash=tm["content_hash"],
319 old_ast_hash=bm["ast_hash"], new_ast_hash=tm["ast_hash"],
320 semantic_change=ast_changed,
321 ))
322
323 return diffs
324
325 # ---------------------------------------------------------------------------
326 # Persistence
327 # ---------------------------------------------------------------------------
328
329 def write_code_manifest(repo_root: pathlib.Path, manifest: CodeManifest) -> None:
330 """Persist a :class:`CodeManifest` to ``.muse/code_manifests/<hash>.json``.
331
332 Args:
333 repo_root: Repository root.
334 manifest: The manifest to write.
335 """
336 store_dir = _code_manifests_dir(repo_root)
337 store_dir.mkdir(parents=True, exist_ok=True)
338 path = store_dir / f"{long_id(manifest['manifest_hash'], strip=True)}.json"
339 if not path.exists():
340 path.write_text(json.dumps(manifest))
341
342 def read_code_manifest(
343 repo_root: pathlib.Path, manifest_hash: str
344 ) -> CodeManifest | None:
345 """Load a :class:`CodeManifest` by its hash.
346
347 Args:
348 repo_root: Repository root.
349 manifest_hash: The ``manifest_hash`` of the target manifest.
350
351 Returns:
352 The deserialized :class:`CodeManifest`, or ``None`` if not found.
353 """
354 path = _code_manifests_dir(repo_root) / f"{long_id(manifest_hash, strip=True)}.json"
355 if not path.exists():
356 return None
357 return load_json_file(path)
File History 1 commit
sha256:5eff0848db1d0748f3a195ca871a9275e7455d447690411ec2d83434e11baa65 feat(#216): persist write-once BranchMeta creation provenance Human minor ⚠ 1 day ago