aaronrene / gabriel-muse public
manifest.py python
376 lines 12.8 KB
Raw
sha256:0e3697f29ae6c62f39818a19861d4db6276465de16481fadf866eb93cbf4aae4 test: validate staging push and MP workflow for aaronrene Human 16 days ago
1 """Hierarchical code snapshot manifests for the Muse code domain.
2
3 A code manifest organises a snapshot's files into a three-level hierarchy::
4
5 PackageManifest ← top-level package (directory)
6 └─ ModuleManifest ← one source file
7 └─ FileEntry ← per-file metadata + hashes
8
9 This structure enables *partial re-parsing*: when merging, only files whose
10 ``content_hash`` changed need to be re-parsed by the AST engine. Files in
11 unchanged modules are reused from the cached manifest, making three-way
12 merges on large codebases significantly faster.
13
14 The ``ast_hash`` in :class:`FileEntry` is a SHA-256 of the file's *symbol
15 tree* rather than its raw bytes. Two files that differ only in whitespace or
16 comments will have the same ``ast_hash``, meaning the AST engine reports
17 "no semantic change" for that file.
18
19 Public API
20 ----------
21 - :class:`FileEntry` — per-file metadata.
22 - :class:`ModuleManifest` — one source file's manifest entry.
23 - :class:`PackageManifest` — directory-level grouping.
24 - :class:`CodeManifest` — complete hierarchical snapshot manifest.
25 - :func:`build_code_manifest` — build from a flat snapshot manifest.
26 - :func:`diff_manifests` — find added/removed/modified files between two.
27 - :func:`write_code_manifest` — persist to ``.muse/code_manifests/<id>.json``.
28 - :func:`read_code_manifest` — load from disk.
29 """
30
31 from __future__ import annotations
32
33 import hashlib
34 import json
35 import logging
36 import pathlib
37 from typing import TypedDict
38
39 from muse.core._types import Manifest, Metadata
40 from muse.core.object_store import read_object
41
42 logger = logging.getLogger(__name__)
43
44 type _PkgFileMap = dict[str, list[str]]
45 type _ModuleMap = dict[str, "ModuleManifest"]
46
47 # ---------------------------------------------------------------------------
48 # Data types
49 # ---------------------------------------------------------------------------
50
51
52 class FileEntry(TypedDict):
53 """Metadata for a single source file.
54
55 ``path`` Workspace-relative POSIX path.
56 ``content_hash`` SHA-256 of the raw file bytes (from the object store).
57 ``ast_hash`` SHA-256 of the symbol-tree JSON (semantic identity).
58 Empty string when AST parsing is unavailable.
59 ``language`` Display language name (``"Python"``, ``"TypeScript"``…).
60 ``symbol_count`` Number of top-level + nested symbols extracted.
61 ``size_bytes`` Raw file size in bytes (0 if unavailable).
62 """
63
64 path: str
65 content_hash: str
66 ast_hash: str
67 language: str
68 symbol_count: int
69 size_bytes: int
70
71
72 class ModuleManifest(TypedDict):
73 """Manifest for one source file (module).
74
75 ``module_path`` Workspace-relative POSIX path.
76 ``content_hash`` Raw-bytes SHA-256 (same as ``FileEntry.content_hash``).
77 ``ast_hash`` Symbol-tree SHA-256 for semantic change detection.
78 ``language`` Language name.
79 ``symbol_count`` Number of symbols in this file.
80 """
81
82 module_path: str
83 content_hash: str
84 ast_hash: str
85 language: str
86 symbol_count: int
87
88
89 class PackageManifest(TypedDict):
90 """Directory-level grouping of modules.
91
92 ``package`` Workspace-relative POSIX directory path.
93 ``package_hash`` SHA-256 of all sorted ``content_hash`` values in the
94 package — stable fingerprint for change detection.
95 ``modules`` All ``ModuleManifest`` entries in this package.
96 ``total_files`` Total number of files (all types, not just semantic).
97 ``semantic_files`` Number of AST-parseable files.
98 """
99
100 package: str
101 package_hash: str
102 modules: list[ModuleManifest]
103 total_files: int
104 semantic_files: int
105
106
107 class CodeManifest(TypedDict):
108 """Complete hierarchical manifest for one code snapshot.
109
110 ``snapshot_id`` The snapshot this manifest was built from.
111 ``manifest_hash`` SHA-256 of this manifest's JSON — stable cache key.
112 ``packages`` All :class:`PackageManifest` entries, sorted by path.
113 ``total_files`` Total files in the snapshot.
114 ``semantic_files`` AST-parseable files.
115 ``total_symbols`` Sum of ``symbol_count`` across all modules.
116 """
117
118 snapshot_id: str
119 manifest_hash: str
120 packages: list[PackageManifest]
121 total_files: int
122 semantic_files: int
123 total_symbols: int
124
125
126 # ---------------------------------------------------------------------------
127 # Language detection
128 # ---------------------------------------------------------------------------
129
130 _SUFFIX_LANG: Metadata = {
131 ".py": "Python", ".pyi": "Python",
132 ".ts": "TypeScript", ".tsx": "TypeScript",
133 ".js": "JavaScript", ".jsx": "JavaScript",
134 ".mjs": "JavaScript", ".cjs": "JavaScript",
135 ".go": "Go",
136 ".rs": "Rust",
137 ".java": "Java",
138 ".cs": "C#",
139 ".c": "C", ".h": "C",
140 ".cpp": "C++", ".cc": "C++", ".cxx": "C++", ".hpp": "C++",
141 ".rb": "Ruby",
142 ".kt": "Kotlin", ".kts": "Kotlin",
143 }
144
145 _SEMANTIC_SUFFIXES = frozenset(_SUFFIX_LANG)
146
147
148 def _language_of(file_path: str) -> str:
149 suffix = pathlib.PurePosixPath(file_path).suffix.lower()
150 return _SUFFIX_LANG.get(suffix, suffix or "(no ext)")
151
152
153 def _is_semantic(file_path: str) -> bool:
154 suffix = pathlib.PurePosixPath(file_path).suffix.lower()
155 return suffix in _SEMANTIC_SUFFIXES
156
157
158 # ---------------------------------------------------------------------------
159 # Builder
160 # ---------------------------------------------------------------------------
161
162
163 def build_code_manifest(
164 snapshot_id: str,
165 flat_manifest: Manifest,
166 repo_root: pathlib.Path,
167 ) -> CodeManifest:
168 """Build a :class:`CodeManifest` from a flat ``{path: content_hash}`` dict.
169
170 Attempts to parse each semantic file into a symbol tree to compute
171 ``ast_hash`` and ``symbol_count``. Falls back gracefully for binary files
172 or parse errors.
173
174 Args:
175 snapshot_id: The snapshot this manifest represents.
176 flat_manifest: ``{workspace_path: sha256}`` from the snapshot.
177 repo_root: Repository root for object store access.
178
179 Returns:
180 A fully populated :class:`CodeManifest`.
181 """
182 # Late import to avoid circular dependencies.
183 from muse.plugins.code.ast_parser import parse_symbols
184
185 # Group files by parent directory.
186 pkg_files: _PkgFileMap = {}
187 for file_path in sorted(flat_manifest):
188 pkg = str(pathlib.PurePosixPath(file_path).parent)
189 pkg_files.setdefault(pkg, []).append(file_path)
190
191 packages: list[PackageManifest] = []
192 total_symbols = 0
193 total_semantic = 0
194
195 for pkg_path, file_paths in sorted(pkg_files.items()):
196 modules: list[ModuleManifest] = []
197 pkg_hashes: list[str] = []
198 pkg_semantic = 0
199
200 for file_path in sorted(file_paths):
201 content_hash = flat_manifest[file_path]
202 lang = _language_of(file_path)
203 is_sem = _is_semantic(file_path)
204 ast_hash = ""
205 sym_count = 0
206
207 if is_sem:
208 source = read_object(repo_root, content_hash)
209 if source is not None:
210 try:
211 symbols = parse_symbols(source, file_path)
212 sym_count = len(symbols)
213 # ast_hash = SHA-256 of sorted symbol content IDs.
214 sig = hashlib.sha256(
215 "|".join(
216 sorted(s["content_id"] for s in symbols.values())
217 ).encode()
218 ).hexdigest()
219 ast_hash = sig
220 except Exception:
221 logger.debug("AST parse failed for %s", file_path)
222 pkg_semantic += 1
223 total_semantic += 1
224
225 total_symbols += sym_count
226 pkg_hashes.append(content_hash)
227 modules.append(ModuleManifest(
228 module_path=file_path,
229 content_hash=content_hash,
230 ast_hash=ast_hash,
231 language=lang,
232 symbol_count=sym_count,
233 ))
234
235 pkg_hash = hashlib.sha256("|".join(sorted(pkg_hashes)).encode()).hexdigest()
236 packages.append(PackageManifest(
237 package=pkg_path,
238 package_hash=pkg_hash,
239 modules=modules,
240 total_files=len(file_paths),
241 semantic_files=pkg_semantic,
242 ))
243
244 manifest_json = json.dumps(
245 {"snapshot_id": snapshot_id, "packages": packages}, sort_keys=True
246 )
247 manifest_hash = hashlib.sha256(manifest_json.encode()).hexdigest()
248
249 return CodeManifest(
250 snapshot_id=snapshot_id,
251 manifest_hash=manifest_hash,
252 packages=packages,
253 total_files=len(flat_manifest),
254 semantic_files=total_semantic,
255 total_symbols=total_symbols,
256 )
257
258
259 # ---------------------------------------------------------------------------
260 # Diff
261 # ---------------------------------------------------------------------------
262
263
264 class ManifestFileDiff(TypedDict):
265 """Change record from :func:`diff_manifests` for one file."""
266
267 path: str
268 change: str # "added" | "removed" | "modified" | "ast_changed"
269 old_hash: str
270 new_hash: str
271 old_ast_hash: str
272 new_ast_hash: str
273 semantic_change: bool # True when ast_hash differs (real code change)
274
275
276 def diff_manifests(
277 base: CodeManifest,
278 target: CodeManifest,
279 ) -> list[ManifestFileDiff]:
280 """Produce a per-file change list between two :class:`CodeManifest` objects.
281
282 Files with identical ``content_hash`` values are skipped (no change).
283 Files where only the ``content_hash`` changed but ``ast_hash`` is the same
284 are marked ``"modified"`` with ``semantic_change=False`` — e.g. whitespace
285 or comment-only diffs. Files where ``ast_hash`` changed are ``"ast_changed"``
286 with ``semantic_change=True``.
287
288 Args:
289 base: Manifest for the earlier state (e.g. parent commit).
290 target: Manifest for the later state (e.g. current commit).
291
292 Returns:
293 Sorted list of :class:`ManifestFileDiff` records.
294 """
295 base_modules: _ModuleMap = {}
296 for pkg in base["packages"]:
297 for mod in pkg["modules"]:
298 base_modules[mod["module_path"]] = mod
299
300 target_modules: _ModuleMap = {}
301 for pkg in target["packages"]:
302 for mod in pkg["modules"]:
303 target_modules[mod["module_path"]] = mod
304
305 diffs: list[ManifestFileDiff] = []
306
307 all_paths = sorted(set(base_modules) | set(target_modules))
308 for path in all_paths:
309 bm = base_modules.get(path)
310 tm = target_modules.get(path)
311
312 if bm is None and tm is not None:
313 diffs.append(ManifestFileDiff(
314 path=path, change="added",
315 old_hash="", new_hash=tm["content_hash"],
316 old_ast_hash="", new_ast_hash=tm["ast_hash"],
317 semantic_change=True,
318 ))
319 elif bm is not None and tm is None:
320 diffs.append(ManifestFileDiff(
321 path=path, change="removed",
322 old_hash=bm["content_hash"], new_hash="",
323 old_ast_hash=bm["ast_hash"], new_ast_hash="",
324 semantic_change=True,
325 ))
326 elif bm is not None and tm is not None:
327 if bm["content_hash"] == tm["content_hash"]:
328 continue # No change at all.
329 ast_changed = bm["ast_hash"] != tm["ast_hash"]
330 diffs.append(ManifestFileDiff(
331 path=path,
332 change="ast_changed" if ast_changed else "modified",
333 old_hash=bm["content_hash"], new_hash=tm["content_hash"],
334 old_ast_hash=bm["ast_hash"], new_ast_hash=tm["ast_hash"],
335 semantic_change=ast_changed,
336 ))
337
338 return diffs
339
340
341 # ---------------------------------------------------------------------------
342 # Persistence
343 # ---------------------------------------------------------------------------
344
345
346 def write_code_manifest(repo_root: pathlib.Path, manifest: CodeManifest) -> None:
347 """Persist a :class:`CodeManifest` to ``.muse/code_manifests/<hash>.json``.
348
349 Args:
350 repo_root: Repository root.
351 manifest: The manifest to write.
352 """
353 store_dir = repo_root / ".muse" / "code_manifests"
354 store_dir.mkdir(parents=True, exist_ok=True)
355 path = store_dir / f"{manifest['manifest_hash']}.json"
356 if not path.exists():
357 path.write_text(json.dumps(manifest))
358
359
360 def read_code_manifest(
361 repo_root: pathlib.Path, manifest_hash: str
362 ) -> CodeManifest | None:
363 """Load a :class:`CodeManifest` by its hash.
364
365 Args:
366 repo_root: Repository root.
367 manifest_hash: The ``manifest_hash`` of the target manifest.
368
369 Returns:
370 The deserialized :class:`CodeManifest`, or ``None`` if not found.
371 """
372 path = repo_root / ".muse" / "code_manifests" / f"{manifest_hash}.json"
373 if not path.exists():
374 return None
375 raw: CodeManifest = json.loads(path.read_text())
376 return raw
File History 1 commit
sha256:0e3697f29ae6c62f39818a19861d4db6276465de16481fadf866eb93cbf4aae4 test: validate staging push and MP workflow for aaronrene Human 16 days ago