aaronrene / gabriel-muse public
verify_pack.py python
469 lines 15.2 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
1 """muse plumbing verify-pack — verify the integrity of a PackBundle.
2
3 Reads a PackBundle msgpack binary from stdin (or ``--file``) and performs
4 three levels of integrity checking:
5
6 1. **Object integrity** — every ``objects`` entry has its SHA-256 recomputed
7 from the raw ``content`` bytes. The digest must match the declared
8 ``object_id``.
9
10 2. **Snapshot consistency** — every snapshot in the bundle references only
11 object IDs that are either in the bundle itself or already present in the
12 local store. Orphaned manifest entries are reported as failures.
13
14 3. **Commit consistency** — every commit in the bundle references a
15 ``snapshot_id`` that is either in the bundle or already in the local store.
16
17 Pipe from ``pack-objects`` to validate before sending to a remote::
18
19 muse plumbing pack-objects <sha> | muse plumbing verify-pack
20
21 Or verify a saved bundle file::
22
23 muse plumbing verify-pack --file bundle.muse
24
25 Quick structural inspection without full hash verification::
26
27 muse plumbing verify-pack --stat --file bundle.muse
28
29 Output (JSON, default)::
30
31 {
32 "objects_checked": 42,
33 "snapshots_checked": 5,
34 "commits_checked": 5,
35 "all_ok": true,
36 "failures": []
37 }
38
39 With failures::
40
41 {
42 "objects_checked": 42,
43 "snapshots_checked": 5,
44 "commits_checked": 5,
45 "all_ok": false,
46 "failures": [
47 {"kind": "object", "id": "<sha256>", "error": "hash mismatch"},
48 {"kind": "snapshot", "id": "<sha256>", "error": "missing object: <sha256>"}
49 ]
50 }
51
52 Stat-only output (``--stat``)::
53
54 {"objects": 42, "snapshots": 5, "commits": 5}
55
56 Plumbing contract
57 -----------------
58
59 - Exit 0: bundle is fully intact; or ``--stat`` completed.
60 - Exit 1: one or more integrity failures; malformed msgpack input; bad format.
61 - Exit 3: I/O error reading stdin or the bundle file.
62
63 Agent use
64 ---------
65
66 Verify before pushing::
67
68 muse plumbing pack-objects "$TIP" \\
69 | muse plumbing verify-pack --json \\
70 | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d['all_ok'] else 1)"
71
72 Inspect bundle structure without hashing (fast)::
73
74 muse plumbing verify-pack --stat --file bundle.muse --json
75
76 Quiet CI gate — fails pipeline if bundle is corrupt::
77
78 muse plumbing pack-objects "$TIP" --file bundle.muse
79 muse plumbing verify-pack --quiet --file bundle.muse
80 """
81
82 from __future__ import annotations
83
84 import argparse
85 import hashlib
86 import json
87 import logging
88 import pathlib
89 import sys
90 from typing import TypedDict
91
92 from muse.core.errors import ExitCode
93 from muse.core.object_store import read_object
94 from muse.core.repo import require_repo
95 from muse.core.store import MAX_PACK_MSGPACK_BYTES, read_snapshot, safe_unpackb
96 from muse.core.validation import sanitize_display, validate_object_id
97
98 logger = logging.getLogger(__name__)
99
100 _FORMAT_CHOICES = ("json", "text")
101
102
103 class _Failure(TypedDict):
104 kind: str
105 id: str
106 error: str
107
108
109 class _VerifyPackResult(TypedDict):
110 objects_checked: int
111 snapshots_checked: int
112 commits_checked: int
113 all_ok: bool
114 failures: list[_Failure]
115
116
117 class _StatResult(TypedDict):
118 objects: int
119 snapshots: int
120 commits: int
121
122
123 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
124 """Register the verify-pack subcommand."""
125 parser = subparsers.add_parser(
126 "verify-pack",
127 help="Verify the integrity of a PackBundle.",
128 description=__doc__,
129 formatter_class=argparse.RawDescriptionHelpFormatter,
130 )
131 parser.add_argument(
132 "--file", "-i",
133 default=None,
134 dest="bundle_file",
135 metavar="PATH",
136 help="Path to a PackBundle file. Reads from stdin when omitted.",
137 )
138 parser.add_argument(
139 "--stat",
140 action="store_true",
141 dest="stat_only",
142 help=(
143 "Fast structural inspection: count objects, snapshots, and commits "
144 "without computing any hashes. Exits 0 on valid msgpack structure."
145 ),
146 )
147 parser.add_argument(
148 "--quiet", "-q",
149 action="store_true",
150 help="No output. Exit 0 if all checks pass, exit 1 otherwise.",
151 )
152 parser.add_argument(
153 "--no-local", "-L",
154 action="store_true",
155 dest="skip_local_check",
156 help="Skip checking the local store for missing snapshot/commit refs.",
157 )
158 parser.add_argument(
159 "--format", "-f",
160 dest="fmt",
161 default="json",
162 metavar="FORMAT",
163 help="Output format: json or text. (default: json)",
164 )
165 parser.add_argument(
166 "--json", action="store_const", const="json", dest="fmt",
167 help="Shorthand for --format json.",
168 )
169 parser.set_defaults(func=run)
170
171
172 def run(args: argparse.Namespace) -> None:
173 """Verify the integrity of a PackBundle.
174
175 Reads a PackBundle from stdin or ``--file`` and checks:
176
177 - Every object's payload re-hashes to its declared SHA-256 ID.
178 - Every snapshot's manifest references objects present in the bundle or
179 the local store.
180 - Every commit's snapshot ID is present in the bundle or the local store.
181
182 Use ``--stat`` for a fast structural count without hash verification.
183 """
184 fmt: str = args.fmt
185 bundle_file: str | None = args.bundle_file
186 quiet: bool = args.quiet
187 skip_local_check: bool = args.skip_local_check
188 stat_only: bool = args.stat_only
189
190 if fmt not in _FORMAT_CHOICES:
191 print(
192 json.dumps(
193 {"error": f"Unknown format {fmt!r}. Valid: {', '.join(_FORMAT_CHOICES)}"}
194 ),
195 file=sys.stderr,
196 )
197 raise SystemExit(ExitCode.USER_ERROR)
198
199 # Read bundle bytes.
200 if bundle_file is not None:
201 try:
202 raw_bytes = pathlib.Path(bundle_file).read_bytes()
203 except OSError as exc:
204 print(
205 json.dumps({"error": f"Cannot read file: {sanitize_display(str(exc))}"}),
206 file=sys.stderr,
207 )
208 raise SystemExit(ExitCode.INTERNAL_ERROR)
209 else:
210 try:
211 raw_bytes = sys.stdin.buffer.read()
212 except OSError as exc:
213 print(
214 json.dumps({"error": f"Cannot read stdin: {exc}"}),
215 file=sys.stderr,
216 )
217 raise SystemExit(ExitCode.INTERNAL_ERROR)
218
219 try:
220 bundle = safe_unpackb(
221 raw_bytes,
222 context="pack input",
223 max_bytes=MAX_PACK_MSGPACK_BYTES,
224 allow_binary=True,
225 )
226 except (ValueError, TypeError, Exception) as exc:
227 print(json.dumps({"error": f"Invalid msgpack: {exc}"}), file=sys.stderr)
228 raise SystemExit(ExitCode.USER_ERROR)
229
230 if not isinstance(bundle, dict):
231 print(
232 json.dumps({"error": "PackBundle must be a msgpack map."}),
233 file=sys.stderr,
234 )
235 raise SystemExit(ExitCode.USER_ERROR)
236
237 # --stat: fast structural count — no hash verification.
238 if stat_only:
239 objects_raw = bundle.get("objects", [])
240 snapshots_raw = bundle.get("snapshots", [])
241 commits_raw = bundle.get("commits", [])
242 stat_result: _StatResult = {
243 "objects": len(objects_raw) if isinstance(objects_raw, list) else 0,
244 "snapshots": len(snapshots_raw) if isinstance(snapshots_raw, list) else 0,
245 "commits": len(commits_raw) if isinstance(commits_raw, list) else 0,
246 }
247 if fmt == "text":
248 print(
249 f"objects={stat_result['objects']} "
250 f"snapshots={stat_result['snapshots']} "
251 f"commits={stat_result['commits']}"
252 )
253 else:
254 print(json.dumps(stat_result))
255 return
256
257 # We need the repo root for local-store checks (optional).
258 root: pathlib.Path | None = require_repo() if not skip_local_check else None
259
260 failures: list[_Failure] = []
261
262 # -----------------------------------------------------------------------
263 # 1. Object integrity — re-hash each payload.
264 # -----------------------------------------------------------------------
265 bundle_object_ids: set[str] = set()
266 objects_raw = bundle.get("objects", [])
267 if not isinstance(objects_raw, list):
268 print(
269 json.dumps({"error": "'objects' field must be a list."}),
270 file=sys.stderr,
271 )
272 raise SystemExit(ExitCode.USER_ERROR)
273
274 for entry in objects_raw:
275 if not isinstance(entry, dict):
276 failures.append(
277 _Failure(kind="object", id="(unknown)", error="entry is not a dict")
278 )
279 continue
280 oid = entry.get("object_id", "")
281 content = entry.get("content")
282 if not isinstance(oid, str) or not isinstance(content, (bytes, bytearray)):
283 failures.append(
284 _Failure(
285 kind="object",
286 id="(unknown)",
287 error="missing or invalid object_id / content fields",
288 )
289 )
290 continue
291
292 # Validate object ID format before embedding it anywhere.
293 try:
294 validate_object_id(oid)
295 except ValueError:
296 failures.append(
297 _Failure(
298 kind="object",
299 id="(invalid)",
300 error=f"object_id is not a valid 64-char hex SHA-256: {oid[:24]!r}",
301 )
302 )
303 continue
304
305 # Hash without copying: msgpack returns bytes; hashlib accepts bytes/bytearray.
306 actual = hashlib.sha256(content).hexdigest()
307 if actual != oid:
308 failures.append(
309 _Failure(
310 kind="object",
311 id=oid,
312 error=(
313 f"hash mismatch: declared {oid[:12]}... "
314 f"recomputed {actual[:12]}..."
315 ),
316 )
317 )
318 else:
319 bundle_object_ids.add(oid)
320
321 objects_checked = len(objects_raw)
322
323 # -----------------------------------------------------------------------
324 # 2. Snapshot consistency — manifest entries must be present.
325 # -----------------------------------------------------------------------
326 bundle_snapshot_ids: set[str] = set()
327 snapshots_raw = bundle.get("snapshots", [])
328 if not isinstance(snapshots_raw, list):
329 print(
330 json.dumps({"error": "'snapshots' field must be a list."}),
331 file=sys.stderr,
332 )
333 raise SystemExit(ExitCode.USER_ERROR)
334
335 for snap_entry in snapshots_raw:
336 if not isinstance(snap_entry, dict):
337 failures.append(
338 _Failure(
339 kind="snapshot", id="(unknown)", error="snapshot entry is not a dict"
340 )
341 )
342 continue
343 snap_id = snap_entry.get("snapshot_id", "")
344 if not isinstance(snap_id, str):
345 failures.append(
346 _Failure(kind="snapshot", id="(unknown)", error="missing snapshot_id")
347 )
348 continue
349
350 bundle_snapshot_ids.add(snap_id)
351 manifest = snap_entry.get("manifest", {})
352 if not isinstance(manifest, dict):
353 continue
354
355 for path, obj_id in manifest.items():
356 if not isinstance(obj_id, str):
357 continue
358 if obj_id in bundle_object_ids:
359 continue
360 # Check local store if allowed. Use read_object (not has_object)
361 # so that a bit-flipped on-disk object is caught here rather than
362 # silently declared "present". read_object re-hashes every byte
363 # and raises OSError on any mismatch.
364 if root is not None:
365 try:
366 local_content = read_object(root, obj_id)
367 except OSError as exc:
368 # Object exists on disk but fails the SHA-256 integrity check.
369 failures.append(
370 _Failure(
371 kind="object",
372 id=obj_id,
373 error=(
374 f"local store object {obj_id[:12]}... failed "
375 f"SHA-256 integrity check: {exc}"
376 ),
377 )
378 )
379 continue
380 if local_content is not None:
381 continue # present and verified
382 failures.append(
383 _Failure(
384 kind="snapshot",
385 id=snap_id,
386 error=(
387 f"manifest path {path!r} references "
388 f"missing object {obj_id[:12]}..."
389 ),
390 )
391 )
392
393 snapshots_checked = len(snapshots_raw)
394
395 # -----------------------------------------------------------------------
396 # 3. Commit consistency — snapshot_id must be resolvable.
397 # -----------------------------------------------------------------------
398 commits_raw = bundle.get("commits", [])
399 if not isinstance(commits_raw, list):
400 print(
401 json.dumps({"error": "'commits' field must be a list."}),
402 file=sys.stderr,
403 )
404 raise SystemExit(ExitCode.USER_ERROR)
405
406 for commit_entry in commits_raw:
407 if not isinstance(commit_entry, dict):
408 failures.append(
409 _Failure(
410 kind="commit", id="(unknown)", error="commit entry is not a dict"
411 )
412 )
413 continue
414 commit_id = commit_entry.get("commit_id", "")
415 snap_id = commit_entry.get("snapshot_id", "")
416 if not isinstance(commit_id, str) or not isinstance(snap_id, str):
417 failures.append(
418 _Failure(
419 kind="commit",
420 id="(unknown)",
421 error="missing commit_id or snapshot_id",
422 )
423 )
424 continue
425
426 if snap_id in bundle_snapshot_ids:
427 continue
428 if root is not None and read_snapshot(root, snap_id) is not None:
429 continue
430 if not skip_local_check:
431 failures.append(
432 _Failure(
433 kind="commit",
434 id=commit_id,
435 error=f"references snapshot {snap_id[:12]}... not in bundle or local store",
436 )
437 )
438
439 commits_checked = len(commits_raw)
440 all_ok = len(failures) == 0
441
442 if quiet:
443 raise SystemExit(0 if all_ok else ExitCode.USER_ERROR)
444
445 if fmt == "text":
446 print(
447 f"objects={objects_checked} snapshots={snapshots_checked} "
448 f"commits={commits_checked} all_ok={all_ok}"
449 )
450 for f in failures:
451 print(
452 f" FAIL [{sanitize_display(f['kind'])}] "
453 f"{sanitize_display(f['id'][:16])}... "
454 f"{sanitize_display(f['error'])}"
455 )
456 if not all_ok:
457 raise SystemExit(ExitCode.USER_ERROR)
458 return
459
460 result: _VerifyPackResult = {
461 "objects_checked": objects_checked,
462 "snapshots_checked": snapshots_checked,
463 "commits_checked": commits_checked,
464 "all_ok": all_ok,
465 "failures": failures,
466 }
467 print(json.dumps(result))
468 if not all_ok:
469 raise SystemExit(ExitCode.USER_ERROR)
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 33 days ago