gabriel / muse public
verify_pack.py python
571 lines 20.0 KB
Raw
sha256:1a5c0bab2cb0d4c3eed597349ccaa433a4a6add8726890f2d6f577c17dc54972 docs: file follow-up issue for symlog filename-too-long (muse#61) Sonnet 4.6 21 days ago
1 """muse verify-pack — verify the integrity of a MPack.
2
3 Reads a MPack msgpack binary from stdin (or ``--file``) and performs
4 three levels of integrity checking:
5
6 1. **Blob integrity** — every ``blobs`` entry has its SHA-256 recomputed
7 from the raw ``content`` bytes. The digest must match the declared
8 ``object_id``.
9
10 2. **Snapshot consistency** — every snapshot in the mpack references only
11 object IDs that are either in the mpack itself or already present in the
12 local store. Orphaned manifest entries are reported as failures.
13
14 3. **Commit consistency** — every commit in the mpack references a
15 ``snapshot_id`` that is either in the mpack or already in the local store.
16
17 Pipe from ``pack-objects`` to validate before sending to a remote::
18
19 muse pack-objects <sha> | muse verify-pack
20
21 Or verify a saved mpack file::
22
23 muse verify-pack --file mpack.muse
24
25 Quick structural inspection without full hash verification::
26
27 muse verify-pack --stat --file mpack.muse
28
29 Output (JSON, default)::
30
31 {
32 "blobs_checked": 42,
33 "snapshots_checked": 5,
34 "commits_checked": 5,
35 "all_ok": true,
36 "failures": [],
37 "promised_objects": 0,
38 "base_objects": 0,
39 "bundle_mode": "full",
40 "base_commits": [],
41 "duration_ms": 1.234,
42 "exit_code": 0
43 }
44
45 With failures::
46
47 {
48 "blobs_checked": 42,
49 "snapshots_checked": 5,
50 "commits_checked": 5,
51 "all_ok": false,
52 "failures": [
53 {"kind": "object", "id": "<sha256>", "error": "hash mismatch"},
54 {"kind": "snapshot", "id": "<sha256>", "error": "missing object: <sha256>"}
55 ],
56 "promised_objects": 3,
57 "base_objects": 0,
58 "bundle_mode": "full",
59 "base_commits": [],
60 "duration_ms": 1.234,
61 "exit_code": 1
62 }
63
64 Stat-only output (``--stat``)::
65
66 {"blobs": 42, "snapshots": 5, "commits": 5, "duration_ms": 0.456, "exit_code": 0}
67
68 Object availability model
69 -------------------------
70
71 Objects absent from the mpack are resolved against the local store using a
72 three-state model identical to ``muse verify``:
73
74 - **PRESENT** — object exists in the local ``.muse/objects/`` store (hash
75 verified). Not a failure.
76 - **PROMISED** — absent locally but at least one promisor remote is
77 configured (``.muse/config.toml``). Counted in ``promised_objects``; does
78 not affect ``all_ok``. Use ``--strict`` to treat promised objects as
79 failures.
80 - **MISSING** — absent locally with no promisor remote. Always a failure.
81
82 Output contract
83 ---------------
84
85 - Exit 0: mpack is fully intact; or ``--stat`` completed.
86 - Exit 1: one or more integrity failures; malformed msgpack input; bad format.
87 - Exit 3: I/O error reading stdin or the mpack file.
88
89 Agent use
90 ---------
91
92 Verify before pushing::
93
94 muse pack-objects "$TIP" \\
95 | muse verify-pack --json \\
96 | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d['all_ok'] else 1)"
97
98 Inspect mpack structure without hashing (fast)::
99
100 muse verify-pack --stat --file mpack.muse --json
101
102 Quiet CI gate — fails pipeline if mpack is corrupt::
103
104 muse pack-objects "$TIP" --file mpack.muse
105 muse verify-pack --quiet --file mpack.muse
106 """
107
108 import argparse
109 import json
110 import logging
111 import pathlib
112 import sys
113 from typing import TypedDict
114
115 from muse.core.types import blob_id
116 from muse.core.envelope import EnvelopeJson, make_envelope
117 from muse.core.errors import ExitCode
118 from muse.core.object_availability import ObjectState, load_promisor_remotes, object_state
119 from muse.core.object_store import read_object
120 from muse.core.repo import require_repo
121 from muse.core.io import MAX_PACK_MSGPACK_BYTES, safe_unpackb
122 from muse.core.snapshots import read_snapshot
123 from muse.core.validation import sanitize_display, validate_object_id
124 from muse.core.timing import start_timer
125
126 logger = logging.getLogger(__name__)
127
128 class _Failure(TypedDict):
129 kind: str
130 id: str
131 error: str
132
133 class _VerifyPackJson(EnvelopeJson):
134 blobs_checked: int
135 snapshots_checked: int
136 commits_checked: int
137 all_ok: bool
138 failures: list[_Failure]
139 promised_objects: int
140 base_objects: int
141 bundle_mode: str
142 base_commits: list[str]
143
144 class _StatResultJson(EnvelopeJson):
145 blobs: int
146 snapshots: int
147 commits: int
148
149 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
150 """Register the verify-pack subcommand."""
151 parser = subparsers.add_parser(
152 "verify-pack",
153 help="Verify the integrity of a MPack.",
154 description=__doc__,
155 formatter_class=argparse.RawDescriptionHelpFormatter,
156 )
157 parser.add_argument(
158 "--file", "-i",
159 default=None,
160 dest="bundle_file",
161 metavar="PATH",
162 help="Path to a MPack file. Reads from stdin when omitted.",
163 )
164 parser.add_argument(
165 "--stat",
166 action="store_true",
167 dest="stat_only",
168 help=(
169 "Fast structural inspection: count objects, snapshots, and commits "
170 "without computing any hashes. Exits 0 on valid msgpack structure."
171 ),
172 )
173 parser.add_argument(
174 "--quiet", "-q",
175 action="store_true",
176 help="No output. Exit 0 if all checks pass, exit 1 otherwise.",
177 )
178 parser.add_argument(
179 "--no-local", "-L",
180 action="store_true",
181 dest="skip_local_check",
182 help="Skip checking the local store for missing snapshot/commit refs.",
183 )
184 parser.add_argument(
185 "--json", "-j",
186 action="store_true",
187 dest="json_out",
188 help="Emit machine-readable JSON on stdout.",
189 )
190 parser.add_argument(
191 "--strict",
192 action="store_true",
193 help=(
194 "Treat promised objects (absent locally but covered by a promisor remote) "
195 "as integrity failures. By default promised objects are counted in "
196 "``promised_objects`` and do not affect ``all_ok``."
197 ),
198 )
199 parser.set_defaults(func=run, json_out=False)
200
201 def run(args: argparse.Namespace) -> None:
202 """Verify the integrity of a MPack.
203
204 Reads a MPack from stdin or ``--file`` and checks:
205
206 - Every object's payload re-hashes to its declared SHA-256 ID.
207 - Every snapshot's manifest references objects present in the mpack,
208 the local store, or a configured promisor remote.
209 - Every commit's snapshot ID is present in the mpack or the local store.
210
211 Objects absent from the mpack are resolved with the three-state model:
212 PRESENT (local store, hash-verified), PROMISED (absent locally but a
213 promisor remote is configured — counted in ``promised_objects``, not a
214 failure unless ``--strict``), or MISSING (absent with no promisor — always
215 a failure).
216
217 Use ``--stat`` for a fast structural count without hash verification.
218 Use ``--strict`` to require full self-containment (treats promised objects
219 as failures).
220
221 Agent quickstart::
222
223 muse pack-objects HEAD | muse verify-pack --json
224 muse verify-pack --file mpack.muse --json
225 muse verify-pack --stat --file mpack.muse --json
226 muse verify-pack --strict --file mpack.muse --json
227
228 JSON fields::
229
230 blobs_checked Number of blobs whose SHA-256 was recomputed.
231 snapshots_checked Number of snapshot manifests checked.
232 commits_checked Number of commit records checked.
233 all_ok true when every check passed.
234 failures List of {kind, id, error} failure records.
235 promised_objects Count of objects deferred to promisor remotes.
236 base_objects Count of objects expected at the incremental base.
237 bundle_mode "full" or "incremental".
238 base_commits List of base commit IDs for incremental bundles.
239 muse_version Muse release that produced this output.
240 schema Envelope schema version (int).
241 exit_code 0 all ok, 1 any failure.
242 duration_ms Wall-clock milliseconds for the command.
243 timestamp ISO-8601 UTC timestamp of command completion.
244 warnings List of non-fatal advisory messages.
245
246 Exit codes::
247
248 0 MPack fully intact, or --stat completed.
249 1 One or more integrity failures; malformed msgpack; bad format.
250 3 I/O error reading stdin or the mpack file.
251 """
252 elapsed = start_timer()
253 json_out: bool = args.json_out
254 bundle_file: str | None = args.bundle_file
255 quiet: bool = args.quiet
256 skip_local_check: bool = args.skip_local_check
257 stat_only: bool = args.stat_only
258 strict: bool = args.strict
259
260 # Read mpack bytes.
261 if bundle_file is not None:
262 try:
263 raw_bytes = pathlib.Path(bundle_file).read_bytes()
264 except OSError as exc:
265 print(
266 json.dumps({"error": f"Cannot read file: {sanitize_display(str(exc))}"}),
267 file=sys.stderr,
268 )
269 raise SystemExit(ExitCode.INTERNAL_ERROR)
270 else:
271 try:
272 raw_bytes = sys.stdin.buffer.read()
273 except OSError as exc:
274 print(
275 json.dumps({"error": f"Cannot read stdin: {exc}"}),
276 file=sys.stderr,
277 )
278 raise SystemExit(ExitCode.INTERNAL_ERROR)
279
280 try:
281 mpack = safe_unpackb(
282 raw_bytes,
283 context="pack input",
284 max_bytes=MAX_PACK_MSGPACK_BYTES,
285 allow_binary=True,
286 )
287 except (ValueError, TypeError, Exception) as exc:
288 print(json.dumps({"error": f"Invalid msgpack: {exc}"}), file=sys.stderr)
289 raise SystemExit(ExitCode.USER_ERROR)
290
291 if not isinstance(mpack, dict):
292 print(
293 json.dumps({"error": "MPack must be a msgpack map."}),
294 file=sys.stderr,
295 )
296 raise SystemExit(ExitCode.USER_ERROR)
297
298 # --stat: fast structural count — no hash verification.
299 if stat_only:
300 blobs_raw = mpack.get("blobs", [])
301 snapshots_raw = mpack.get("snapshots", [])
302 commits_raw = mpack.get("commits", [])
303 stat_result = {
304 "blobs": len(blobs_raw) if isinstance(blobs_raw, list) else 0,
305 "snapshots": len(snapshots_raw) if isinstance(snapshots_raw, list) else 0,
306 "commits": len(commits_raw) if isinstance(commits_raw, list) else 0,
307 }
308 if not json_out:
309 print(
310 f"blobs={stat_result['blobs']} "
311 f"snapshots={stat_result['snapshots']} "
312 f"commits={stat_result['commits']} "
313 f"duration_ms={elapsed()}"
314 )
315 else:
316 print(json.dumps(_StatResultJson(
317 **make_envelope(elapsed),
318 blobs=stat_result["blobs"],
319 snapshots=stat_result["snapshots"],
320 commits=stat_result["commits"],
321 )))
322 return
323
324 # Every mpack must carry a meta field — it is always written by build_mpack.
325 bundle_meta = mpack.get("meta")
326 if not isinstance(bundle_meta, dict):
327 print(json.dumps({"error": "MPack is missing required 'meta' field."}), file=sys.stderr)
328 raise SystemExit(ExitCode.USER_ERROR)
329 bundle_mode: str = bundle_meta.get("mode", "")
330 if bundle_mode not in ("full", "incremental"):
331 print(json.dumps({"error": f"meta.mode must be 'full' or 'incremental', got {bundle_mode!r}."}), file=sys.stderr)
332 raise SystemExit(ExitCode.USER_ERROR)
333 raw_base = bundle_meta.get("base_commits")
334 if not isinstance(raw_base, list):
335 print(json.dumps({"error": "meta.base_commits must be a list."}), file=sys.stderr)
336 raise SystemExit(ExitCode.USER_ERROR)
337 base_commits: list[str] = [str(c) for c in raw_base]
338 # Incremental bundles have unresolved refs expected at the base — track separately.
339 base_objects_count = 0
340
341 # We need the repo root for local-store checks (optional).
342 root: pathlib.Path | None = require_repo() if not skip_local_check else None
343 # Load promisor remotes once — used in snapshot manifest checks to distinguish
344 # PROMISED objects (absent locally but expected on a remote) from MISSING ones.
345 promisor_remotes: list[str] = load_promisor_remotes(root) if root is not None else []
346 promised_count = 0
347
348 failures: list[_Failure] = []
349
350 # -----------------------------------------------------------------------
351 # 1. Blob integrity — re-hash each payload.
352 # -----------------------------------------------------------------------
353 bundle_object_ids: set[str] = set()
354 blobs_raw = mpack.get("blobs", [])
355 if not isinstance(blobs_raw, list):
356 print(
357 json.dumps({"error": "'blobs' field must be a list."}),
358 file=sys.stderr,
359 )
360 raise SystemExit(ExitCode.USER_ERROR)
361
362 for entry in blobs_raw:
363 if not isinstance(entry, dict):
364 failures.append(
365 _Failure(kind="object", id="(unknown)", error="entry is not a dict")
366 )
367 continue
368 oid = entry.get("object_id", "")
369 content = entry.get("content")
370 if not isinstance(oid, str) or not isinstance(content, (bytes, bytearray)):
371 failures.append(
372 _Failure(
373 kind="object",
374 id="(unknown)",
375 error="missing or invalid object_id / content fields",
376 )
377 )
378 continue
379
380 # Validate object ID format before embedding it anywhere.
381 try:
382 validate_object_id(oid)
383 except ValueError:
384 failures.append(
385 _Failure(
386 kind="object",
387 id="(invalid)",
388 error=f"object_id is not a valid sha256-prefixed hex ID: {oid[:24]!r}",
389 )
390 )
391 continue
392
393 # Hash without copying: msgpack returns bytes; hashlib accepts bytes/bytearray.
394 actual = blob_id(content)
395 if actual != oid:
396 failures.append(
397 _Failure(
398 kind="object",
399 id=oid,
400 error=(
401 f"hash mismatch: declared {oid} "
402 f"recomputed {actual}"
403 ),
404 )
405 )
406 else:
407 bundle_object_ids.add(oid)
408
409 blobs_checked = len(blobs_raw)
410
411 # -----------------------------------------------------------------------
412 # 2. Snapshot consistency — manifest entries must be present.
413 # -----------------------------------------------------------------------
414 bundle_snapshot_ids: set[str] = set()
415 snapshots_raw = mpack.get("snapshots", [])
416 if not isinstance(snapshots_raw, list):
417 print(
418 json.dumps({"error": "'snapshots' field must be a list."}),
419 file=sys.stderr,
420 )
421 raise SystemExit(ExitCode.USER_ERROR)
422
423 for snap_entry in snapshots_raw:
424 if not isinstance(snap_entry, dict):
425 failures.append(
426 _Failure(
427 kind="snapshot", id="(unknown)", error="snapshot entry is not a dict"
428 )
429 )
430 continue
431 snap_id = snap_entry.get("snapshot_id", "")
432 if not isinstance(snap_id, str):
433 failures.append(
434 _Failure(kind="snapshot", id="(unknown)", error="missing snapshot_id")
435 )
436 continue
437
438 bundle_snapshot_ids.add(snap_id)
439 manifest = snap_entry.get("manifest", {})
440 if not isinstance(manifest, dict):
441 continue
442
443 for path, obj_id in manifest.items():
444 if not isinstance(obj_id, str):
445 continue
446 if obj_id in bundle_object_ids:
447 continue
448 # Object not in mpack.
449 if root is not None:
450 # Local store available — use the PRESENT / PROMISED / MISSING tristate.
451 try:
452 local_content = read_object(root, obj_id)
453 except OSError as exc:
454 failures.append(
455 _Failure(
456 kind="object",
457 id=obj_id,
458 error=(
459 f"local store object {obj_id} failed "
460 f"SHA-256 integrity check: {exc}"
461 ),
462 )
463 )
464 continue
465 if local_content is not None:
466 continue # PRESENT and verified
467
468 state = object_state(root, obj_id, promisor_remotes)
469 if state == ObjectState.PROMISED and not strict:
470 promised_count += 1
471 continue
472 else:
473 # No local store (--no-local). For incremental bundles, unresolved
474 # refs are expected to exist at the declared base — not a failure
475 # unless --strict is set.
476 if bundle_mode == "incremental" and not strict:
477 base_objects_count += 1
478 continue
479
480 failures.append(
481 _Failure(
482 kind="snapshot",
483 id=snap_id,
484 error=(
485 f"manifest path {path!r} references "
486 f"missing object {obj_id}"
487 ),
488 )
489 )
490
491 snapshots_checked = len(snapshots_raw)
492
493 # -----------------------------------------------------------------------
494 # 3. Commit consistency — snapshot_id must be resolvable.
495 # -----------------------------------------------------------------------
496 commits_raw = mpack.get("commits", [])
497 if not isinstance(commits_raw, list):
498 print(
499 json.dumps({"error": "'commits' field must be a list."}),
500 file=sys.stderr,
501 )
502 raise SystemExit(ExitCode.USER_ERROR)
503
504 for commit_entry in commits_raw:
505 if not isinstance(commit_entry, dict):
506 failures.append(
507 _Failure(
508 kind="commit", id="(unknown)", error="commit entry is not a dict"
509 )
510 )
511 continue
512 commit_id = commit_entry.get("commit_id", "")
513 snap_id = commit_entry.get("snapshot_id", "")
514 if not isinstance(commit_id, str) or not isinstance(snap_id, str) or not snap_id:
515 failures.append(
516 _Failure(
517 kind="commit",
518 id=commit_id if isinstance(commit_id, str) else "(unknown)",
519 error="missing commit_id or snapshot_id",
520 )
521 )
522 continue
523
524 if snap_id in bundle_snapshot_ids:
525 continue
526 if root is not None and read_snapshot(root, snap_id) is not None:
527 continue
528 if not skip_local_check:
529 failures.append(
530 _Failure(
531 kind="commit",
532 id=commit_id,
533 error=f"references snapshot {snap_id} not in mpack or local store",
534 )
535 )
536
537 commits_checked = len(commits_raw)
538 all_ok = len(failures) == 0
539
540 if quiet:
541 raise SystemExit(0 if all_ok else ExitCode.USER_ERROR)
542
543 if not json_out:
544 print(
545 f"blobs={blobs_checked} snapshots={snapshots_checked} "
546 f"commits={commits_checked} all_ok={all_ok}"
547 )
548 for f in failures:
549 print(
550 f" FAIL [{sanitize_display(f['kind'])}] "
551 f"{sanitize_display(f['id'])} "
552 f"{sanitize_display(f['error'])}"
553 )
554 if not all_ok:
555 raise SystemExit(ExitCode.USER_ERROR)
556 return
557
558 print(json.dumps(_VerifyPackJson(
559 **make_envelope(elapsed, exit_code=0 if all_ok else int(ExitCode.USER_ERROR)),
560 blobs_checked=blobs_checked,
561 snapshots_checked=snapshots_checked,
562 commits_checked=commits_checked,
563 all_ok=all_ok,
564 failures=failures,
565 promised_objects=promised_count,
566 base_objects=base_objects_count,
567 bundle_mode=bundle_mode,
568 base_commits=base_commits,
569 )))
570 if not all_ok:
571 raise SystemExit(ExitCode.USER_ERROR)
File History 1 commit
sha256:1a5c0bab2cb0d4c3eed597349ccaa433a4a6add8726890f2d6f577c17dc54972 docs: file follow-up issue for symlog filename-too-long (muse#61) Sonnet 4.6 21 days ago