verify.py python
1,067 lines 41.0 KB
Raw
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago
1 """Whole-repository integrity verification for ``muse verify``.
2
3 Walks every reachable commit from every branch ref (or a single branch when
4 ``branch`` is specified) and performs five tiers of integrity checking:
5
6 1. **Ref integrity** — every branch ref file points to an existing commit.
7 2. **Commit → snapshot integrity** — every commit's ``snapshot_id`` resolves.
8 3. **Snapshot → object integrity** — every object path in every manifest exists.
9 4. **Object hash integrity** — re-hashes each object file and compares against
10 its declared ID (the SHA-256 content address). Optional; controlled by the
11 ``check_objects`` flag. Skipped when ``check_objects=False`` for speed.
12 5. **Signature integrity** — for every commit that carries a non-empty
13 ``signature`` and ``agent_id``, the HMAC-SHA256 signature is verified
14 against the agent key stored under ``.muse/keys/<agent_id>.key``.
15
16 The walk is BFS across all reachable commits from all branches; each commit is
17 checked exactly once even when reachable from multiple branches.
18
19 Security model::
20
21 Branch ref files are validated before being read:
22 - Symlinks inside ``.muse/refs/heads/`` are silently skipped — a symlink
23 could point to a file outside the repository boundary.
24 - Ref file content is capped at 65 bytes (SHA-256 hex + newline); anything
25 larger is treated as an invalid ref rather than read into memory.
26
27 Missing agent keys are reported as ``kind="key_missing"`` warnings in the
28 failure list — distinct from ``kind="signature"`` (genuine tamper detection).
29 Agents can filter by ``kind`` to distinguish key-rotation events from
30 corruption.
31
32 Walk budget::
33
34 ``_MAX_COMMITS = 500_000``. When the budget is reached (using ``>=`` so
35 exactly 500 000 commits are visited) a warning is logged and the walk stops.
36 The result reflects only the commits visited up to that point.
37 """
38
39 from __future__ import annotations
40
41 import hashlib
42 import json
43 import logging
44 import pathlib
45 import urllib.error
46 import urllib.request
47 from collections import deque
48 from typing import Any, Literal, TypedDict
49
50 from muse.core.attestation import (
51 AttestationVerifyError,
52 IdentityChainResult,
53 IdentityRecord,
54 MuseIdentityProvider,
55 get_attestation_provider,
56 get_identity_provider,
57 )
58 from muse.core.object_store import object_path
59 from muse.core.provenance import provenance_payload, verify_commit_ed25519
60 from muse.core.store import read_commit, read_snapshot
61
62 logger = logging.getLogger(__name__)
63
64 _CHUNK_SIZE = 65_536
65 _MAX_COMMITS = 500_000
66 # Maximum bytes we'll read from a ref file. A valid SHA-256 hex string is 64
67 # chars + optional newline = 65 bytes. Anything larger is treated as corrupt.
68 _MAX_REF_BYTES = 65
69
70
71 class VerifyFailure(TypedDict):
72 """A single integrity failure detected during ``muse verify``.
73
74 ``kind`` is one of:
75
76 - ``"ref"`` — branch ref file is malformed or missing
77 - ``"commit"`` — commit file is missing or unreadable
78 - ``"snapshot"`` — snapshot file is missing or unreadable
79 - ``"object"`` — object file is missing or its hash does not match
80 - ``"signature"`` — Ed25519 signature verification failed (tamper detected)
81 - ``"key_missing"`` — signer_public_key absent; signature could not be verified
82 """
83
84 kind: Literal["ref", "commit", "snapshot", "object", "signature", "key_missing"]
85 id: str
86 error: str
87
88
89 class VerifyResult(TypedDict):
90 """Aggregate result of a full repository integrity walk."""
91
92 refs_checked: int
93 commits_checked: int
94 snapshots_checked: int
95 objects_checked: int
96 signatures_checked: int
97 all_ok: bool
98 failures: list[VerifyFailure]
99
100
101 def _branch_refs(
102 root: pathlib.Path,
103 *,
104 branch: str | None = None,
105 ) -> list[tuple[str, str]]:
106 """Return ``[(branch_name, commit_id)]`` for branch ref files.
107
108 When *branch* is given only that branch's ref is returned. Otherwise all
109 refs under ``.muse/refs/heads/`` are returned.
110
111 Security:
112 Symlinks are silently skipped — a symlink could point outside the
113 repository boundary. Ref file content is capped at ``_MAX_REF_BYTES``
114 (65 bytes); oversized files are treated as invalid refs.
115 """
116 heads_dir = root / ".muse" / "refs" / "heads"
117 if not heads_dir.exists():
118 return []
119
120 if branch is not None:
121 ref_file = heads_dir / branch
122 if not ref_file.exists() or ref_file.is_symlink():
123 return []
124 raw_bytes = ref_file.read_bytes()[:_MAX_REF_BYTES]
125 raw = raw_bytes.decode("utf-8", errors="replace").strip()
126 return [(branch, raw)] if raw else []
127
128 refs: list[tuple[str, str]] = []
129 for ref_file in sorted(heads_dir.rglob("*")):
130 if ref_file.is_symlink():
131 logger.warning("⚠️ verify: skipping symlink ref: %s", ref_file)
132 continue
133 if not ref_file.is_file():
134 continue
135 raw_bytes = ref_file.read_bytes()[:_MAX_REF_BYTES]
136 raw = raw_bytes.decode("utf-8", errors="replace").strip()
137 if raw:
138 br = str(ref_file.relative_to(heads_dir).as_posix())
139 refs.append((br, raw))
140 return refs
141
142
143 def _rehash_object(path: pathlib.Path) -> str:
144 """Re-compute SHA-256 of the object file at *path* using streaming reads."""
145 h = hashlib.sha256()
146 with path.open("rb") as fh:
147 for chunk in iter(lambda: fh.read(_CHUNK_SIZE), b""):
148 h.update(chunk)
149 return h.hexdigest()
150
151
152 def run_verify(
153 root: pathlib.Path,
154 *,
155 check_objects: bool = True,
156 branch: str | None = None,
157 fail_fast: bool = False,
158 ) -> VerifyResult:
159 """Perform a full repository integrity walk.
160
161 Args:
162 root: Repository root (the directory containing ``.muse/``).
163 check_objects: When ``True`` (the default), every object file is
164 re-hashed and its digest compared against its declared
165 content address. Pass ``False`` for a faster
166 existence-only check.
167 branch: When given, only the named branch is walked. All
168 other branches are ignored. Useful for targeted
169 verification of a single branch without paying the
170 full BFS cost.
171 fail_fast: When ``True``, the walk stops after the first failure
172 is recorded. Useful in CI pipelines where the
173 presence of any failure is sufficient to fail the job.
174
175 Returns:
176 A :class:`VerifyResult` summarising what was checked and any failures.
177 Failures with ``kind="key_missing"`` indicate that an agent key could
178 not be found (possible key rotation); failures with
179 ``kind="signature"`` indicate that a signature did not verify
180 (possible tampering).
181 """
182 failures: list[VerifyFailure] = []
183 refs_checked = 0
184 commits_checked = 0
185 snapshots_checked = 0
186 objects_checked = 0
187 signatures_checked = 0
188
189 # Track which objects and snapshots we've already verified to avoid
190 # re-checking the same content multiple times (shared objects / snapshots).
191 verified_snapshots: set[str] = set()
192 verified_objects: set[str] = set()
193
194 # Phase 1: enumerate branch refs and collect tip commit IDs.
195 branch_refs = _branch_refs(root, branch=branch)
196 tip_commit_ids: list[str] = []
197
198 for br, commit_id in branch_refs:
199 refs_checked += 1
200 # Validate the ref format: exactly 64 lowercase hex characters.
201 if len(commit_id) != 64 or not all(c in "0123456789abcdef" for c in commit_id):
202 failures.append(VerifyFailure(
203 kind="ref",
204 id=br,
205 error=f"invalid commit ID in ref: {commit_id!r}",
206 ))
207 if fail_fast:
208 return _make_result(
209 refs_checked, commits_checked, snapshots_checked,
210 objects_checked, signatures_checked, failures,
211 )
212 continue
213 tip_commit_ids.append(commit_id)
214
215 # Phase 2: BFS walk of the commit DAG.
216 visited: set[str] = set()
217 queue: deque[str] = deque(tip_commit_ids)
218
219 while queue:
220 cid = queue.popleft()
221 if cid in visited:
222 continue
223 visited.add(cid)
224
225 if len(visited) >= _MAX_COMMITS:
226 logger.warning(
227 "⚠️ verify: reached %d-commit limit — stopping early", _MAX_COMMITS
228 )
229 break
230
231 commit = read_commit(root, cid)
232 if commit is None:
233 failures.append(VerifyFailure(
234 kind="commit",
235 id=cid,
236 error="commit file missing or unreadable",
237 ))
238 if fail_fast:
239 break
240 continue
241
242 commits_checked += 1
243
244 # Phase 5: verify Ed25519 signature when the commit carries one.
245 # format_version == 7: Ed25519 signature with embedded public key.
246 # format_version <= 6: legacy HMAC (no longer verifiable) — skip silently.
247 # A commit is considered signed when signature and signer_public_key are both set.
248 if commit.signature and commit.format_version >= 7:
249 if not commit.signer_public_key:
250 logger.warning(
251 "⚠️ verify: signed commit %s missing signer_public_key — cannot verify",
252 cid[:12],
253 )
254 failures.append(VerifyFailure(
255 kind="key_missing",
256 id=cid,
257 error=(
258 f"signed commit {cid[:12]} (format_version=7) is missing "
259 "signer_public_key — cannot verify Ed25519 signature"
260 ),
261 ))
262 if fail_fast:
263 break
264 else:
265 import base64
266 signatures_checked += 1
267 signed_input = provenance_payload(
268 cid,
269 author=commit.author,
270 agent_id=commit.agent_id,
271 model_id=commit.model_id,
272 toolchain_id=commit.toolchain_id,
273 prompt_hash=commit.prompt_hash,
274 committed_at=commit.committed_at.isoformat(),
275 )
276 padded = commit.signer_public_key + "=" * (-len(commit.signer_public_key) % 4)
277 try:
278 pub_bytes = base64.urlsafe_b64decode(padded)
279 except Exception:
280 pub_bytes = b""
281 if not pub_bytes or not verify_commit_ed25519(signed_input, commit.signature, pub_bytes):
282 failures.append(VerifyFailure(
283 kind="signature",
284 id=cid,
285 error=(
286 f"Ed25519 signature INVALID for commit {cid[:12]} "
287 f"(agent {commit.agent_id!r}, key {commit.signer_key_id!r}) "
288 "— commit record may have been tampered with"
289 ),
290 ))
291 if fail_fast:
292 break
293
294 # Phase 3: check snapshot.
295 snap_id = commit.snapshot_id
296 if snap_id not in verified_snapshots:
297 verified_snapshots.add(snap_id)
298 snap = read_snapshot(root, snap_id)
299 if snap is None:
300 failures.append(VerifyFailure(
301 kind="snapshot",
302 id=snap_id,
303 error=f"snapshot missing (referenced by commit {cid[:12]})",
304 ))
305 if fail_fast:
306 break
307 else:
308 snapshots_checked += 1
309
310 # Phase 4: check objects referenced by this snapshot.
311 for rel_path, obj_id in snap.manifest.items():
312 if obj_id in verified_objects:
313 continue
314 verified_objects.add(obj_id)
315
316 obj_file = object_path(root, obj_id)
317 if not obj_file.exists():
318 failures.append(VerifyFailure(
319 kind="object",
320 id=obj_id,
321 error=f"object file missing (path={rel_path})",
322 ))
323 if fail_fast:
324 break
325 continue
326
327 objects_checked += 1
328
329 if check_objects:
330 actual = _rehash_object(obj_file)
331 if actual != obj_id:
332 failures.append(VerifyFailure(
333 kind="object",
334 id=obj_id,
335 error=(
336 f"hash mismatch: expected {obj_id[:12]} "
337 f"got {actual[:12]} — data corruption detected"
338 ),
339 ))
340 if fail_fast:
341 break
342 if fail_fast and failures:
343 break
344
345 # Enqueue parents for the BFS walk.
346 if commit.parent_commit_id:
347 queue.append(commit.parent_commit_id)
348 if commit.parent2_commit_id:
349 queue.append(commit.parent2_commit_id)
350
351 if fail_fast and failures:
352 break
353
354 return _make_result(
355 refs_checked, commits_checked, snapshots_checked,
356 objects_checked, signatures_checked, failures,
357 )
358
359
360 def _make_result(
361 refs_checked: int,
362 commits_checked: int,
363 snapshots_checked: int,
364 objects_checked: int,
365 signatures_checked: int,
366 failures: list[VerifyFailure],
367 ) -> VerifyResult:
368 """Construct a :class:`VerifyResult` from the walk counters."""
369 return VerifyResult(
370 refs_checked=refs_checked,
371 commits_checked=commits_checked,
372 snapshots_checked=snapshots_checked,
373 objects_checked=objects_checked,
374 signatures_checked=signatures_checked,
375 all_ok=len(failures) == 0,
376 failures=failures,
377 )
378
379
380 # ===========================================================================
381 # Phase 7.5: per-commit attestation verification
382 # ===========================================================================
383
384
385 #: Tier values for the attestation verification result.
386 TIER_UNSIGNED: int = 0
387 TIER_SIGNED_ONLY: int = 1
388 TIER_HMAC_ATTESTED: int = 2
389 TIER_ICP_ANCHORED: int = 3
390
391
392 class AttestationVerifyResult(TypedDict, total=False):
393 """Per-commit attestation verification result.
394
395 Returned by :func:`verify_attestation`. Maps directly to the four-tier
396 CLI output of ``muse verify-commit --attest``:
397
398 * ``tier == 0`` (UNSIGNED) — no Ed25519 signature.
399 * ``tier == 1`` (SIGNED_ONLY) — Ed25519 OK, no attestation record.
400 * ``tier == 2`` (HMAC_ATTESTED) — Ed25519 + HMAC valid, no anchor proof.
401 * ``tier == 3`` (ICP_ANCHORED) — Ed25519 + HMAC + ICP canister anchor.
402
403 Attributes:
404 valid: ``True`` iff ``tier >= 1`` and the verifier completed
405 the chain WITHOUT raising. ``False`` only for
406 provably-bad records (e.g. tampered HMAC, mismatched
407 anchor content).
408 reason: Human-readable diagnostic. Empty when
409 ``tier >= 1`` and ``valid=True``. When the verifier
410 could not complete a partial check, this carries
411 a stable code prefix like ``"ICP_UNREACHABLE — ..."``
412 so the CLI can surface "verification incomplete"
413 distinctly from "verified" or "invalid".
414 tier: One of :data:`TIER_UNSIGNED`,
415 :data:`TIER_SIGNED_ONLY`, :data:`TIER_HMAC_ATTESTED`,
416 :data:`TIER_ICP_ANCHORED`.
417 hmac_valid: ``True`` iff the HMAC signature on the attestation
418 record verified. ``False`` for unsigned commits or
419 provably-bad HMACs.
420 icp_anchored: ``True`` iff the canister returned a 200 for
421 ``GET /attest/<id>`` AND the anchored copy matches
422 the local record (same id, content_hash, path).
423 anchor_tx_id: Anchor receipt's tx_id when anchored, else ``""``.
424 identity_depth: Identity-chain depth validated. ``0`` for
425 single-signer commits; non-zero for org-quorum
426 commits (Phase 7.7).
427 provider_id: Domain string of the provider used (e.g.
428 ``"knowtation"``), or ``""`` when no provider was
429 consulted.
430 partial: ``True`` when the verifier could not complete the
431 full chain (e.g. ICP unreachable). When ``True``,
432 the CLI MUST report the partial state — never
433 silently degrade to "signed only".
434 """
435
436 valid: bool
437 reason: str
438 tier: int
439 hmac_valid: bool
440 icp_anchored: bool
441 anchor_tx_id: str
442 identity_depth: int
443 provider_id: str
444 partial: bool
445
446
447 def _check_ed25519_signature(commit: Any) -> tuple[bool, str]:
448 """Verify the commit's Ed25519 signature.
449
450 Args:
451 commit: A ``CommitRecord`` from :func:`muse.core.store.read_commit`.
452
453 Returns:
454 ``(valid, reason)`` — ``valid`` is ``True`` iff the signature exists
455 and verified. When ``False``, ``reason`` describes why.
456 """
457 if not commit.signature or commit.format_version < 7:
458 return False, "no Ed25519 signature on commit"
459 if not commit.signer_public_key:
460 return False, "signed commit missing signer_public_key"
461 import base64
462 payload = provenance_payload(
463 commit.commit_id,
464 author=commit.author,
465 agent_id=commit.agent_id,
466 model_id=commit.model_id,
467 toolchain_id=commit.toolchain_id,
468 prompt_hash=commit.prompt_hash,
469 committed_at=commit.committed_at.isoformat(),
470 )
471 padded = commit.signer_public_key + "=" * (-len(commit.signer_public_key) % 4)
472 try:
473 pub_bytes = base64.urlsafe_b64decode(padded)
474 except Exception:
475 return False, "signer_public_key is not valid base64url"
476 if not verify_commit_ed25519(payload, commit.signature, pub_bytes):
477 return False, "Ed25519 signature INVALID — commit may have been tampered with"
478 return True, ""
479
480
481 def _check_icp_anchor(
482 record: dict[str, Any],
483 canister_id: str,
484 *,
485 timeout: float = 5.0,
486 ) -> tuple[bool, str | None, str]:
487 """Check whether *record* is anchored on the ICP canister.
488
489 Performs a ``GET /attest/<id>`` and verifies that the response body
490 matches the local record on the fields that should be invariant
491 (``id``, ``content_hash``, ``path``, ``action``).
492
493 Args:
494 record: Local attestation record dict.
495 canister_id: Validated canister ID.
496 timeout: HTTP socket timeout (seconds).
497
498 Returns:
499 ``(anchored, tx_id, reason)``.
500
501 * ``anchored=True`` — canister returned a matching record;
502 ``tx_id`` populated; ``reason=""``.
503 * ``anchored=False`` — canister returned a *different* record;
504 ``tx_id=None``; ``reason`` explains the mismatch.
505 * ``(None, None, "ICP_UNREACHABLE — ...")`` — verifier could not
506 complete the check; caller must surface this as partial, not as
507 "anchored=False". Returned via the typed sentinel — the function
508 raises :class:`AttestationVerifyError` instead.
509
510 Raises:
511 AttestationVerifyError: When the canister is unreachable or the
512 response is malformed — the caller surfaces this as
513 ``partial=True``. Never silently returns ``anchored=False``.
514 """
515 from muse.plugins.knowtation.icp_push_hook import (
516 canister_http_get_url,
517 fetch_attestation_via_http,
518 )
519 rid = str(record.get("id", ""))
520 if not rid:
521 raise AttestationVerifyError("record has no id", code="MISSING_ID")
522 try:
523 anchored = fetch_attestation_via_http(canister_id, rid, timeout=timeout)
524 except OSError as exc:
525 raise AttestationVerifyError(
526 f"ICP canister unreachable: {exc}", code="ICP_UNREACHABLE"
527 ) from exc
528 if anchored is None:
529 return False, None, "canister has no record for this id (404)"
530 # Cross-check invariant fields.
531 for field in ("id", "content_hash", "path", "action"):
532 if str(record.get(field, "")) != str(anchored.get(field, "")):
533 return (
534 False, None,
535 f"anchored {field!r} differs from local record",
536 )
537 tx_id = str(anchored.get("seq", ""))
538 return True, f"icp-seq-{tx_id}" if tx_id else "icp-anchored", ""
539
540
541 def verify_attestation(
542 commit_record: Any,
543 repo_path: pathlib.Path,
544 config: dict[str, Any] | None = None,
545 ) -> AttestationVerifyResult:
546 """Verify the four-tier attestation chain for a single commit.
547
548 Tiers:
549
550 * 0 — unsigned (no Ed25519 signature)
551 * 1 — signed only (Ed25519 OK but no attestation record)
552 * 2 — signed + HMAC-attested (AIR record present and HMAC valid)
553 * 3 — signed + HMAC-attested + ICP-anchored (canister returned matching
554 record on GET /attest/<id>)
555
556 Partial-verification handling (security-critical):
557 When the verifier cannot complete a step — typically because the ICP
558 canister is unreachable — it sets ``partial=True`` and embeds a
559 structured reason like ``"ICP_UNREACHABLE — ..."`` *without* silently
560 degrading the tier. The CLI MUST surface this as "incomplete", never
561 as "verified" or "signed only".
562
563 Args:
564 commit_record: A ``CommitRecord`` (from :func:`muse.core.store.read_commit`).
565 repo_path: Repository root.
566 config: Optional config dict. Reads:
567 - ``"icp_canister_id"``: override the canister ID.
568 - ``"check_icp"``: bool, whether to attempt the canister query
569 (default ``True`` if an attestation record is present).
570
571 Returns:
572 An :class:`AttestationVerifyResult`.
573
574 Raises:
575 Never — partial verifications are surfaced via the ``partial``
576 field, NOT via exceptions. The Phase 7.5 contract is:
577 "Never silently degrade". Exceptions are caught and converted
578 to ``partial=True`` results with the appropriate reason code.
579 """
580 cfg = config or {}
581 result: AttestationVerifyResult = {
582 "valid": False,
583 "reason": "",
584 "tier": TIER_UNSIGNED,
585 "hmac_valid": False,
586 "icp_anchored": False,
587 "anchor_tx_id": "",
588 "identity_depth": 0,
589 "provider_id": "",
590 "partial": False,
591 }
592
593 # ── Tier 0 → 1: Ed25519 signature ────────────────────────────────────
594 sig_ok, sig_reason = _check_ed25519_signature(commit_record)
595 if not sig_ok:
596 result["reason"] = sig_reason or "unsigned"
597 return result
598 result["tier"] = TIER_SIGNED_ONLY
599 result["valid"] = True
600
601 # ── Tier 1 → 2: HMAC attestation ─────────────────────────────────────
602 metadata = commit_record.metadata or {}
603 blob = metadata.get("attestation")
604 if not blob:
605 return result
606 try:
607 record = json.loads(blob)
608 except json.JSONDecodeError as exc:
609 result["partial"] = True
610 result["reason"] = f"MALFORMED_RECORD — attestation JSON unparseable: {exc}"
611 return result
612 if not isinstance(record, dict):
613 result["partial"] = True
614 result["reason"] = "MALFORMED_RECORD — attestation is not a JSON object"
615 return result
616
617 provider_id = str(record.get("provider_id", ""))
618 result["provider_id"] = provider_id
619 provider = get_attestation_provider(provider_id)
620 if provider is None:
621 result["partial"] = True
622 result["reason"] = (
623 f"NO_PROVIDER — domain {provider_id!r} has no registered "
624 "attestation provider; cannot verify HMAC"
625 )
626 return result
627
628 synthetic_receipt: dict[str, Any] = {
629 "tx_id": "verify-only",
630 "anchor_url": "",
631 "anchored_at": str(record.get("timestamp", "")),
632 "provider_id": provider_id,
633 }
634 try:
635 verify_result = provider.verify(record, synthetic_receipt) # type: ignore[arg-type]
636 except AttestationVerifyError as exc:
637 result["partial"] = True
638 result["reason"] = f"{exc.code} — {exc}"
639 return result
640 if not verify_result.get("valid"):
641 result["partial"] = False
642 result["reason"] = (
643 f"HMAC_INVALID — {verify_result.get('reason') or 'provider returned valid=False'}"
644 )
645 result["valid"] = False
646 return result
647
648 result["hmac_valid"] = True
649 result["tier"] = TIER_HMAC_ATTESTED
650 result["identity_depth"] = int(verify_result.get("depth", 0) or 0)
651
652 # ── Tier 2 → 3: ICP anchor verification ──────────────────────────────
653 if not cfg.get("check_icp", True):
654 return result
655
656 canister_id = str(
657 cfg.get("icp_canister_id")
658 or _resolve_default_canister_id()
659 )
660 try:
661 anchored, tx_id, anchor_reason = _check_icp_anchor(
662 record, canister_id, timeout=float(cfg.get("icp_timeout", 5.0))
663 )
664 except AttestationVerifyError as exc:
665 result["partial"] = True
666 result["reason"] = f"{exc.code} — cannot confirm anchor; HMAC valid"
667 return result
668
669 if anchored:
670 result["icp_anchored"] = True
671 result["anchor_tx_id"] = tx_id or ""
672 result["tier"] = TIER_ICP_ANCHORED
673 return result
674
675 result["reason"] = f"ICP_NOT_ANCHORED — {anchor_reason}"
676 return result
677
678
679 def _resolve_default_canister_id() -> str:
680 """Return the default ICP canister ID, lazily importing the constant."""
681 from muse.plugins.knowtation.icp_push_hook import DEFAULT_CANISTER_ID
682 return DEFAULT_CANISTER_ID
683
684
685 # ---------------------------------------------------------------------------
686 # Phase 7.7 — identity chain walker and quorum-signature verifier
687 # ---------------------------------------------------------------------------
688
689
690 #: Hard cap on identity-chain walk depth. Prevents pathological recursion
691 #: caused by adversarial deep org-of-orgs nesting. 10 is enough for any
692 #: realistic governance hierarchy and bounds verification cost.
693 DEFAULT_IDENTITY_DEPTH_LIMIT: int = 10
694
695
696 class QuorumVerifyResult(TypedDict, total=False):
697 """Result of :func:`verify_quorum_signatures` for a single commit.
698
699 Attributes:
700 valid: ``True`` iff the number of independently-verified
701 member signatures is at least the live
702 ``quorum_threshold`` AND the org identity chain
703 validated AND no signer was revoked.
704 reason: Human-readable diagnostic. Empty when
705 ``valid=True``. When ``valid=False``, names the
706 exact failure (``BELOW_THRESHOLD``,
707 ``SIGNATURE_INVALID``, ``UNKNOWN_SIGNER``,
708 ``CYCLE_DETECTED``, ``REVOKED_MEMBER``,
709 ``IDENTITY_PROVIDER_MISSING``).
710 threshold: Live quorum threshold from the org identity record
711 (NOT the threshold copied into commit metadata —
712 that copy is for audit only).
713 valid_signers: Number of member signatures that verified against
714 their member identity's public key.
715 total_signers: Number of signers listed in ``commit.metadata['quorum']``.
716 depth: Maximum identity-chain depth walked while resolving
717 members. Surfaced for audit.
718 partial: ``True`` when the verifier could not complete the
719 check (e.g. identity provider unreachable). When
720 ``True``, the CLI MUST NOT report "verified" — the
721 result is "incomplete".
722 org_handle: The org handle from commit metadata (echoed for
723 machine-readable output).
724 """
725
726 valid: bool
727 reason: str
728 threshold: int
729 valid_signers: int
730 total_signers: int
731 depth: int
732 partial: bool
733 org_handle: str
734
735
736 def verify_identity_chain(
737 handle: str,
738 provider: MuseIdentityProvider,
739 *,
740 depth_limit: int = DEFAULT_IDENTITY_DEPTH_LIMIT,
741 ) -> IdentityChainResult:
742 """Walk the identity chain rooted at *handle* using *provider*.
743
744 Performs a depth-bounded BFS:
745
746 * Resolves *handle* via ``provider.get_identity``.
747 * For org records, recursively resolves every member.
748 * Maintains a seen-set to detect cycles defensively (the identity
749 domain rejects cycles at write time, but a tampered local cache
750 could still introduce one).
751
752 Args:
753 handle: Root handle to walk from.
754 provider: A :class:`MuseIdentityProvider` (typically resolved via
755 :func:`muse.core.attestation.get_identity_provider`).
756 depth_limit: Maximum walk depth. When the BFS reaches this depth
757 the walk terminates with ``valid=False`` and reason
758 ``"DEPTH_EXCEEDED"``.
759
760 Returns:
761 :class:`IdentityChainResult` summarising the walk.
762
763 Raises:
764 Never — any provider error is wrapped into ``valid=False`` plus
765 a stable reason code. Callers receive a complete result.
766 """
767 result: IdentityChainResult = {
768 "valid": False, "depth": 0, "reason": "", "cycle_detected": False,
769 }
770 seen: set[str] = set()
771 stack: list[tuple[str, int]] = [(handle, 1)]
772 max_depth = 0
773
774 while stack:
775 cur, depth = stack.pop()
776 if depth > depth_limit:
777 result["depth"] = max_depth
778 result["reason"] = f"DEPTH_EXCEEDED — chain deeper than {depth_limit}"
779 return result
780 if cur in seen:
781 result["depth"] = max_depth
782 result["cycle_detected"] = True
783 result["reason"] = f"CYCLE_DETECTED — handle {cur!r} appears twice on the chain"
784 return result
785 seen.add(cur)
786 max_depth = max(max_depth, depth)
787 try:
788 record = provider.get_identity(cur)
789 except AttestationVerifyError as exc:
790 result["depth"] = max_depth
791 result["reason"] = f"{exc.code} — {exc}"
792 return result
793 if record is None:
794 result["depth"] = max_depth
795 result["reason"] = f"UNKNOWN_HANDLE — no identity record for {cur!r}"
796 return result
797 if record.get("revoked_at"):
798 result["depth"] = max_depth
799 result["reason"] = f"REVOKED — {cur!r} revoked at {record['revoked_at']}"
800 return result
801 for m in record.get("members", []) or []:
802 stack.append((m, depth + 1))
803
804 result["valid"] = True
805 result["depth"] = max_depth
806 return result
807
808
809 def _verify_member_signature(
810 signer: dict[str, str],
811 member_record: IdentityRecord,
812 payload: str,
813 ) -> tuple[bool, str]:
814 """Verify a single member's Ed25519 signature on *payload*.
815
816 Cross-checks the signer's stated ``public_key`` against the member's
817 canonical ``public_key`` from the identity record — a mismatch is a
818 hard failure (a malicious commit could otherwise embed a key they
819 control alongside a real member's handle).
820
821 Args:
822 signer: ``{"handle", "public_key", "signature"}`` dict from
823 commit metadata.
824 member_record: Canonical :class:`IdentityRecord` for the same
825 handle, fetched live from the identity provider.
826 payload: The :func:`muse.core.provenance.provenance_payload`
827 digest that the member signed.
828
829 Returns:
830 ``(valid, reason)``. ``valid=True`` only when the public key
831 matches the canonical record AND the Ed25519 signature verifies.
832 """
833 import base64
834
835 declared_pk = signer.get("public_key", "")
836 canonical_pk = member_record.get("public_key", "")
837 if not canonical_pk:
838 return False, f"member {signer.get('handle')!r} has no canonical public_key"
839 if declared_pk != canonical_pk:
840 return False, (
841 f"signer {signer.get('handle')!r} public_key does not match "
842 "canonical identity record public_key"
843 )
844
845 sig_b64 = signer.get("signature", "")
846 if not sig_b64:
847 return False, "missing signature"
848
849 padded = canonical_pk + "=" * (-len(canonical_pk) % 4)
850 try:
851 pub_bytes = base64.urlsafe_b64decode(padded)
852 except Exception:
853 return False, "canonical public_key is not valid base64url"
854 if not verify_commit_ed25519(payload, sig_b64, pub_bytes):
855 return False, "Ed25519 signature INVALID"
856 return True, ""
857
858
859 def verify_quorum_signatures(
860 commit_record: Any,
861 repo_path: pathlib.Path,
862 config: dict[str, Any] | None = None,
863 ) -> QuorumVerifyResult:
864 """Verify that *commit_record* satisfies its org's quorum threshold.
865
866 The verifier:
867
868 1. Reads ``commit.metadata["quorum"]`` and parses the canonical shape
869 produced by
870 :func:`muse.plugins.knowtation.quorum.build_quorum_metadata`.
871 2. Resolves the org identity record live via the registered identity
872 provider — does NOT trust the threshold embedded in commit metadata
873 (which is for audit only; an attacker could otherwise reduce it
874 client-side).
875 3. Walks the org identity chain via :func:`verify_identity_chain`,
876 confirming all members exist, none are revoked, and no cycle
877 appears in the membership graph.
878 4. For each declared signer, fetches the member's identity record,
879 compares the embedded public_key to the canonical key (rejecting
880 any mismatch as forgery), and verifies the Ed25519 signature over
881 the same :func:`muse.core.provenance.provenance_payload` the
882 primary signer signed.
883 5. Counts unique valid member signatures. Result is
884 ``valid=True`` iff the count is at least the live
885 ``quorum_threshold`` AND no signer is unknown / outside the
886 member set.
887
888 Threats handled (per the Phase 7.7 spec):
889
890 * **Partial-quorum forgery** — only valid member signatures count;
891 ``floor(threshold - 1)`` valid sigs is rejected.
892 * **Signer collusion below threshold** — same as above; counting is
893 deterministic.
894 * **Signature stripping** — when commit metadata contains fewer valid
895 sigs than the live threshold the result is ``valid=False`` with
896 reason ``BELOW_THRESHOLD``. No silent pass.
897 * **Replay across branches** — the signed payload binds the
898 ``commit_id`` so it cannot be reused for a different commit.
899 * **Cycle-introducing membership** — :func:`verify_identity_chain`
900 sets ``cycle_detected=True`` and the verifier rejects.
901
902 Args:
903 commit_record: A ``CommitRecord`` from
904 :func:`muse.core.store.read_commit`.
905 repo_path: Repository root. Reserved for future per-repo
906 identity caches (currently unused — the identity provider
907 owns its own resolution).
908 config: Optional dict. Reads:
909
910 * ``"identity_domain"`` — domain string to resolve the
911 identity provider with (default ``"knowtation"``).
912 * ``"depth_limit"`` — override
913 :data:`DEFAULT_IDENTITY_DEPTH_LIMIT`.
914
915 Returns:
916 :class:`QuorumVerifyResult`.
917
918 Raises:
919 Never — provider errors and partial-verifications are converted
920 to ``partial=True``/``valid=False`` with stable reason codes.
921 """
922 cfg = config or {}
923 result: QuorumVerifyResult = {
924 "valid": False, "reason": "", "threshold": 0,
925 "valid_signers": 0, "total_signers": 0, "depth": 0,
926 "partial": False, "org_handle": "",
927 }
928
929 metadata = commit_record.metadata or {}
930 blob = metadata.get("quorum")
931 if not blob:
932 result["reason"] = "NO_QUORUM_METADATA — commit has no metadata['quorum']"
933 return result
934 try:
935 quorum = json.loads(blob) if isinstance(blob, str) else blob
936 except json.JSONDecodeError as exc:
937 result["partial"] = True
938 result["reason"] = f"MALFORMED_QUORUM — {exc}"
939 return result
940 if not isinstance(quorum, dict):
941 result["partial"] = True
942 result["reason"] = "MALFORMED_QUORUM — not a JSON object"
943 return result
944
945 org_handle = str(quorum.get("org_handle", ""))
946 result["org_handle"] = org_handle
947 if not org_handle.startswith("@"):
948 result["reason"] = f"INVALID_ORG_HANDLE — {org_handle!r} does not start with '@'"
949 return result
950
951 signers = quorum.get("signers", [])
952 if not isinstance(signers, list) or not signers:
953 result["reason"] = "MALFORMED_QUORUM — signers must be a non-empty list"
954 return result
955 result["total_signers"] = len(signers)
956
957 domain = str(cfg.get("identity_domain", "knowtation"))
958 provider = get_identity_provider(domain)
959 if provider is None:
960 result["partial"] = True
961 result["reason"] = (
962 f"IDENTITY_PROVIDER_MISSING — no identity provider registered "
963 f"for domain {domain!r}; cannot verify quorum"
964 )
965 return result
966
967 depth_limit = int(cfg.get("depth_limit", DEFAULT_IDENTITY_DEPTH_LIMIT))
968
969 # Step 1 — fetch and validate the org record (live, not from metadata).
970 try:
971 org_record = provider.get_identity(org_handle)
972 except AttestationVerifyError as exc:
973 result["partial"] = True
974 result["reason"] = f"{exc.code} — fetching org {org_handle!r}: {exc}"
975 return result
976 if org_record is None:
977 result["reason"] = f"UNKNOWN_ORG — no identity record for {org_handle!r}"
978 return result
979 if org_record.get("kind") != "org":
980 result["reason"] = (
981 f"NOT_AN_ORG — handle {org_handle!r} kind is "
982 f"{org_record.get('kind')!r}, expected 'org'"
983 )
984 return result
985 if org_record.get("revoked_at"):
986 result["reason"] = (
987 f"ORG_REVOKED — {org_handle!r} revoked at "
988 f"{org_record['revoked_at']}"
989 )
990 return result
991
992 # Step 2 — walk the identity chain (cycle detection + member resolution).
993 chain = verify_identity_chain(org_handle, provider, depth_limit=depth_limit)
994 result["depth"] = chain.get("depth", 0)
995 if not chain.get("valid"):
996 if chain.get("cycle_detected"):
997 result["reason"] = chain.get("reason", "CYCLE_DETECTED")
998 else:
999 result["reason"] = chain.get("reason", "CHAIN_INVALID")
1000 return result
1001
1002 # Step 3 — recompute the signed payload from canonical commit fields.
1003 payload = provenance_payload(
1004 commit_record.commit_id,
1005 author=getattr(commit_record, "author", "") or "",
1006 agent_id=getattr(commit_record, "agent_id", "") or "",
1007 model_id=getattr(commit_record, "model_id", "") or "",
1008 toolchain_id=getattr(commit_record, "toolchain_id", "") or "",
1009 prompt_hash=getattr(commit_record, "prompt_hash", "") or "",
1010 committed_at=commit_record.committed_at.isoformat(),
1011 )
1012
1013 # Step 4 — verify each declared signer's signature.
1014 threshold = int(org_record.get("quorum_threshold", 0))
1015 result["threshold"] = threshold
1016 member_set = set(org_record.get("members", []) or [])
1017 counted: set[str] = set()
1018
1019 for signer in signers:
1020 if not isinstance(signer, dict):
1021 result["reason"] = "MALFORMED_QUORUM — signer entry is not a dict"
1022 return result
1023 handle = str(signer.get("handle", ""))
1024 if handle not in member_set:
1025 result["reason"] = (
1026 f"UNKNOWN_SIGNER — {handle!r} is not a member of org {org_handle!r}"
1027 )
1028 return result
1029 if handle in counted:
1030 result["reason"] = (
1031 f"DUPLICATE_SIGNER — {handle!r} appears twice; duplicates do not"
1032 " count toward quorum"
1033 )
1034 return result
1035
1036 try:
1037 member_record = provider.get_identity(handle)
1038 except AttestationVerifyError as exc:
1039 result["partial"] = True
1040 result["reason"] = f"{exc.code} — fetching member {handle!r}: {exc}"
1041 return result
1042 if member_record is None:
1043 result["reason"] = f"UNKNOWN_MEMBER — identity record missing for {handle!r}"
1044 return result
1045 if member_record.get("revoked_at"):
1046 result["reason"] = (
1047 f"REVOKED_MEMBER — {handle!r} revoked at "
1048 f"{member_record['revoked_at']}"
1049 )
1050 return result
1051
1052 ok, why = _verify_member_signature(signer, member_record, payload)
1053 if not ok:
1054 result["reason"] = f"SIGNATURE_INVALID — {handle!r}: {why}"
1055 return result
1056 counted.add(handle)
1057
1058 result["valid_signers"] = len(counted)
1059 if len(counted) < threshold:
1060 result["reason"] = (
1061 f"BELOW_THRESHOLD — {len(counted)} valid signatures, "
1062 f"threshold is {threshold}"
1063 )
1064 return result
1065
1066 result["valid"] = True
1067 return result
File History 1 commit
sha256:cf6265cea8c21d9228d90dec13ef6ec2dab5103d466db9cc4590681832de4bf8 docs(KD-STAGING): sync governance after KD-6b DONE Human 13 days ago