aaronrene / overseer-kit public
test_provenance_integrity.py python
127 lines 5.1 KB
Raw
sha256:0e9549ec7b463911bc08b7d586dc320b1ac9b1f5c943ee7e3865dcc6cb0f6f83 chore(governance): sync handover+roadmap to 84db8c8 (drift:… Human 1 day ago
1 """Data-integrity tests for Track P / P1 provenance (§P0.8)."""
2
3 from __future__ import annotations
4
5 import json
6
7 from tests.support import (
8 attach_signed_provenance,
9 generate_ed25519_keypair,
10 load_honesty_config,
11 load_honesty_entry,
12 sign_append_body,
13 )
14 from tools.honesty.canonical import compute_entry_hash
15 from tools.honesty.genesis import build_genesis_entry
16 from tools.honesty.ledger import append_entry, verify_ledger_file, verify_chain
17 from tools.honesty.ledger_io import read_ledger_entries, serialize_entry
18 from tools.honesty.types import LedgerAppendOptions
19
20
21 def _append_signed_verdict(repo_root) -> tuple[dict, dict]:
22 genesis = build_genesis_entry(ts="2026-01-01T00:00:00Z")
23 ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl"
24 ledger.parent.mkdir(parents=True, exist_ok=True)
25 ledger.write_text(serialize_entry(genesis), encoding="utf-8")
26 config = load_honesty_config(repo_root)
27 private_key, pubkey = generate_ed25519_keypair()
28 body = load_honesty_entry(repo_root, "verdict-pass.json")
29 body["ts"] = "2026-01-02T00:00:00Z"
30 body = attach_signed_provenance(body, pubkey_token=pubkey)
31 body = sign_append_body(
32 body,
33 kind="verdict",
34 prev_hash=genesis["entry_hash"],
35 private_key=private_key,
36 pubkey_token=pubkey,
37 )
38 assert append_entry(
39 config=config,
40 repo_root=repo_root,
41 options=LedgerAppendOptions(kind="verdict", body=body),
42 ).exit_code == 0
43 entries = read_ledger_entries(ledger)
44 return entries[-1], private_key
45
46
47 def test_verify_idempotent_with_signatures(repo_root) -> None:
48 _append_signed_verdict(repo_root)
49 config = load_honesty_config(repo_root)
50 first = verify_ledger_file(config=config, repo_root=repo_root)
51 second = verify_ledger_file(config=config, repo_root=repo_root)
52 assert first.exit_code == 0
53 assert second.exit_code == 0
54
55
56 def test_body_tamper_flips_22(repo_root) -> None:
57 entry, _ = _append_signed_verdict(repo_root)
58 config = load_honesty_config(repo_root)
59 ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl"
60 tampered = dict(entry)
61 tampered["passed"] = False
62 ledger.write_text(
63 serialize_entry(build_genesis_entry(ts="2026-01-01T00:00:00Z")) + serialize_entry(tampered),
64 encoding="utf-8",
65 )
66 assert verify_ledger_file(config=config, repo_root=repo_root).exit_code == 22
67
68
69 def test_sig_tamper_flips_25(repo_root) -> None:
70 entry, _ = _append_signed_verdict(repo_root)
71 config = load_honesty_config(repo_root)
72 ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl"
73 tampered = dict(entry)
74 tampered["provenance"] = dict(entry["provenance"])
75 tampered["provenance"]["sig"] = "ed25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
76 ledger.write_text(
77 serialize_entry(build_genesis_entry(ts="2026-01-01T00:00:00Z")) + serialize_entry(tampered),
78 encoding="utf-8",
79 )
80 assert verify_ledger_file(config=config, repo_root=repo_root).exit_code == 25
81
82
83 def test_excising_sig_does_not_change_entry_hash(repo_root) -> None:
84 entry, _ = _append_signed_verdict(repo_root)
85 with_sig_hash = compute_entry_hash(entry)
86 stripped = dict(entry)
87 stripped["provenance"] = {key: val for key, val in entry["provenance"].items() if key != "sig"}
88 without_sig_hash = compute_entry_hash(stripped)
89 assert with_sig_hash == without_sig_hash
90
91
92 def test_verify_chain_direct_mix(repo_root) -> None:
93 entry, _ = _append_signed_verdict(repo_root)
94 ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl"
95 entries = read_ledger_entries(ledger)
96 assert verify_chain(entries, regime="git-only") == 0
97 assert entry["provenance"]["sig"].startswith("ed25519:")
98
99
100 def test_verify_flags_malformed_provenance_exit_2(repo_root) -> None:
101 """§P0.6: `verify` must emit exit 2 on a hash-consistent but malformed provenance.
102
103 A tamperer who inserts an unknown provenance key and recomputes ``entry_hash``
104 keeps the chain hash-consistent (so it is not a ``22``); the structural defect
105 must still surface as ``2`` — the malformed-provenance code the frozen contract
106 names for ``verify``.
107 """
108 config = load_honesty_config(repo_root)
109 genesis = build_genesis_entry(ts="2026-01-01T00:00:00Z")
110 entry = {
111 "v": 1,
112 "ts": "2026-01-02T00:00:00Z",
113 "kind": "hook_check",
114 "actor_role": "overseer",
115 "actor_session_id": "s1",
116 "hook": "handoff",
117 "ok": True,
118 "prev_hash": genesis["entry_hash"],
119 "provenance": {"agent_id": "a", "model_id": "m", "rogue": True},
120 }
121 entry["entry_hash"] = compute_entry_hash(entry)
122 ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl"
123 ledger.parent.mkdir(parents=True, exist_ok=True)
124 ledger.write_text(serialize_entry(genesis) + serialize_entry(entry), encoding="utf-8")
125 entries = read_ledger_entries(ledger)
126 assert verify_chain(entries, regime="git-only") == 2
127 assert verify_ledger_file(config=config, repo_root=repo_root).exit_code == 2
File History 1 commit
sha256:6abcf1fa82a7a621ccbc945f19acdba5bc0db54569599404a1452fb4a096a199 fix(ISR): default require_independent_second_reviewer to require Human minor ⚠ 1 day ago