test_aff_integrity.py
python
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4
docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit.
Human
3 days ago
| 1 | """Data-integrity tests for AFF last-verdict + hash-consistent malformations (§AFF.14).""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | from pathlib import Path |
| 7 | |
| 8 | from tests.fixtures.aff import ( |
| 9 | AFF_FREEZE_REL, |
| 10 | aff_body, |
| 11 | current_artifact_digest, |
| 12 | seed_aff_repo, |
| 13 | write_mechanical_stamp, |
| 14 | ) |
| 15 | from tools.adversarial_freeze import adversarial_authorization_state |
| 16 | from tools.honesty.canonical import compute_entry_hash |
| 17 | from tools.honesty.genesis import GENESIS_PREV, build_genesis_entry |
| 18 | from tools.honesty.ledger import append_entry, verify_chain, verify_ledger_file |
| 19 | from tools.honesty.ledger_io import read_ledger_entries, serialize_entry |
| 20 | from tools.honesty.status import ( |
| 21 | EXIT_MISSING_ADVERSARIAL_FREEZE, |
| 22 | HonestyStatusOptions, |
| 23 | run_honesty_status, |
| 24 | ) |
| 25 | from tools.honesty.types import LedgerAppendOptions |
| 26 | from tools.honesty.validate import ( |
| 27 | find_latest_adversarial_freeze_verdict, |
| 28 | find_matching_adversarial_freeze_pass, |
| 29 | validate_append_body, |
| 30 | ) |
| 31 | from tools.governance_hygiene.next_regen import plan_next_regen, render_paste_ready |
| 32 | |
| 33 | |
| 34 | def _write_ledger(repo_root: Path, entries: list[dict]) -> None: |
| 35 | ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 36 | ledger.parent.mkdir(parents=True, exist_ok=True) |
| 37 | lines = [serialize_entry(e) for e in entries] |
| 38 | ledger.write_text("\n".join(lines) + "\n", encoding="utf-8") |
| 39 | |
| 40 | |
| 41 | def _chain_append(prev: dict, body: dict) -> dict: |
| 42 | entry = dict(body) |
| 43 | if "ts" not in entry: |
| 44 | entry["ts"] = "2026-01-01T00:00:00Z" |
| 45 | if "v" not in entry: |
| 46 | entry["v"] = 1 |
| 47 | entry["prev_hash"] = prev["entry_hash"] |
| 48 | entry["entry_hash"] = compute_entry_hash(entry) |
| 49 | return entry |
| 50 | |
| 51 | |
| 52 | def test_last_verdict_matrix_uses_append_accepted_bodies(repo_root: Path) -> None: |
| 53 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 54 | digest = write_mechanical_stamp(repo_root) |
| 55 | path = AFF_FREEZE_REL |
| 56 | |
| 57 | def append(name: str, **kw): |
| 58 | body = aff_body(name, artifact_digest=digest, **kw) |
| 59 | # prove validate accepts before append |
| 60 | validate_append_body(kind="adversarial_freeze", body=body) |
| 61 | assert ( |
| 62 | append_entry( |
| 63 | config=config, |
| 64 | repo_root=repo_root, |
| 65 | options=LedgerAppendOptions(kind="adversarial_freeze", body=body), |
| 66 | ).exit_code |
| 67 | == 0 |
| 68 | ) |
| 69 | |
| 70 | append("aff-pass.json", round=1, actor_session_id="adv-1") |
| 71 | entries = read_ledger_entries( |
| 72 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 73 | ) |
| 74 | assert find_matching_adversarial_freeze_pass( |
| 75 | entries, frozen_spec=path, artifact_digest=digest |
| 76 | ) is not None |
| 77 | |
| 78 | # pass → findings = pending (older pass revoked) |
| 79 | append("aff-findings.json", round=2, actor_session_id="adv-2") |
| 80 | entries = read_ledger_entries( |
| 81 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 82 | ) |
| 83 | assert ( |
| 84 | find_matching_adversarial_freeze_pass( |
| 85 | entries, frozen_spec=path, artifact_digest=digest |
| 86 | ) |
| 87 | is None |
| 88 | ) |
| 89 | latest = find_latest_adversarial_freeze_verdict( |
| 90 | entries, frozen_spec=path, artifact_digest=digest |
| 91 | ) |
| 92 | assert latest["aff_verdict"] == "findings" |
| 93 | |
| 94 | # findings → pass = pass |
| 95 | append("aff-pass.json", round=3, actor_session_id="adv-3") |
| 96 | entries = read_ledger_entries( |
| 97 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 98 | ) |
| 99 | assert ( |
| 100 | find_matching_adversarial_freeze_pass( |
| 101 | entries, frozen_spec=path, artifact_digest=digest |
| 102 | ) |
| 103 | is not None |
| 104 | ) |
| 105 | |
| 106 | # pass → blocked = pending |
| 107 | append("aff-blocked.json", round=4, actor_session_id="adv-4") |
| 108 | entries = read_ledger_entries( |
| 109 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 110 | ) |
| 111 | assert ( |
| 112 | find_matching_adversarial_freeze_pass( |
| 113 | entries, frozen_spec=path, artifact_digest=digest |
| 114 | ) |
| 115 | is None |
| 116 | ) |
| 117 | |
| 118 | # blocked → pass, then pass → skip under suggest |
| 119 | append("aff-pass.json", round=5, actor_session_id="adv-5") |
| 120 | append("aff-skip.json", round=6) |
| 121 | entries = read_ledger_entries( |
| 122 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 123 | ) |
| 124 | latest = find_latest_adversarial_freeze_verdict( |
| 125 | entries, frozen_spec=path, artifact_digest=digest |
| 126 | ) |
| 127 | assert latest["aff_verdict"] == "skip" |
| 128 | auth = adversarial_authorization_state( |
| 129 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 130 | ) |
| 131 | assert auth.state == "skipped" |
| 132 | |
| 133 | # skip → findings = pending |
| 134 | append("aff-findings.json", round=7, actor_session_id="adv-7") |
| 135 | auth = adversarial_authorization_state( |
| 136 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 137 | ) |
| 138 | assert auth.state == "pending" |
| 139 | |
| 140 | |
| 141 | def test_different_digest_path_do_not_revoke(repo_root: Path) -> None: |
| 142 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 143 | digest = write_mechanical_stamp(repo_root) |
| 144 | other = "sha256:" + ("b" * 64) |
| 145 | append_entry( |
| 146 | config=config, |
| 147 | repo_root=repo_root, |
| 148 | options=LedgerAppendOptions( |
| 149 | kind="adversarial_freeze", |
| 150 | body=aff_body(artifact_digest=digest, actor_session_id="a1"), |
| 151 | ), |
| 152 | ) |
| 153 | append_entry( |
| 154 | config=config, |
| 155 | repo_root=repo_root, |
| 156 | options=LedgerAppendOptions( |
| 157 | kind="adversarial_freeze", |
| 158 | body=aff_body( |
| 159 | "aff-findings.json", |
| 160 | artifact_digest=other, |
| 161 | actor_session_id="a2", |
| 162 | frozen_spec="docs/other.md", |
| 163 | ), |
| 164 | ), |
| 165 | ) |
| 166 | entries = read_ledger_entries( |
| 167 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 168 | ) |
| 169 | assert ( |
| 170 | find_matching_adversarial_freeze_pass( |
| 171 | entries, frozen_spec=AFF_FREEZE_REL, artifact_digest=digest |
| 172 | ) |
| 173 | is not None |
| 174 | ) |
| 175 | |
| 176 | |
| 177 | def test_skip_digest_d_does_not_authorize_d_prime(repo_root: Path) -> None: |
| 178 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 179 | digest = write_mechanical_stamp(repo_root) |
| 180 | other = "sha256:" + ("c" * 64) |
| 181 | append_entry( |
| 182 | config=config, |
| 183 | repo_root=repo_root, |
| 184 | options=LedgerAppendOptions( |
| 185 | kind="adversarial_freeze", |
| 186 | body=aff_body("aff-skip.json", artifact_digest=digest), |
| 187 | ), |
| 188 | ) |
| 189 | entries = read_ledger_entries( |
| 190 | repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 191 | ) |
| 192 | assert ( |
| 193 | find_latest_adversarial_freeze_verdict( |
| 194 | entries, frozen_spec=AFF_FREEZE_REL, artifact_digest=other |
| 195 | ) |
| 196 | is None |
| 197 | ) |
| 198 | |
| 199 | |
| 200 | def test_ok_next_twice_identical_fence(repo_root: Path) -> None: |
| 201 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 202 | write_mechanical_stamp(repo_root) |
| 203 | roadmap = ( |
| 204 | "# Roadmap\n\n## Build queue\n\n" |
| 205 | "| Phase | Model | Status | Deliverable |\n" |
| 206 | "| --- | --- | --- | --- |\n" |
| 207 | f"| **AFF-a** | Thinking | **NEXT** | `{AFF_FREEZE_REL}` |\n" |
| 208 | ) |
| 209 | handover = "## NEXT SESSION — AFF-a\n\n| | |\n| **ID** | **AFF-a** |\n\n" |
| 210 | d1 = plan_next_regen( |
| 211 | roadmap_text=roadmap, handover_text=handover, config=config, repo_root=repo_root |
| 212 | ) |
| 213 | d2 = plan_next_regen( |
| 214 | roadmap_text=roadmap, handover_text=handover, config=config, repo_root=repo_root |
| 215 | ) |
| 216 | assert render_paste_ready(decision=d1, config=config) == render_paste_ready( |
| 217 | decision=d2, config=config |
| 218 | ) |
| 219 | |
| 220 | |
| 221 | def test_broken_ledger_absent_hold_never_auto(repo_root: Path) -> None: |
| 222 | config = seed_aff_repo(repo_root, adversarial_freeze="require") |
| 223 | digest = write_mechanical_stamp(repo_root) |
| 224 | append_entry( |
| 225 | config=config, |
| 226 | repo_root=repo_root, |
| 227 | options=LedgerAppendOptions( |
| 228 | kind="freeze_review", |
| 229 | body={ |
| 230 | "actor_role": "verifier", |
| 231 | "actor_session_id": "frv-1", |
| 232 | "phase_id": "AFF-b", |
| 233 | "frozen_spec": AFF_FREEZE_REL, |
| 234 | "round": 1, |
| 235 | "gate": "substantive", |
| 236 | "freeze_verdict": "pass", |
| 237 | "artifact_digest": digest, |
| 238 | "reviewer_model": "thinking-high", |
| 239 | }, |
| 240 | ), |
| 241 | ) |
| 242 | ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 243 | ledger.write_text(ledger.read_text(encoding="utf-8") + "{bad}\n", encoding="utf-8") |
| 244 | |
| 245 | auth = adversarial_authorization_state( |
| 246 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 247 | ) |
| 248 | assert auth.state == "absent" |
| 249 | |
| 250 | roadmap = ( |
| 251 | "# Roadmap\n\n## Build queue\n\n" |
| 252 | "| Phase | Model | Status | Deliverable |\n" |
| 253 | "| --- | --- | --- | --- |\n" |
| 254 | f"| **AFF-b** | Auto | **NEXT** | `{AFF_FREEZE_REL}` |\n" |
| 255 | ) |
| 256 | decision = plan_next_regen( |
| 257 | roadmap_text=roadmap, |
| 258 | handover_text="## NEXT\n| **ID** | **AFF-b** |\n", |
| 259 | config=config, |
| 260 | repo_root=repo_root, |
| 261 | ) |
| 262 | assert decision.emit_model != "Auto" |
| 263 | |
| 264 | |
| 265 | def test_mode_e_tamper_integrity_exits(repo_root: Path) -> None: |
| 266 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 267 | digest = write_mechanical_stamp(repo_root) |
| 268 | append_entry( |
| 269 | config=config, |
| 270 | repo_root=repo_root, |
| 271 | options=LedgerAppendOptions( |
| 272 | kind="adversarial_freeze", |
| 273 | body=aff_body(artifact_digest=digest), |
| 274 | ), |
| 275 | ) |
| 276 | # mutate prev_hash |
| 277 | ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 278 | lines = ledger.read_text(encoding="utf-8").strip().splitlines() |
| 279 | last = json.loads(lines[-1]) |
| 280 | last["prev_hash"] = "0" * 64 |
| 281 | lines[-1] = json.dumps(last, separators=(",", ":"), sort_keys=True) |
| 282 | ledger.write_text("\n".join(lines) + "\n", encoding="utf-8") |
| 283 | |
| 284 | result = run_honesty_status( |
| 285 | config=config, |
| 286 | repo_root=repo_root, |
| 287 | options=HonestyStatusOptions( |
| 288 | hook=None, |
| 289 | artifact=None, |
| 290 | adversarial_freeze="AFF", |
| 291 | frozen_spec=AFF_FREEZE_REL, |
| 292 | ), |
| 293 | ) |
| 294 | assert result.exit_code == 22 |
| 295 | assert result.exit_code != 0 |
| 296 | |
| 297 | |
| 298 | def test_hash_consistent_malformed_ineligible(repo_root: Path) -> None: |
| 299 | config = seed_aff_repo(repo_root, adversarial_freeze="require") |
| 300 | digest = write_mechanical_stamp(repo_root) |
| 301 | genesis = build_genesis_entry("2026-01-01T00:00:00Z") |
| 302 | |
| 303 | def make_pass(**mut) -> dict: |
| 304 | body = aff_body(artifact_digest=digest) |
| 305 | body.update(mut) |
| 306 | body["ts"] = "2026-01-01T00:00:01Z" |
| 307 | body["v"] = 1 |
| 308 | return body |
| 309 | |
| 310 | cases = [ |
| 311 | make_pass(phase_id=""), |
| 312 | make_pass(round=True), |
| 313 | {k: v for k, v in make_pass().items() if k != "reviewer_model"}, |
| 314 | make_pass(phase_id=" "), |
| 315 | make_pass(round=0), |
| 316 | ] |
| 317 | # ensure round:true retains envelope fields |
| 318 | for body in cases: |
| 319 | if "ts" not in body: |
| 320 | body["ts"] = "2026-01-01T00:00:01Z" |
| 321 | if "v" not in body: |
| 322 | body["v"] = 1 |
| 323 | |
| 324 | for body in cases: |
| 325 | entry = _chain_append(genesis, body) |
| 326 | assert verify_chain([genesis, entry]) == 0 |
| 327 | winner = find_latest_adversarial_freeze_verdict( |
| 328 | [genesis, entry], |
| 329 | frozen_spec=AFF_FREEZE_REL, |
| 330 | artifact_digest=digest, |
| 331 | ) |
| 332 | assert winner is None |
| 333 | _write_ledger(repo_root, [genesis, entry]) |
| 334 | auth = adversarial_authorization_state( |
| 335 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 336 | ) |
| 337 | assert auth.state in {"pending", "absent"} |
| 338 | assert auth.matched_via != "pass" |
| 339 | result = run_honesty_status( |
| 340 | config=config, |
| 341 | repo_root=repo_root, |
| 342 | options=HonestyStatusOptions( |
| 343 | hook=None, |
| 344 | artifact=None, |
| 345 | adversarial_freeze="AFF", |
| 346 | frozen_spec=AFF_FREEZE_REL, |
| 347 | ), |
| 348 | ) |
| 349 | assert result.exit_code == EXIT_MISSING_ADVERSARIAL_FREEZE |
| 350 | assert result.json_payload.adversarial_freeze["matched_via"] is None |
| 351 | |
| 352 | |
| 353 | def test_hash_consistent_missing_ts_and_v_true_fail_closed(repo_root: Path) -> None: |
| 354 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 355 | digest = write_mechanical_stamp(repo_root) |
| 356 | genesis = build_genesis_entry("2026-01-01T00:00:00Z") |
| 357 | |
| 358 | # omit ts, recompute hash |
| 359 | body = aff_body(artifact_digest=digest) |
| 360 | body["v"] = 1 |
| 361 | body.pop("ts", None) |
| 362 | entry = dict(body) |
| 363 | entry["prev_hash"] = genesis["entry_hash"] |
| 364 | entry["entry_hash"] = compute_entry_hash(entry) |
| 365 | assert verify_chain([genesis, entry]) == 22 |
| 366 | _write_ledger(repo_root, [genesis, entry]) |
| 367 | auth = adversarial_authorization_state( |
| 368 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 369 | ) |
| 370 | assert auth.state == "absent" |
| 371 | result = run_honesty_status( |
| 372 | config=config, |
| 373 | repo_root=repo_root, |
| 374 | options=HonestyStatusOptions( |
| 375 | hook=None, |
| 376 | artifact=None, |
| 377 | adversarial_freeze="AFF", |
| 378 | frozen_spec=AFF_FREEZE_REL, |
| 379 | ), |
| 380 | ) |
| 381 | assert result.exit_code == 22 |
| 382 | |
| 383 | # v: true with matching hash |
| 384 | body2 = aff_body(artifact_digest=digest, actor_session_id="adv-x") |
| 385 | body2["v"] = True |
| 386 | body2["ts"] = "2026-01-01T00:00:02Z" |
| 387 | entry2 = dict(body2) |
| 388 | entry2["prev_hash"] = genesis["entry_hash"] |
| 389 | entry2["entry_hash"] = compute_entry_hash(entry2) |
| 390 | assert verify_chain([genesis, entry2]) == 22 |
| 391 | _write_ledger(repo_root, [genesis, entry2]) |
| 392 | result2 = run_honesty_status( |
| 393 | config=config, |
| 394 | repo_root=repo_root, |
| 395 | options=HonestyStatusOptions( |
| 396 | hook=None, |
| 397 | artifact=None, |
| 398 | adversarial_freeze="AFF", |
| 399 | frozen_spec=AFF_FREEZE_REL, |
| 400 | ), |
| 401 | ) |
| 402 | assert result2.exit_code == 22 |
File History
1 commit
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4
docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit.
Human
3 days ago