test_aff_security.py
python
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4
docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit.
Human
4 days ago
| 1 | """Security tests for AFF fail-closed gates (§AFF.14).""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import inspect |
| 6 | from pathlib import Path |
| 7 | |
| 8 | from cli.kit_root import kit_root |
| 9 | from tests.fixtures.aff import ( |
| 10 | AFF_FREEZE_REL, |
| 11 | aff_body, |
| 12 | seed_aff_repo, |
| 13 | write_mechanical_stamp, |
| 14 | ) |
| 15 | from tools.adversarial_freeze import ( |
| 16 | adversarial_authorization_state, |
| 17 | aff_hold_bypassed, |
| 18 | author_loop_complete, |
| 19 | build_adversarial_freeze_gate, |
| 20 | ) |
| 21 | from tools.honesty.canonical import compute_entry_hash |
| 22 | from tools.honesty.genesis import build_genesis_entry |
| 23 | from tools.honesty.ledger import append_entry, verify_chain |
| 24 | from tools.honesty.ledger_io import serialize_entry |
| 25 | from tools.honesty.status import ( |
| 26 | EXIT_MISSING_ADVERSARIAL_FREEZE, |
| 27 | HonestyStatusOptions, |
| 28 | run_honesty_status, |
| 29 | ) |
| 30 | from tools.honesty.types import LedgerAppendOptions |
| 31 | from tools.honesty.validate import find_latest_adversarial_freeze_verdict |
| 32 | from tools.governance_hygiene.next_regen import ( |
| 33 | ADVISORY_ADVERSARIAL_FREEZE_PENDING, |
| 34 | decide_split_emission, |
| 35 | plan_next_regen, |
| 36 | ) |
| 37 | from tools.governance_hygiene.types import QueueRow |
| 38 | |
| 39 | |
| 40 | def test_no_secret_key_url_in_skill_and_paste() -> None: |
| 41 | skill = ( |
| 42 | kit_root() |
| 43 | / "cursor" |
| 44 | / "skills" |
| 45 | / "adversarial-freeze-review" |
| 46 | / "SKILL.md" |
| 47 | ).read_text(encoding="utf-8") |
| 48 | assert "sk-" not in skill |
| 49 | assert "BEGIN PRIVATE" not in skill |
| 50 | assert "api_key" not in skill.lower() |
| 51 | # placeholders only |
| 52 | assert "<THIS_CHAT_SESSION_ID>" in skill or "actor_session_id" in skill |
| 53 | |
| 54 | |
| 55 | def test_freeze_review_loop_forbids_auto_may_start_true() -> None: |
| 56 | skill = ( |
| 57 | kit_root() / "cursor" / "skills" / "freeze-review-loop" / "SKILL.md" |
| 58 | ).read_text(encoding="utf-8") |
| 59 | assert "MUST NOT write, insert, or flip `auto_may_start` to `true`" in skill |
| 60 | assert "MUST NOT write `auto_may_start` at all" in skill |
| 61 | # must not instruct agents to enable the flag |
| 62 | assert "set auto_may_start to true" not in skill.lower() |
| 63 | assert "flip auto_may_start to true and" not in skill.lower() |
| 64 | |
| 65 | |
| 66 | def test_url_like_session_ids_opaque(repo_root) -> None: |
| 67 | config = seed_aff_repo(repo_root) |
| 68 | digest = write_mechanical_stamp(repo_root) |
| 69 | body = aff_body( |
| 70 | artifact_digest=digest, |
| 71 | actor_session_id="https://evil.example/$(curl)", |
| 72 | producer_session_id="file:///etc/passwd; rm -rf /", |
| 73 | notes="https://example.com/hook?x=`id`", |
| 74 | ) |
| 75 | assert ( |
| 76 | append_entry( |
| 77 | config=config, |
| 78 | repo_root=repo_root, |
| 79 | options=LedgerAppendOptions(kind="adversarial_freeze", body=body), |
| 80 | ).exit_code |
| 81 | == 0 |
| 82 | ) |
| 83 | |
| 84 | |
| 85 | def test_no_network_or_model_imports_on_aff_paths() -> None: |
| 86 | import tools.adversarial_freeze.authorize as auth_mod |
| 87 | import tools.adversarial_freeze.surface as surface_mod |
| 88 | import tools.honesty.status as status_mod |
| 89 | import tools.honesty.validate as validate_mod |
| 90 | |
| 91 | for module in (auth_mod, surface_mod, status_mod, validate_mod): |
| 92 | source = inspect.getsource(module) |
| 93 | assert "urllib" not in source |
| 94 | assert "requests" not in source |
| 95 | assert "httpx" not in source |
| 96 | assert "openai" not in source |
| 97 | assert "anthropic" not in source |
| 98 | |
| 99 | |
| 100 | def test_author_session_cannot_append_pass_equal_ids(repo_root) -> None: |
| 101 | config = seed_aff_repo(repo_root) |
| 102 | digest = write_mechanical_stamp(repo_root) |
| 103 | body = aff_body( |
| 104 | artifact_digest=digest, |
| 105 | actor_session_id="same-session", |
| 106 | producer_session_id="same-session", |
| 107 | ) |
| 108 | result = append_entry( |
| 109 | config=config, |
| 110 | repo_root=repo_root, |
| 111 | options=LedgerAppendOptions(kind="adversarial_freeze", body=body), |
| 112 | ) |
| 113 | assert result.exit_code == 2 |
| 114 | |
| 115 | |
| 116 | def test_mechanical_stamp_without_aff_holds_auto(repo_root) -> None: |
| 117 | for mode in ("suggest", "require"): |
| 118 | config = seed_aff_repo(repo_root, adversarial_freeze=mode) |
| 119 | digest = write_mechanical_stamp(repo_root) |
| 120 | append_entry( |
| 121 | config=config, |
| 122 | repo_root=repo_root, |
| 123 | options=LedgerAppendOptions( |
| 124 | kind="freeze_review", |
| 125 | body={ |
| 126 | "actor_role": "verifier", |
| 127 | "actor_session_id": "same-as-author-would-be", |
| 128 | "phase_id": "AFF-b", |
| 129 | "frozen_spec": AFF_FREEZE_REL, |
| 130 | "round": 1, |
| 131 | "gate": "substantive", |
| 132 | "freeze_verdict": "pass", |
| 133 | "artifact_digest": digest, |
| 134 | "reviewer_model": "thinking-high", |
| 135 | }, |
| 136 | ), |
| 137 | ) |
| 138 | roadmap = ( |
| 139 | "# Roadmap\n\n## Build queue\n\n" |
| 140 | "| Phase | Model | Status | Deliverable |\n" |
| 141 | "| --- | --- | --- | --- |\n" |
| 142 | f"| **AFF-b** | Auto | **NEXT** | `{AFF_FREEZE_REL}` |\n" |
| 143 | ) |
| 144 | decision = plan_next_regen( |
| 145 | roadmap_text=roadmap, |
| 146 | handover_text="## NEXT\n| **ID** | **AFF-b** |\n", |
| 147 | config=config, |
| 148 | repo_root=repo_root, |
| 149 | ) |
| 150 | assert decision.emit_model == "Thinking" |
| 151 | assert decision.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING |
| 152 | |
| 153 | |
| 154 | def test_stale_mechanical_only_does_not_satisfy_trigger_b(repo_root) -> None: |
| 155 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 156 | write_mechanical_stamp(repo_root) |
| 157 | art = repo_root / AFF_FREEZE_REL |
| 158 | art.write_text(art.read_text(encoding="utf-8") + "\nstale\n", encoding="utf-8") |
| 159 | # stamp digest now unequal to current |
| 160 | from tools.freeze_reviewer.artifact import extract_existing_stamp, parse_artifact, artifact_digest |
| 161 | |
| 162 | parsed = parse_artifact(art, rel_path=AFF_FREEZE_REL) |
| 163 | stamp = extract_existing_stamp(parsed) |
| 164 | current = artifact_digest(parsed) |
| 165 | assert stamp is not None |
| 166 | assert stamp.get("artifact_digest") != current |
| 167 | assert ( |
| 168 | author_loop_complete( |
| 169 | "mechanical_only", |
| 170 | stamp_digest=stamp.get("artifact_digest"), |
| 171 | current_digest=current, |
| 172 | ) |
| 173 | is False |
| 174 | ) |
| 175 | decision = plan_next_regen( |
| 176 | roadmap_text=( |
| 177 | "# Roadmap\n\n## Build queue\n\n" |
| 178 | "| Phase | Model | Status | Deliverable |\n" |
| 179 | "| --- | --- | --- | --- |\n" |
| 180 | f"| **AFF-a** | Thinking | **NEXT** | `{AFF_FREEZE_REL}` |\n" |
| 181 | ), |
| 182 | handover_text="## NEXT\n| **ID** | **AFF-a** |\n", |
| 183 | config=config, |
| 184 | repo_root=repo_root, |
| 185 | ) |
| 186 | assert decision.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING |
| 187 | |
| 188 | |
| 189 | def test_hash_consistent_malformed_cannot_authorize(repo_root) -> None: |
| 190 | config = seed_aff_repo(repo_root, adversarial_freeze="require") |
| 191 | digest = write_mechanical_stamp(repo_root) |
| 192 | genesis = build_genesis_entry("2026-01-01T00:00:00Z") |
| 193 | body = aff_body(artifact_digest=digest) |
| 194 | body.pop("reviewer_model") |
| 195 | body["ts"] = "2026-01-01T00:00:01Z" |
| 196 | body["v"] = 1 |
| 197 | entry = dict(body) |
| 198 | entry["prev_hash"] = genesis["entry_hash"] |
| 199 | entry["entry_hash"] = compute_entry_hash(entry) |
| 200 | assert verify_chain([genesis, entry]) == 0 |
| 201 | assert ( |
| 202 | find_latest_adversarial_freeze_verdict( |
| 203 | [genesis, entry], frozen_spec=AFF_FREEZE_REL, artifact_digest=digest |
| 204 | ) |
| 205 | is None |
| 206 | ) |
| 207 | ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 208 | ledger.parent.mkdir(parents=True, exist_ok=True) |
| 209 | ledger.write_text( |
| 210 | serialize_entry(genesis) + "\n" + serialize_entry(entry) + "\n", |
| 211 | encoding="utf-8", |
| 212 | ) |
| 213 | auth = adversarial_authorization_state( |
| 214 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 215 | ) |
| 216 | assert auth.state != "pass" |
| 217 | |
| 218 | |
| 219 | def test_helper_off_under_disabled_honesty_cannot_hold(repo_root) -> None: |
| 220 | config = seed_aff_repo( |
| 221 | repo_root, adversarial_freeze="suggest", honesty_enabled=False |
| 222 | ) |
| 223 | assert aff_hold_bypassed(config) is True |
| 224 | auth = adversarial_authorization_state( |
| 225 | repo_root, repo_root / AFF_FREEZE_REL, config=config |
| 226 | ) |
| 227 | assert auth.state == "off" |
| 228 | gate = build_adversarial_freeze_gate(config, repo_root) |
| 229 | assert gate.skipped is True |
| 230 | |
| 231 | |
| 232 | def test_mode_e_suggest_cannot_soften_integrity(repo_root) -> None: |
| 233 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 234 | digest = write_mechanical_stamp(repo_root) |
| 235 | append_entry( |
| 236 | config=config, |
| 237 | repo_root=repo_root, |
| 238 | options=LedgerAppendOptions( |
| 239 | kind="adversarial_freeze", |
| 240 | body=aff_body(artifact_digest=digest), |
| 241 | ), |
| 242 | ) |
| 243 | ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 244 | text = ledger.read_text(encoding="utf-8") |
| 245 | ledger.write_text(text.replace("thinking-high", "tampered-model", 1), encoding="utf-8") |
| 246 | result = run_honesty_status( |
| 247 | config=config, |
| 248 | repo_root=repo_root, |
| 249 | options=HonestyStatusOptions( |
| 250 | hook=None, |
| 251 | artifact=None, |
| 252 | adversarial_freeze="AFF", |
| 253 | frozen_spec=AFF_FREEZE_REL, |
| 254 | ), |
| 255 | ) |
| 256 | assert result.exit_code == 22 |
| 257 | assert result.exit_code != 0 |
| 258 | |
| 259 | |
| 260 | def test_fail_closed_read_failure_suggest_surfaces_absent(repo_root) -> None: |
| 261 | from unittest.mock import patch |
| 262 | |
| 263 | config = seed_aff_repo(repo_root, adversarial_freeze="suggest") |
| 264 | digest = write_mechanical_stamp(repo_root) |
| 265 | append_entry( |
| 266 | config=config, |
| 267 | repo_root=repo_root, |
| 268 | options=LedgerAppendOptions( |
| 269 | kind="freeze_review", |
| 270 | body={ |
| 271 | "actor_role": "verifier", |
| 272 | "actor_session_id": "frv-1", |
| 273 | "phase_id": "AFF-b", |
| 274 | "frozen_spec": AFF_FREEZE_REL, |
| 275 | "round": 1, |
| 276 | "gate": "substantive", |
| 277 | "freeze_verdict": "pass", |
| 278 | "artifact_digest": digest, |
| 279 | "reviewer_model": "thinking-high", |
| 280 | }, |
| 281 | ), |
| 282 | ) |
| 283 | ledger = repo_root / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl" |
| 284 | ledger.write_text("not-json\n", encoding="utf-8") |
| 285 | art = repo_root / AFF_FREEZE_REL |
| 286 | with patch( |
| 287 | "tools.adversarial_freeze.surface.frv_authorizing_freeze_paths", |
| 288 | return_value=[art], |
| 289 | ): |
| 290 | gate = build_adversarial_freeze_gate( |
| 291 | config, |
| 292 | repo_root, |
| 293 | handover_text="## NEXT\n| **ID** | **AFF-b** |\n", |
| 294 | roadmap_text=( |
| 295 | "# Roadmap\n\n## Build queue\n\n" |
| 296 | "| Phase | Model | Status | Deliverable |\n" |
| 297 | "| --- | --- | --- | --- |\n" |
| 298 | f"| **AFF-b** | Auto | **WIP** | `{AFF_FREEZE_REL}` |\n" |
| 299 | ), |
| 300 | ) |
| 301 | assert gate.skipped is False |
| 302 | assert gate.state == "absent" |
| 303 | assert gate.ok is True |
| 304 | assert gate.message is not None |
| 305 | assert "unreadable" in gate.message |
File History
1 commit
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4
docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit.
Human
4 days ago