aaronrene / overseer-kit public
test_aff_e2e.py python
551 lines 20.8 KB
Raw
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4 docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit. Human 3 days ago
1 """E2E tests for AFF Trigger A / Trigger B (§AFF.14)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6
7 import yaml
8
9 from adapters.config import load_config
10 from tests.fixtures.aff import (
11 AFF_FREEZE_REL,
12 aff_body,
13 current_artifact_digest,
14 seed_aff_repo,
15 write_mechanical_stamp,
16 )
17 from tools.adversarial_freeze import adversarial_authorization_state
18 from tools.freeze_reviewer.artifact import artifact_digest, parse_artifact
19 from tools.governance_hygiene.next_regen import (
20 ADVISORY_ADVERSARIAL_FREEZE_PENDING,
21 ADVISORY_ADVERSARIAL_FREEZE_SKIP,
22 ADVISORY_MECHANICAL_ONLY,
23 decide_split_emission,
24 plan_next_regen,
25 render_next_session,
26 render_paste_ready,
27 )
28 from tools.governance_hygiene.types import QueueRow
29 from tools.honesty.ledger import append_entry
30 from tools.honesty.types import LedgerAppendOptions
31
32 DELIVERABLE = f"`{AFF_FREEZE_REL}`"
33
34
35 def _roadmap_two_row(*, a_status: str = "**NEXT**", b_status: str = "QUEUED") -> str:
36 return (
37 "# Roadmap\n\n## Build queue\n\n"
38 "| Phase | Model | Status | Deliverable |\n"
39 "| --- | --- | --- | --- |\n"
40 f"| **AFF-a Adversarial freeze author** | Thinking | {a_status} | {DELIVERABLE} |\n"
41 f"| **AFF-b Adversarial freeze build** | Auto | {b_status} | {DELIVERABLE} |\n"
42 )
43
44
45 def _roadmap_split(*, status: str = "**NEXT**") -> str:
46 return (
47 "# Roadmap\n\n## Build queue\n\n"
48 "| Phase | Model | Status | Deliverable |\n"
49 "| --- | --- | --- | --- |\n"
50 f"| **AFF Adversarial freeze build** | Thinking → Auto | {status} | {DELIVERABLE} |\n"
51 )
52
53
54 def _handover(phase: str = "AFF-a") -> str:
55 return f"## NEXT SESSION — {phase}\n\n| | |\n| **ID** | **{phase}** |\n\n"
56
57
58 def _append_frv(config, repo_root: Path, *, phase_id: str, digest: str) -> None:
59 append_entry(
60 config=config,
61 repo_root=repo_root,
62 options=LedgerAppendOptions(
63 kind="freeze_review",
64 body={
65 "actor_role": "verifier",
66 "actor_session_id": "frv-chat",
67 "phase_id": phase_id,
68 "frozen_spec": AFF_FREEZE_REL,
69 "round": 1,
70 "gate": "substantive",
71 "freeze_verdict": "pass",
72 "artifact_digest": digest,
73 "reviewer_model": "thinking-high",
74 },
75 ),
76 )
77
78
79 def _append_aff_pass(config, repo_root: Path, *, digest: str, phase_id: str = "AFF") -> None:
80 append_entry(
81 config=config,
82 repo_root=repo_root,
83 options=LedgerAppendOptions(
84 kind="adversarial_freeze",
85 body=aff_body(artifact_digest=digest, phase_id=phase_id),
86 ),
87 )
88
89
90 def test_aff_trigger_b_paste_has_frozen_attack_contract(repo_root: Path) -> None:
91 config = seed_aff_repo(repo_root, adversarial_freeze="suggest")
92 write_mechanical_stamp(repo_root)
93 decision = plan_next_regen(
94 roadmap_text=_roadmap_two_row(),
95 handover_text=_handover("AFF-a"),
96 config=config,
97 repo_root=repo_root,
98 )
99 assert decision.emit_model == "Thinking"
100 assert decision.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
101 paste = render_paste_ready(decision=decision, config=config)
102 assert "adversarial" in paste.lower() or "attack" in paste.lower()
103 assert "Model: Thinking" in paste
104 assert "Posture: attack — try to kill the freeze; do not rubber-stamp close" in paste
105 assert "try to kill the freeze" in paste
106 assert "different chat from the author" in paste
107 assert "If this is the author session, stop." in paste
108 assert "Prefer a different model than the author" in paste
109 assert AFF_FREEZE_REL in paste
110 assert "path:line" in paste
111 assert "aff_posture: attack" in paste
112 assert "<AUTHOR_PRODUCER_SESSION_NONCE>" in paste
113 assert "On findings/blocked: append that negative verdict" in paste
114 assert "never omit a negative append" in paste
115 assert "Under suggest, operator skip is a separate owner append" in paste
116 # §AFF.7.2 exact no-key contract — reversed "holds a key" must fail.
117 assert "The kit performs no model call and holds no key." in paste
118 assert "The kit performs no model call and holds a key." not in paste
119 assert "No merge to main." in paste
120
121 # §AFF.7.2 pass instruction must emit the complete §AFF.5.2 field set
122 # from the real render_paste_ready path (not a paraphrased stand-in).
123 assert "final binding operation" in paste
124 assert (
125 "On pass: §AFF.5.2 fields only — actor_role: verifier,\n"
126 " actor_session_id: <THIS_CHAT_SESSION_ID>, phase_id, round,\n"
127 " reviewer_model, frozen_spec, artifact_digest,\n"
128 " aff_verdict: pass, aff_posture: attack, and\n"
129 " producer_session_id: <AUTHOR_PRODUCER_SESSION_NONCE>"
130 ) in paste
131
132 # §AFF.7.4's order is part of the freeze contract, not prose decoration.
133 assert paste.index("1. Finish the review") < paste.index("2. Write the adversarial round")
134 assert paste.index("2. Write the adversarial round") < paste.index("3. Run ok review")
135 assert paste.index("3. Run ok review") < paste.index("4. If Auto requires FRV")
136 assert paste.index("4. If Auto requires FRV") < paste.index("5. Append adversarial_freeze")
137 assert paste.index("5. Append adversarial_freeze") < paste.index("6. Do not edit")
138
139 # §AFF.7.2 also freezes render_next_session bytes (not only the paste fence).
140 next_session = render_next_session(
141 decision=decision,
142 roadmap_text=_roadmap_two_row(),
143 config=config,
144 sync_date="2026-09-20",
145 )
146 assert "**Model:** Thinking" in next_session
147 assert "### THE ONE NEXT STEP — **Model: Thinking**" in next_session
148 assert (
149 "Adversarial freeze review (attack posture) — different chat from "
150 "the author; try to kill the freeze before Auto may start."
151 ) in next_session
152 assert "Auto build" not in next_session
153
154
155 def test_aff_thinking_to_auto_split_holds_and_releases_by_digest(repo_root: Path) -> None:
156 """Exercise §AFF.14's split convention through Trigger B then Trigger A."""
157 config = seed_aff_repo(repo_root, adversarial_freeze="require")
158 digest = write_mechanical_stamp(repo_root)
159
160 # The open split emits its Thinking (-a) side, but a fresh stamp makes
161 # Trigger B replace the author paste with the adversarial one.
162 trigger_b = plan_next_regen(
163 roadmap_text=_roadmap_split(),
164 handover_text=_handover("AFF-a"),
165 config=config,
166 repo_root=repo_root,
167 )
168 assert trigger_b.emit_model == "Thinking"
169 assert trigger_b.step_label == "AFF-a"
170 assert trigger_b.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
171
172 # A digest-bound AFF pass releases Trigger B without promoting the still
173 # open Thinking side to Auto.
174 _append_aff_pass(config, repo_root, digest=digest, phase_id="AFF")
175 released_b = plan_next_regen(
176 roadmap_text=_roadmap_split(),
177 handover_text=_handover("AFF-a"),
178 config=config,
179 repo_root=repo_root,
180 )
181 assert released_b.emit_model == "Thinking"
182 assert released_b.step_label == "AFF-a"
183 assert released_b.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
184
185 # A later negative verdict revokes that pass. Once FRV makes the split's
186 # Auto (-b) side eligible, Trigger A must hold until a fresh matching pass.
187 append_entry(
188 config=config,
189 repo_root=repo_root,
190 options=LedgerAppendOptions(
191 kind="adversarial_freeze",
192 body=aff_body("aff-findings.json", artifact_digest=digest, phase_id="AFF"),
193 ),
194 )
195 _append_frv(config, repo_root, phase_id="AFF", digest=digest)
196 trigger_a = plan_next_regen(
197 roadmap_text=_roadmap_split(),
198 handover_text=_handover("AFF-b"),
199 config=config,
200 repo_root=repo_root,
201 )
202 assert trigger_a.emit_model == "Thinking"
203 assert trigger_a.step_label == "AFF-b"
204 assert trigger_a.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
205
206 _append_aff_pass(config, repo_root, digest=digest, phase_id="AFF")
207 released_a = plan_next_regen(
208 roadmap_text=_roadmap_split(),
209 handover_text=_handover("AFF-b"),
210 config=config,
211 repo_root=repo_root,
212 )
213 assert released_a.emit_model == "Auto"
214 assert released_a.step_label == "AFF-b"
215 assert released_a.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
216
217
218 def test_post_stamp_mutation_returns_to_author_paste(repo_root: Path) -> None:
219 config = seed_aff_repo(repo_root, adversarial_freeze="suggest")
220 write_mechanical_stamp(repo_root)
221 art = repo_root / AFF_FREEZE_REL
222 # mutate outside review_stamp
223 text = art.read_text(encoding="utf-8")
224 art.write_text(text + "\n# byte-edit\n", encoding="utf-8")
225
226 decision = plan_next_regen(
227 roadmap_text=_roadmap_two_row(),
228 handover_text=_handover("AFF-a"),
229 config=config,
230 repo_root=repo_root,
231 )
232 assert decision.emit_model == "Thinking"
233 assert decision.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
234 # FRV mechanical_only or plain Thinking — not adversarial hold
235 assert decision.advisory in {ADVISORY_MECHANICAL_ONLY, None} or (
236 decision.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
237 )
238
239 # Auto row must not emit Auto (FRV still requires substantive)
240 auto_row = QueueRow(
241 phase_label="**AFF-b**",
242 model="Auto",
243 status="**NEXT**",
244 deliverable=DELIVERABLE,
245 raw_line="",
246 )
247 emit, reason, _, _ = decide_split_emission(auto_row, repo_root, config=config)
248 assert emit is None
249 assert reason == "freeze_not_substantive"
250
251 # restamp → Trigger B holds again
252 write_mechanical_stamp(repo_root)
253 decision2 = plan_next_regen(
254 roadmap_text=_roadmap_two_row(),
255 handover_text=_handover("AFF-a"),
256 config=config,
257 repo_root=repo_root,
258 )
259 assert decision2.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
260
261
262 def test_aff_pass_phase_id_aff_releases_trigger_b_and_a(repo_root: Path) -> None:
263 config = seed_aff_repo(repo_root, adversarial_freeze="suggest")
264 digest = write_mechanical_stamp(repo_root)
265 _append_aff_pass(config, repo_root, digest=digest, phase_id="AFF")
266
267 # Trigger B released → ordinary Thinking (mechanical_only advisory ok)
268 decision = plan_next_regen(
269 roadmap_text=_roadmap_two_row(),
270 handover_text=_handover("AFF-a"),
271 config=config,
272 repo_root=repo_root,
273 )
274 assert decision.emit_model == "Thinking"
275 assert decision.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
276
277 # Trigger A: open AFF-b + FRV → same path+digest pass clears Auto
278 digest2 = write_mechanical_stamp(repo_root) # may change digest — rebind
279 # after restamp digest changes; re-append AFF for new digest + FRV
280 _append_frv(config, repo_root, phase_id="AFF-b", digest=digest2)
281 # old AFF won't match new digest — append pass for new digest with phase AFF
282 _append_aff_pass(config, repo_root, digest=digest2, phase_id="AFF")
283
284 roadmap_b = _roadmap_two_row(a_status="**DONE**", b_status="**NEXT**")
285 decision_b = plan_next_regen(
286 roadmap_text=roadmap_b,
287 handover_text=_handover("AFF-b"),
288 config=config,
289 repo_root=repo_root,
290 )
291 assert decision_b.emit_model == "Auto"
292 assert decision_b.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
293
294
295 def test_section_aff_7_4_transaction(repo_root: Path) -> None:
296 config = seed_aff_repo(repo_root, adversarial_freeze="require")
297 digest = write_mechanical_stamp(repo_root)
298 _append_frv(config, repo_root, phase_id="AFF-b", digest=digest)
299 _append_aff_pass(config, repo_root, digest=digest, phase_id="AFF")
300
301 roadmap_b = _roadmap_two_row(a_status="**DONE**", b_status="**NEXT**")
302 decision = plan_next_regen(
303 roadmap_text=roadmap_b,
304 handover_text=_handover("AFF-b"),
305 config=config,
306 repo_root=repo_root,
307 )
308 assert decision.emit_model == "Auto"
309
310 # Review-record row (prose) invalidates digest — simulate by body edit
311 art = repo_root / AFF_FREEZE_REL
312 art.write_text(art.read_text(encoding="utf-8") + "\n| AFF-ADV-rX | pass |\n", encoding="utf-8")
313 # restamp alone does not restore Auto (AFF/FRV digest stale)
314 new_digest = write_mechanical_stamp(repo_root)
315 decision2 = plan_next_regen(
316 roadmap_text=roadmap_b,
317 handover_text=_handover("AFF-b"),
318 config=config,
319 repo_root=repo_root,
320 )
321 assert decision2.emit_model != "Auto" or decision2.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
322
323 # rebind FRV alone still leaves AFF pending
324 _append_frv(config, repo_root, phase_id="AFF-b", digest=new_digest)
325 decision3 = plan_next_regen(
326 roadmap_text=roadmap_b,
327 handover_text=_handover("AFF-b"),
328 config=config,
329 repo_root=repo_root,
330 )
331 assert decision3.emit_model == "Thinking"
332 assert decision3.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
333
334 # AFF last restores both
335 _append_aff_pass(config, repo_root, digest=new_digest, phase_id="AFF")
336 decision4 = plan_next_regen(
337 roadmap_text=roadmap_b,
338 handover_text=_handover("AFF-b"),
339 config=config,
340 repo_root=repo_root,
341 )
342 assert decision4.emit_model == "Auto"
343
344 # one-byte later edit withdraws Auto
345 art.write_text(art.read_text(encoding="utf-8") + "x", encoding="utf-8")
346 decision5 = plan_next_regen(
347 roadmap_text=roadmap_b,
348 handover_text=_handover("AFF-b"),
349 config=config,
350 repo_root=repo_root,
351 )
352 assert decision5.emit_model != "Auto"
353
354
355 def test_suggest_skip_emits_auto_with_advisory(repo_root: Path) -> None:
356 config = seed_aff_repo(repo_root, adversarial_freeze="suggest")
357 digest = write_mechanical_stamp(repo_root)
358 _append_frv(config, repo_root, phase_id="AFF-b", digest=digest)
359 append_entry(
360 config=config,
361 repo_root=repo_root,
362 options=LedgerAppendOptions(
363 kind="adversarial_freeze",
364 body=aff_body("aff-skip.json", artifact_digest=digest),
365 ),
366 )
367 roadmap_b = _roadmap_two_row(a_status="**DONE**", b_status="**NEXT**")
368 decision = plan_next_regen(
369 roadmap_text=roadmap_b,
370 handover_text=_handover("AFF-b"),
371 config=config,
372 repo_root=repo_root,
373 )
374 assert decision.emit_model == "Auto"
375 assert decision.advisory == ADVISORY_ADVERSARIAL_FREEZE_SKIP
376
377
378 def test_off_emits_auto_immediately_after_frv(repo_root: Path) -> None:
379 config = seed_aff_repo(repo_root, adversarial_freeze="off")
380 digest = write_mechanical_stamp(repo_root)
381 _append_frv(config, repo_root, phase_id="AFF-b", digest=digest)
382 roadmap_b = _roadmap_two_row(a_status="**DONE**", b_status="**NEXT**")
383 decision = plan_next_regen(
384 roadmap_text=roadmap_b,
385 handover_text=_handover("AFF-b"),
386 config=config,
387 repo_root=repo_root,
388 )
389 assert decision.emit_model == "Auto"
390 assert decision.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
391
392
393 def _disabled_honesty_loops(repo_root: Path, *, explicit_suggest: bool) -> None:
394 if explicit_suggest:
395 config = seed_aff_repo(
396 repo_root, adversarial_freeze="suggest", honesty_enabled=False
397 )
398 else:
399 config = seed_aff_repo(
400 repo_root,
401 adversarial_freeze=None,
402 omit_adversarial_freeze_key=True,
403 honesty_enabled=False,
404 human_escalation=["security"],
405 )
406 assert config.honesty.enabled is False
407
408 write_mechanical_stamp(repo_root)
409 # Trigger B: author freeze paste, not adversarial_freeze_pending
410 decision = plan_next_regen(
411 roadmap_text=_roadmap_two_row(),
412 handover_text=_handover("AFF-a"),
413 config=config,
414 repo_root=repo_root,
415 )
416 assert decision.emit_model == "Thinking"
417 assert decision.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
418
419 digest = current_artifact_digest(repo_root)
420 # Trigger A with freeze candidates, no FRV → freeze_not_substantive (not AFF)
421 auto_row = QueueRow(
422 phase_label="**AFF-b**",
423 model="Auto",
424 status="**NEXT**",
425 deliverable=DELIVERABLE,
426 raw_line="",
427 )
428 emit, reason, _, adv = decide_split_emission(auto_row, repo_root, config=config)
429 assert emit is None
430 assert reason == "freeze_not_substantive"
431
432 # with FRV substantive — Auto (AFF bypassed)
433 _append_frv(config, repo_root, phase_id="AFF-b", digest=digest)
434 # honesty disabled → append may refuse; write FRV by temporarily enabling
435 # Actually append_entry refuses when honesty disabled. Seed FRV while enabled then disable.
436 pass
437
438
439 def test_disabled_honesty_explicit_and_derived_suggest(repo_root: Path) -> None:
440 # Append FRV while honesty enabled, then disable (ledger match skipped when off).
441 config = seed_aff_repo(repo_root, adversarial_freeze="suggest", honesty_enabled=True)
442 write_mechanical_stamp(repo_root)
443
444 cfg_path = repo_root / ".overseer" / "config.yaml"
445 data = yaml.safe_load(cfg_path.read_text(encoding="utf-8"))
446 data["honesty"]["enabled"] = False
447 data["honesty"]["adversarial_freeze"] = "suggest"
448 if "modules" in data and isinstance(data["modules"], dict):
449 data["modules"].setdefault("honesty", {})["enabled"] = False
450 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
451 config = load_config(cfg_path)
452
453 # Trigger B: author freeze paste, not adversarial_freeze_pending
454 decision_b = plan_next_regen(
455 roadmap_text=_roadmap_two_row(),
456 handover_text=_handover("AFF-a"),
457 config=config,
458 repo_root=repo_root,
459 )
460 assert decision_b.emit_model == "Thinking"
461 assert decision_b.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
462
463 # Trigger A with freeze candidates: FRV freeze_not_substantive (not AFF hold)
464 # (honesty off → ledger FRV ignored; stamp alone is mechanical_only)
465 auto_row = QueueRow(
466 phase_label="**AFF-b**",
467 model="Auto",
468 status="**NEXT**",
469 deliverable=DELIVERABLE,
470 raw_line="",
471 )
472 emit, reason, _, adv = decide_split_emission(auto_row, repo_root, config=config)
473 assert emit is None
474 assert reason == "freeze_not_substantive"
475 assert adv != ADVISORY_ADVERSARIAL_FREEZE_PENDING
476
477 # Trigger A with no freeze candidates → Auto
478 no_cand = QueueRow(
479 phase_label="**OTHER-b**",
480 model="Auto",
481 status="**NEXT**",
482 deliverable="no freeze cite",
483 raw_line="",
484 )
485 emit2, reason2, _, _ = decide_split_emission(no_cand, repo_root, config=config)
486 assert emit2 == "Auto"
487 assert reason2 is None
488
489 from tools.adversarial_freeze import build_adversarial_freeze_gate
490
491 assert build_adversarial_freeze_gate(config, repo_root).skipped is True
492
493 # derived-suggest shape (key absent + security) + disabled
494 data["honesty"].pop("adversarial_freeze", None)
495 data["honesty"]["enabled"] = False
496 if "modules" in data and isinstance(data["modules"], dict):
497 data["modules"].setdefault("honesty", {})["enabled"] = False
498 data["freeze_contract"]["human_escalation"] = ["security"]
499 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
500 config2 = load_config(cfg_path)
501 decision2 = plan_next_regen(
502 roadmap_text=_roadmap_two_row(),
503 handover_text=_handover("AFF-a"),
504 config=config2,
505 repo_root=repo_root,
506 )
507 assert decision2.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
508 assert build_adversarial_freeze_gate(config2, repo_root).skipped is True
509 emit3, reason3, _, _ = decide_split_emission(no_cand, repo_root, config=config2)
510 assert emit3 == "Auto"
511
512
513 def test_operator_plus_auto_unchanged(repo_root: Path) -> None:
514 config = seed_aff_repo(repo_root, adversarial_freeze="require")
515 write_mechanical_stamp(repo_root)
516 row = QueueRow(
517 phase_label="**LAND**",
518 model="Operator + Auto",
519 status="**NEXT**",
520 deliverable=DELIVERABLE,
521 raw_line="",
522 )
523 emit, reason, is_b, advisory = decide_split_emission(row, repo_root, config=config)
524 assert emit == "Operator + Auto"
525 assert reason is None
526
527
528 def test_muse_regime_trigger_loop(repo_root: Path) -> None:
529 config = seed_aff_repo(
530 repo_root,
531 adversarial_freeze="suggest",
532 regime_config="config-muse-git-mirror.yaml",
533 )
534 digest = write_mechanical_stamp(repo_root)
535 decision = plan_next_regen(
536 roadmap_text=_roadmap_two_row(),
537 handover_text=_handover("AFF-a"),
538 config=config,
539 repo_root=repo_root,
540 )
541 assert decision.advisory == ADVISORY_ADVERSARIAL_FREEZE_PENDING
542 _append_aff_pass(config, repo_root, digest=digest, phase_id="AFF")
543 decision2 = plan_next_regen(
544 roadmap_text=_roadmap_two_row(),
545 handover_text=_handover("AFF-a"),
546 config=config,
547 repo_root=repo_root,
548 )
549 assert decision2.advisory != ADVISORY_ADVERSARIAL_FREEZE_PENDING
550 auth = adversarial_authorization_state(repo_root, repo_root / AFF_FREEZE_REL, config=config)
551 assert auth.state == "pass"
File History 1 commit
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4 docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit. Human 3 days ago