aaronrene / overseer-kit public
test_aff_integration.py python
591 lines 19.9 KB
Raw
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4 docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit. Human 4 days ago
1 """Integration tests for AFF ledger append and Mode E honesty-status (§AFF.14)."""
2
3 from __future__ import annotations
4
5 import json
6 from io import StringIO
7 from pathlib import Path
8 from unittest.mock import patch
9
10 import yaml
11
12 from cli.context import CliContext
13 from cli.main import main
14 from cli.output import OutputContext
15 from tests.fixtures.aff import (
16 AFF_FREEZE_REL,
17 aff_body,
18 current_artifact_digest,
19 seed_aff_repo,
20 write_mechanical_stamp,
21 )
22 from tests.support import git_status_runner, make_runner, ok, seed_governance_freshness
23 from tools.adversarial_freeze.surface import WARN_ABSENT, build_adversarial_freeze_gate
24 from tools.honesty.ledger import append_entry, verify_ledger_file
25 from tools.honesty.status import (
26 EXIT_MISSING_ADVERSARIAL_FREEZE,
27 HonestyStatusOptions,
28 run_honesty_status,
29 )
30 from tools.honesty.types import LedgerAppendOptions
31
32
33 def _seed_aff_public_cli_repo(tmp_path: Path, *, mode: str) -> None:
34 """Seed an otherwise-green active Auto slice for public CLI assertions."""
35 _seed_status_fixture(tmp_path, mode=mode)
36 (tmp_path / ".overseer" / "version.lock").write_text(
37 "lock_version: 1\n"
38 "kit_version: 0.1.0\n"
39 "config_version: 1\n"
40 "installed_at: '2026-01-01T00:00:00Z'\n"
41 "synced_at: '2026-01-01T00:00:00Z'\n"
42 "footprint_digest: sha256:0\n"
43 "footprint: []\n",
44 encoding="utf-8",
45 )
46 seed_governance_freshness(tmp_path)
47
48
49 def _run_aff_cli(tmp_path: Path, argv: list[str], *, runner=None) -> tuple[int, str]:
50 stdout = StringIO()
51 with patch("sys.stdout", stdout):
52 code = main(
53 argv,
54 ctx=CliContext.create(
55 cwd=tmp_path,
56 runner=runner or git_status_runner(),
57 output=OutputContext(),
58 ),
59 )
60 return code, stdout.getvalue()
61
62
63 def test_append_aff_pass_findings_blocked_round_trips(repo_root) -> None:
64 config = seed_aff_repo(repo_root, adversarial_freeze="require")
65 digest = write_mechanical_stamp(repo_root)
66 for name in ("aff-pass.json", "aff-findings.json", "aff-blocked.json"):
67 body = aff_body(name, artifact_digest=digest, actor_session_id=f"adv-{name}")
68 assert (
69 append_entry(
70 config=config,
71 repo_root=repo_root,
72 options=LedgerAppendOptions(kind="adversarial_freeze", body=body),
73 ).exit_code
74 == 0
75 )
76 assert verify_ledger_file(config=config, repo_root=repo_root).exit_code == 0
77
78
79 def test_mode_e_require_no_match_exit_40(repo_root) -> None:
80 config = seed_aff_repo(repo_root, adversarial_freeze="require")
81 write_mechanical_stamp(repo_root)
82 result = run_honesty_status(
83 config=config,
84 repo_root=repo_root,
85 options=HonestyStatusOptions(
86 hook=None,
87 artifact=None,
88 adversarial_freeze="AFF",
89 frozen_spec=AFF_FREEZE_REL,
90 ),
91 )
92 assert result.exit_code == EXIT_MISSING_ADVERSARIAL_FREEZE
93 assert result.json_payload.error == "missing_adversarial_freeze"
94 block = result.json_payload.adversarial_freeze
95 assert block is not None
96 assert block["mode"] == "require"
97 assert block["matched_via"] is None
98 assert block["matched_entry_hash"] is None
99
100
101 def test_mode_e_matching_pass_exit_0_exact_block(repo_root) -> None:
102 config = seed_aff_repo(repo_root, adversarial_freeze="require")
103 digest = write_mechanical_stamp(repo_root)
104 body = aff_body(artifact_digest=digest)
105 append_entry(
106 config=config,
107 repo_root=repo_root,
108 options=LedgerAppendOptions(kind="adversarial_freeze", body=body),
109 )
110 result = run_honesty_status(
111 config=config,
112 repo_root=repo_root,
113 options=HonestyStatusOptions(
114 hook=None,
115 artifact=None,
116 adversarial_freeze="AFF",
117 frozen_spec=AFF_FREEZE_REL,
118 producer_session="aff-a-author-session",
119 ),
120 )
121 assert result.exit_code == 0
122 block = result.json_payload.adversarial_freeze
123 assert block is not None
124 assert block["matched_via"] == "pass"
125 assert block["matched_entry_hash"] is not None
126 assert block["frozen_spec"] == AFF_FREEZE_REL
127 assert block["artifact_digest"] == digest
128 assert block["mode"] == "require"
129 assert result.json_payload.independent_second_review is None
130 assert result.json_payload.verification_evidence is None
131
132
133 def test_mode_e_suggest_miss_warns(repo_root) -> None:
134 config = seed_aff_repo(repo_root, adversarial_freeze="suggest")
135 write_mechanical_stamp(repo_root)
136 result = run_honesty_status(
137 config=config,
138 repo_root=repo_root,
139 options=HonestyStatusOptions(
140 hook=None,
141 artifact=None,
142 adversarial_freeze="AFF",
143 frozen_spec=AFF_FREEZE_REL,
144 ),
145 )
146 assert result.exit_code == 0
147 assert "warning:" in result.stderr_extra
148
149
150 def test_mode_e_off_exact_block_no_ledger_read(repo_root) -> None:
151 config = seed_aff_repo(repo_root, adversarial_freeze="off")
152 write_mechanical_stamp(repo_root)
153 result = run_honesty_status(
154 config=config,
155 repo_root=repo_root,
156 options=HonestyStatusOptions(
157 hook=None,
158 artifact=None,
159 adversarial_freeze="AFF",
160 frozen_spec=AFF_FREEZE_REL,
161 ),
162 )
163 assert result.exit_code == 0
164 block = result.json_payload.adversarial_freeze
165 assert block is not None
166 assert block["mode"] == "off"
167 assert block["matched_via"] is None
168 assert block["matched_entry_hash"] is None
169
170
171 def test_mode_e_plus_mode_d_exit_1(repo_root) -> None:
172 config = seed_aff_repo(repo_root)
173 result = run_honesty_status(
174 config=config,
175 repo_root=repo_root,
176 options=HonestyStatusOptions(
177 hook=None,
178 artifact=None,
179 adversarial_freeze="AFF",
180 independent_second_review="ISR-b",
181 ),
182 )
183 assert result.exit_code == 1
184
185
186 def test_aff_mode_e_cli_parser_command_path(repo_root: Path) -> None:
187 """Mode E must be reached through ``ok honesty-status`` argument wiring."""
188 config = seed_aff_repo(repo_root, adversarial_freeze="require")
189 digest = write_mechanical_stamp(repo_root)
190
191 missing_code, missing_stdout = _run_aff_cli(
192 repo_root,
193 [
194 "--json",
195 "honesty-status",
196 "--adversarial-freeze",
197 "AFF",
198 "--frozen-spec",
199 AFF_FREEZE_REL,
200 "--artifact-digest",
201 digest,
202 ],
203 )
204 assert missing_code == EXIT_MISSING_ADVERSARIAL_FREEZE
205 missing = json.loads(missing_stdout)
206 assert missing["error"] == "missing_adversarial_freeze"
207 assert missing["adversarial_freeze"]["matched_via"] is None
208
209 append_entry(
210 config=config,
211 repo_root=repo_root,
212 options=LedgerAppendOptions(
213 kind="adversarial_freeze",
214 body=aff_body(artifact_digest=digest),
215 ),
216 )
217 passed_code, passed_stdout = _run_aff_cli(
218 repo_root,
219 [
220 "--json",
221 "honesty-status",
222 "--adversarial-freeze",
223 "AFF",
224 "--frozen-spec",
225 AFF_FREEZE_REL,
226 "--artifact-digest",
227 digest,
228 ],
229 )
230 assert passed_code == 0
231 assert json.loads(passed_stdout)["adversarial_freeze"]["matched_via"] == "pass"
232
233 usage_code, _ = _run_aff_cli(
234 repo_root,
235 [
236 "honesty-status",
237 "--adversarial-freeze",
238 "AFF",
239 "--verification-evidence",
240 "AFF-b",
241 ],
242 )
243 assert usage_code == 1
244
245
246 def test_mode_e_digest_without_frozen_usage_1(repo_root) -> None:
247 config = seed_aff_repo(repo_root)
248 result = run_honesty_status(
249 config=config,
250 repo_root=repo_root,
251 options=HonestyStatusOptions(
252 hook=None,
253 artifact=None,
254 adversarial_freeze="AFF",
255 artifact_digest="sha256:" + ("a" * 64),
256 ),
257 )
258 assert result.exit_code == 1
259
260
261 def test_mode_e_malformed_digest_usage_1(repo_root) -> None:
262 config = seed_aff_repo(repo_root)
263 result = run_honesty_status(
264 config=config,
265 repo_root=repo_root,
266 options=HonestyStatusOptions(
267 hook=None,
268 artifact=None,
269 adversarial_freeze="AFF",
270 frozen_spec=AFF_FREEZE_REL,
271 artifact_digest="sha256:not-hex",
272 ),
273 )
274 assert result.exit_code == 1
275
276
277 def test_mode_e_digest_mismatch_usage_1(repo_root) -> None:
278 config = seed_aff_repo(repo_root)
279 write_mechanical_stamp(repo_root)
280 result = run_honesty_status(
281 config=config,
282 repo_root=repo_root,
283 options=HonestyStatusOptions(
284 hook=None,
285 artifact=None,
286 adversarial_freeze="AFF",
287 frozen_spec=AFF_FREEZE_REL,
288 artifact_digest="sha256:" + ("b" * 64),
289 ),
290 )
291 assert result.exit_code == 1
292
293
294 def test_mode_e_non_regular_frozen_spec_plus_digest(repo_root) -> None:
295 config = seed_aff_repo(repo_root, adversarial_freeze="suggest")
296 digest = "sha256:" + ("c" * 64)
297 # directory path under repo
298 docs_dir = "docs"
299 result = run_honesty_status(
300 config=config,
301 repo_root=repo_root,
302 options=HonestyStatusOptions(
303 hook=None,
304 artifact=None,
305 adversarial_freeze="AFF",
306 frozen_spec=docs_dir,
307 artifact_digest=digest,
308 ),
309 )
310 # Mode E explicit ledger query — not usage 1, not refusal 4
311 assert result.exit_code in {0, EXIT_MISSING_ADVERSARIAL_FREEZE}
312 assert result.exit_code != 1
313 assert result.exit_code != 4
314 block = result.json_payload.adversarial_freeze
315 assert block is not None
316 assert block["frozen_spec"] == docs_dir
317 assert block["artifact_digest"] == digest
318
319
320 def test_honesty_disabled_exit_4(repo_root) -> None:
321 config = seed_aff_repo(repo_root, honesty_enabled=False)
322 result = run_honesty_status(
323 config=config,
324 repo_root=repo_root,
325 options=HonestyStatusOptions(
326 hook=None,
327 artifact=None,
328 adversarial_freeze="AFF",
329 frozen_spec=AFF_FREEZE_REL,
330 ),
331 )
332 assert result.exit_code == 4
333
334
335 def test_skip_authorizes_under_suggest_not_require(repo_root) -> None:
336 config_s = seed_aff_repo(repo_root, adversarial_freeze="suggest")
337 digest_s = write_mechanical_stamp(repo_root)
338 skip = aff_body("aff-skip.json", artifact_digest=digest_s)
339 append_entry(
340 config=config_s,
341 repo_root=repo_root,
342 options=LedgerAppendOptions(kind="adversarial_freeze", body=skip),
343 )
344 ok = run_honesty_status(
345 config=config_s,
346 repo_root=repo_root,
347 options=HonestyStatusOptions(
348 hook=None,
349 artifact=None,
350 adversarial_freeze="AFF",
351 frozen_spec=AFF_FREEZE_REL,
352 ),
353 )
354 assert ok.exit_code == 0
355 assert ok.json_payload.adversarial_freeze["matched_via"] == "skip"
356
357 # require: same skip is inert → miss
358 cfg_path = repo_root / ".overseer" / "config.yaml"
359 data = yaml.safe_load(cfg_path.read_text(encoding="utf-8"))
360 data["honesty"]["adversarial_freeze"] = "require"
361 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
362 from adapters.config import load_config
363
364 config_r = load_config(cfg_path)
365 miss = run_honesty_status(
366 config=config_r,
367 repo_root=repo_root,
368 options=HonestyStatusOptions(
369 hook=None,
370 artifact=None,
371 adversarial_freeze="AFF",
372 frozen_spec=AFF_FREEZE_REL,
373 ),
374 )
375 assert miss.exit_code == EXIT_MISSING_ADVERSARIAL_FREEZE
376
377
378 def _seed_status_fixture(tmp_path: Path, *, mode: str) -> None:
379 """Seed Auto-slice AFF status fixture without ``ok init`` (sandbox-safe)."""
380 from adapters.config import load_config
381 from tools.honesty.genesis import build_genesis_entry
382 from tools.honesty.ledger_io import serialize_entry
383
384 config = seed_aff_repo(tmp_path, adversarial_freeze=mode)
385 cfg = tmp_path / ".overseer" / "config.yaml"
386 data = yaml.safe_load(cfg.read_text(encoding="utf-8"))
387 data["honesty"]["adversarial_freeze"] = mode
388 cfg.write_text(yaml.safe_dump(data), encoding="utf-8")
389 config = load_config(cfg)
390
391 ledger_dir = tmp_path / ".overseer" / "honesty"
392 ledger_dir.mkdir(parents=True, exist_ok=True)
393 if not (ledger_dir / "VERDICT-LEDGER.jsonl").is_file():
394 genesis = serialize_entry(build_genesis_entry("2026-01-01T00:00:00Z"))
395 (ledger_dir / "VERDICT-LEDGER.jsonl").write_text(genesis + "\n", encoding="utf-8")
396
397 digest = write_mechanical_stamp(tmp_path)
398 append_entry(
399 config=config,
400 repo_root=tmp_path,
401 options=LedgerAppendOptions(
402 kind="freeze_review",
403 body={
404 "actor_role": "verifier",
405 "actor_session_id": "frv-1",
406 "phase_id": "AFF-b",
407 "frozen_spec": AFF_FREEZE_REL,
408 "round": 1,
409 "gate": "substantive",
410 "freeze_verdict": "pass",
411 "artifact_digest": digest,
412 "reviewer_model": "thinking-high",
413 },
414 ),
415 )
416 docs = tmp_path / "docs"
417 (docs / "ROADMAP.md").write_text(
418 "# Roadmap\n\n## Build queue\n\n"
419 "| Phase | Model | Status | Deliverable |\n"
420 "| --- | --- | --- | --- |\n"
421 f"| **AFF-b Adversarial freeze build** | Auto | **WIP** | "
422 f"`{AFF_FREEZE_REL}` |\n",
423 encoding="utf-8",
424 )
425 (docs / "OVERSEER-HANDOVER.md").write_text(
426 "## NEXT SESSION — AFF-b\n\n| | |\n| **ID** | **AFF-b** |\n\n",
427 encoding="utf-8",
428 )
429
430
431 def _aff_governance_runner():
432 return make_runner(
433 {
434 "git rev-parse --abbrev-ref HEAD": ok("main"),
435 "git status --porcelain": ok(""),
436 "git rev-parse origin/main": ok("cafebabe"),
437 "gh pr list --state merged --limit 5 --json number,title,mergeCommit,mergedAt": ok("[]"),
438 }
439 )
440
441
442 def test_aff_public_status_and_governance_require_miss_then_pass(tmp_path: Path) -> None:
443 """Run the public status and governance commands, not just their AFF helper."""
444 _seed_aff_public_cli_repo(tmp_path, mode="require")
445 from adapters.config import load_config
446
447 missing_code, missing_stdout = _run_aff_cli(
448 tmp_path, ["--json", "status", "--exit-code"]
449 )
450 missing = json.loads(missing_stdout)
451 assert missing_code == 2
452 assert missing["adversarial_freeze_gate"]["token"] == "missing_adversarial_freeze"
453 assert missing["adversarial_freeze_gate"]["state"] == "pending"
454
455 governance_code, governance_stdout = _run_aff_cli(
456 tmp_path, ["governance-sync"], runner=_aff_governance_runner()
457 )
458 assert governance_code == 2
459 assert "missing adversarial_freeze" in governance_stdout
460
461 config = load_config(tmp_path / ".overseer" / "config.yaml")
462 append_entry(
463 config=config,
464 repo_root=tmp_path,
465 options=LedgerAppendOptions(
466 kind="adversarial_freeze",
467 body=aff_body(artifact_digest=current_artifact_digest(tmp_path)),
468 ),
469 )
470 passed_code, passed_stdout = _run_aff_cli(
471 tmp_path, ["--json", "status", "--exit-code"]
472 )
473 passed = json.loads(passed_stdout)
474 assert passed_code == 0
475 assert passed["adversarial_freeze_gate"]["ok"] is True
476 assert passed["adversarial_freeze_gate"]["state"] == "pass"
477
478
479 def test_aff_public_governance_sync_suggest_pending_and_absent(tmp_path: Path) -> None:
480 """Public status JSON + governance-sync for suggest pending and absent (§AFF.14)."""
481 _seed_aff_public_cli_repo(tmp_path, mode="suggest")
482
483 pending_status_code, pending_status_stdout = _run_aff_cli(
484 tmp_path, ["--json", "status", "--exit-code"]
485 )
486 pending_status = json.loads(pending_status_stdout)
487 assert pending_status_code == 0
488 assert "adversarial_freeze_gate" in pending_status
489 assert pending_status["adversarial_freeze_gate"]["state"] == "pending"
490 assert pending_status["adversarial_freeze_gate"]["ok"] is True
491 assert pending_status["adversarial_freeze_gate"]["mode"] == "suggest"
492 assert (
493 "warning: no authorizing adversarial_freeze entry for active Auto slice"
494 in pending_status["warnings"]
495 )
496
497 pending_code, pending_stdout = _run_aff_cli(
498 tmp_path, ["governance-sync"], runner=_aff_governance_runner()
499 )
500 assert pending_code == 0
501 assert "warning: no authorizing adversarial_freeze entry for active Auto slice" in pending_stdout
502
503 ledger = tmp_path / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl"
504 ledger.write_text("not-json\n", encoding="utf-8")
505 with patch(
506 "tools.adversarial_freeze.surface.frv_authorizing_freeze_paths",
507 return_value=[tmp_path / AFF_FREEZE_REL],
508 ):
509 absent_status_code, absent_status_stdout = _run_aff_cli(
510 tmp_path, ["--json", "status", "--exit-code"]
511 )
512 absent_code, absent_stdout = _run_aff_cli(
513 tmp_path, ["governance-sync"], runner=_aff_governance_runner()
514 )
515 absent_status = json.loads(absent_status_stdout)
516 assert absent_status_code == 0
517 assert "adversarial_freeze_gate" in absent_status
518 assert absent_status["adversarial_freeze_gate"]["state"] == "absent"
519 assert absent_status["adversarial_freeze_gate"]["ok"] is True
520 assert absent_status["adversarial_freeze_gate"]["mode"] == "suggest"
521 assert absent_status["adversarial_freeze_gate"]["matched"] is False
522 assert WARN_ABSENT in absent_status["warnings"]
523 assert absent_code == 0
524 assert WARN_ABSENT in absent_stdout
525
526
527 def test_mixed_candidate_aggregate_parity(repo_root) -> None:
528 from tools.adversarial_freeze import aggregate_adversarial_authorization_states
529 from tools.governance_hygiene.next_regen import decide_split_emission, _apply_adversarial_freeze_hold
530 from tools.governance_hygiene.types import QueueRow
531
532 config = seed_aff_repo(repo_root, adversarial_freeze="require")
533 digest = write_mechanical_stamp(repo_root)
534 append_entry(
535 config=config,
536 repo_root=repo_root,
537 options=LedgerAppendOptions(
538 kind="freeze_review",
539 body={
540 "actor_role": "verifier",
541 "actor_session_id": "frv-1",
542 "phase_id": "AFF-b",
543 "frozen_spec": AFF_FREEZE_REL,
544 "round": 1,
545 "gate": "substantive",
546 "freeze_verdict": "pass",
547 "artifact_digest": digest,
548 "reviewer_model": "thinking-high",
549 },
550 ),
551 )
552 # no AFF → pending hold on Auto
553 auto_row = QueueRow(
554 phase_label="**AFF-b**",
555 model="Auto",
556 status="**NEXT**",
557 deliverable=f"`{AFF_FREEZE_REL}`",
558 raw_line="",
559 )
560 emit, reason, is_b, advisory = decide_split_emission(auto_row, repo_root, config=config)
561 assert emit == "Auto"
562 emit2, is_b2, adv2 = _apply_adversarial_freeze_hold(
563 row=auto_row,
564 repo_root=repo_root,
565 config=config,
566 emit_model=emit,
567 is_step_b=is_b,
568 advisory=advisory,
569 step_id="AFF-b",
570 )
571 assert emit2 == "Thinking"
572 assert adv2 == "adversarial_freeze_pending"
573
574 gate = build_adversarial_freeze_gate(
575 config,
576 repo_root,
577 handover_text="## NEXT\n| **ID** | **AFF-b** |\n",
578 roadmap_text=(
579 "# Roadmap\n\n## Build queue\n\n"
580 "| Phase | Model | Status | Deliverable |\n"
581 "| --- | --- | --- | --- |\n"
582 f"| **AFF-b** | Auto | **WIP** | `{AFF_FREEZE_REL}` |\n"
583 ),
584 )
585 assert gate.state == "pending"
586 assert gate.ok is False
587
588 # pass then aggregate ranks
589 assert aggregate_adversarial_authorization_states(["pass", "pending"]) == "pending"
590 assert aggregate_adversarial_authorization_states(["pass", "absent"]) == "absent"
591 assert aggregate_adversarial_authorization_states(["pass", "skipped"]) == "skipped"
File History 1 commit
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4 docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit. Human 4 days ago