test_frv_integration.py
python
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4
docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit.
Human
3 days ago
| 1 | """Integration tests for FRV (§FRV.12).""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | from pathlib import Path |
| 7 | |
| 8 | import yaml |
| 9 | |
| 10 | from cli.kit_root import kit_root |
| 11 | from tests.support import ( |
| 12 | FIXTURES, |
| 13 | git_status_runner, |
| 14 | pass_provider_factory, |
| 15 | run_cli, |
| 16 | seed_freeze_repo, |
| 17 | seed_honesty_repo, |
| 18 | write_config, |
| 19 | ) |
| 20 | from tools.freeze_reviewer.artifact import artifact_digest, extract_existing_stamp, parse_artifact |
| 21 | from tools.freeze_reviewer.engine import EXIT_STAMP_ESCALATION_REFUSED |
| 22 | from tools.governance_gates.scan import scan_governance_gates |
| 23 | from tools.governance_hygiene.next_regen import ( |
| 24 | decide_split_emission, |
| 25 | format_change_log_fragment, |
| 26 | format_next_regen_token, |
| 27 | plan_next_regen, |
| 28 | ) |
| 29 | from tools.governance_hygiene.types import QueueRow |
| 30 | from adapters.config import load_config |
| 31 | from tools.honesty.ledger import append_entry, verify_ledger_file |
| 32 | from tools.honesty.types import LedgerAppendOptions |
| 33 | |
| 34 | |
| 35 | def _enable_freeze_and_honesty(repo: Path) -> None: |
| 36 | seed_honesty_repo(repo) |
| 37 | cfg = repo / ".overseer" / "config.yaml" |
| 38 | data = yaml.safe_load(cfg.read_text(encoding="utf-8")) |
| 39 | data["freeze_contract"] = { |
| 40 | "enabled": True, |
| 41 | "reviewer": "agent", |
| 42 | "human_escalation": ["security"], |
| 43 | } |
| 44 | cfg.write_text(yaml.safe_dump(data), encoding="utf-8") |
| 45 | |
| 46 | |
| 47 | def test_review_writes_fourteen_key_stamp(tmp_path: Path) -> None: |
| 48 | artifact = seed_freeze_repo(tmp_path) |
| 49 | rel = artifact.relative_to(tmp_path).as_posix() |
| 50 | code = run_cli( |
| 51 | ["review", "--freeze", rel], |
| 52 | cwd=tmp_path, |
| 53 | runner=git_status_runner(), |
| 54 | kit=kit_root(), |
| 55 | review_provider_factory=pass_provider_factory(), |
| 56 | ) |
| 57 | assert code == 0 |
| 58 | stamp = extract_existing_stamp(parse_artifact(artifact, rel_path=rel)) |
| 59 | assert stamp is not None |
| 60 | assert stamp.get("gate") == "mechanical" |
| 61 | assert "verdict" not in stamp |
| 62 | assert stamp.get("mechanical_verdict") == "pass" |
| 63 | assert stamp.get("reviewer_model") is None |
| 64 | assert len([k for k in stamp if k != "verdict"]) >= 14 |
| 65 | |
| 66 | |
| 67 | def test_rerun_idempotent(tmp_path: Path) -> None: |
| 68 | artifact = seed_freeze_repo(tmp_path) |
| 69 | rel = artifact.relative_to(tmp_path).as_posix() |
| 70 | run_cli( |
| 71 | ["review", "--freeze", rel], |
| 72 | cwd=tmp_path, |
| 73 | runner=git_status_runner(), |
| 74 | kit=kit_root(), |
| 75 | review_provider_factory=pass_provider_factory(), |
| 76 | ) |
| 77 | first = artifact.read_bytes() |
| 78 | stamp1 = extract_existing_stamp(parse_artifact(artifact, rel_path=rel)) |
| 79 | code = run_cli( |
| 80 | ["review", "--freeze", rel], |
| 81 | cwd=tmp_path, |
| 82 | runner=git_status_runner(), |
| 83 | kit=kit_root(), |
| 84 | review_provider_factory=pass_provider_factory(), |
| 85 | ) |
| 86 | assert code == 0 |
| 87 | assert artifact.read_bytes() == first |
| 88 | stamp2 = extract_existing_stamp(parse_artifact(artifact, rel_path=rel)) |
| 89 | assert stamp1["reviewed_at"] == stamp2["reviewed_at"] |
| 90 | |
| 91 | |
| 92 | def test_escalation_refused_exit_39(tmp_path: Path) -> None: |
| 93 | artifact = seed_freeze_repo(tmp_path) |
| 94 | rel = artifact.relative_to(tmp_path).as_posix() |
| 95 | run_cli( |
| 96 | ["review", "--freeze", rel], |
| 97 | cwd=tmp_path, |
| 98 | runner=git_status_runner(), |
| 99 | kit=kit_root(), |
| 100 | review_provider_factory=pass_provider_factory(), |
| 101 | ) |
| 102 | text = artifact.read_text(encoding="utf-8") |
| 103 | text = text.replace("mechanical_verdict: pass", "mechanical_verdict: findings") |
| 104 | artifact.write_text(text, encoding="utf-8") |
| 105 | before = artifact.read_bytes() |
| 106 | code = run_cli( |
| 107 | ["review", "--freeze", rel, "--json"], |
| 108 | cwd=tmp_path, |
| 109 | runner=git_status_runner(), |
| 110 | kit=kit_root(), |
| 111 | review_provider_factory=pass_provider_factory(), |
| 112 | json_mode=True, |
| 113 | ) |
| 114 | assert code == EXIT_STAMP_ESCALATION_REFUSED |
| 115 | assert artifact.read_bytes() == before |
| 116 | |
| 117 | |
| 118 | def test_escalation_dry_run_and_no_stamp_exit_0(tmp_path: Path) -> None: |
| 119 | artifact = seed_freeze_repo(tmp_path) |
| 120 | rel = artifact.relative_to(tmp_path).as_posix() |
| 121 | run_cli( |
| 122 | ["review", "--freeze", rel], |
| 123 | cwd=tmp_path, |
| 124 | runner=git_status_runner(), |
| 125 | kit=kit_root(), |
| 126 | review_provider_factory=pass_provider_factory(), |
| 127 | ) |
| 128 | text = artifact.read_text(encoding="utf-8") |
| 129 | artifact.write_text(text.replace("mechanical_verdict: pass", "mechanical_verdict: findings"), encoding="utf-8") |
| 130 | for flags in (["--dry-run"], ["--no-stamp"], ["--dry-run", "--no-stamp"], []): |
| 131 | if not flags: |
| 132 | continue |
| 133 | code = run_cli( |
| 134 | ["review", "--freeze", rel, *flags, "--json"], |
| 135 | cwd=tmp_path, |
| 136 | runner=git_status_runner(), |
| 137 | kit=kit_root(), |
| 138 | review_provider_factory=pass_provider_factory(), |
| 139 | json_mode=True, |
| 140 | ) |
| 141 | assert code == 0, flags |
| 142 | |
| 143 | |
| 144 | def test_override_non_pass_stamp(tmp_path: Path) -> None: |
| 145 | artifact = seed_freeze_repo(tmp_path) |
| 146 | rel = artifact.relative_to(tmp_path).as_posix() |
| 147 | run_cli( |
| 148 | ["review", "--freeze", rel], |
| 149 | cwd=tmp_path, |
| 150 | runner=git_status_runner(), |
| 151 | kit=kit_root(), |
| 152 | review_provider_factory=pass_provider_factory(), |
| 153 | ) |
| 154 | text = artifact.read_text(encoding="utf-8") |
| 155 | artifact.write_text(text.replace("mechanical_verdict: pass", "mechanical_verdict: findings"), encoding="utf-8") |
| 156 | code = run_cli( |
| 157 | ["review", "--freeze", rel, "--override-non-pass-stamp"], |
| 158 | cwd=tmp_path, |
| 159 | runner=git_status_runner(), |
| 160 | kit=kit_root(), |
| 161 | review_provider_factory=pass_provider_factory(), |
| 162 | ) |
| 163 | assert code == 0 |
| 164 | stamp = extract_existing_stamp(parse_artifact(artifact, rel_path=rel)) |
| 165 | assert stamp["mechanical_verdict"] == "pass" |
| 166 | assert stamp["override_applied"] is True |
| 167 | |
| 168 | |
| 169 | def test_forged_substantive_normalized(tmp_path: Path) -> None: |
| 170 | artifact = seed_freeze_repo(tmp_path) |
| 171 | rel = artifact.relative_to(tmp_path).as_posix() |
| 172 | run_cli( |
| 173 | ["review", "--freeze", rel], |
| 174 | cwd=tmp_path, |
| 175 | runner=git_status_runner(), |
| 176 | kit=kit_root(), |
| 177 | review_provider_factory=pass_provider_factory(), |
| 178 | ) |
| 179 | text = artifact.read_text(encoding="utf-8") |
| 180 | artifact.write_text(text.replace("gate: mechanical", "gate: substantive"), encoding="utf-8") |
| 181 | code = run_cli( |
| 182 | ["review", "--freeze", rel], |
| 183 | cwd=tmp_path, |
| 184 | runner=git_status_runner(), |
| 185 | kit=kit_root(), |
| 186 | review_provider_factory=pass_provider_factory(), |
| 187 | ) |
| 188 | assert code == 0 |
| 189 | stamp = extract_existing_stamp(parse_artifact(artifact, rel_path=rel)) |
| 190 | assert stamp["gate"] == "mechanical" |
| 191 | |
| 192 | |
| 193 | def test_escalate_force_pass_still_banned(tmp_path: Path) -> None: |
| 194 | artifact = seed_freeze_repo(tmp_path) |
| 195 | code = run_cli( |
| 196 | ["review", "--freeze", artifact.relative_to(tmp_path).as_posix(), "--escalate-force-pass"], |
| 197 | cwd=tmp_path, |
| 198 | runner=git_status_runner(), |
| 199 | kit=kit_root(), |
| 200 | review_provider_factory=pass_provider_factory(), |
| 201 | ) |
| 202 | assert code == 1 |
| 203 | |
| 204 | |
| 205 | def test_ledger_append_freeze_review(tmp_path: Path) -> None: |
| 206 | _enable_freeze_and_honesty(tmp_path) |
| 207 | config = load_config(tmp_path / ".overseer" / "config.yaml") |
| 208 | body = { |
| 209 | "actor_role": "verifier", |
| 210 | "actor_session_id": "review-chat-1", |
| 211 | "phase_id": "FRV-b", |
| 212 | "frozen_spec": "docs/archive/phases/PHASE-FRV.md", |
| 213 | "round": 1, |
| 214 | "gate": "substantive", |
| 215 | "freeze_verdict": "pass", |
| 216 | "artifact_digest": "sha256:" + ("c" * 64), |
| 217 | "reviewer_model": "thinking-high", |
| 218 | } |
| 219 | assert append_entry( |
| 220 | config=config, |
| 221 | repo_root=tmp_path, |
| 222 | options=LedgerAppendOptions(kind="freeze_review", body=body), |
| 223 | ).exit_code == 0 |
| 224 | assert verify_ledger_file(config=config, repo_root=tmp_path).exit_code == 0 |
| 225 | |
| 226 | |
| 227 | def test_split_and_plain_auto_emission(tmp_path: Path) -> None: |
| 228 | write_config(tmp_path, "config-git-only.yaml") |
| 229 | docs = tmp_path / "docs" |
| 230 | docs.mkdir(parents=True, exist_ok=True) |
| 231 | art = docs / "PHASE-FRV.md" |
| 232 | art.write_text( |
| 233 | "# Freeze\n\n```yaml\nphase: FRV\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n" |
| 234 | "review_stamp:\n verdict: pass\n```\n", |
| 235 | encoding="utf-8", |
| 236 | ) |
| 237 | config = load_config(tmp_path / ".overseer" / "config.yaml") |
| 238 | split_row = QueueRow( |
| 239 | phase_label="**FRV**", |
| 240 | model="Thinking → Auto", |
| 241 | status="**NEXT**", |
| 242 | deliverable="docs/PHASE-FRV.md", |
| 243 | raw_line="", |
| 244 | ) |
| 245 | emit, reason, is_b, advisory = decide_split_emission(split_row, tmp_path, config=config) |
| 246 | assert emit == "Thinking" |
| 247 | assert reason is None |
| 248 | assert advisory == "mechanical_only" |
| 249 | |
| 250 | auto_row = QueueRow( |
| 251 | phase_label="**FRV-b**", |
| 252 | model="Auto", |
| 253 | status="**NEXT**", |
| 254 | deliverable="docs/PHASE-FRV.md", |
| 255 | raw_line="", |
| 256 | ) |
| 257 | emit, reason, is_b, advisory = decide_split_emission(auto_row, tmp_path, config=config) |
| 258 | assert emit is None |
| 259 | assert reason == "freeze_not_substantive" |
| 260 | |
| 261 | no_art = QueueRow( |
| 262 | phase_label="**OTHER-b**", |
| 263 | model="Auto", |
| 264 | status="**NEXT**", |
| 265 | deliverable="no freeze here", |
| 266 | raw_line="", |
| 267 | ) |
| 268 | emit, reason, is_b, advisory = decide_split_emission(no_art, tmp_path, config=config) |
| 269 | assert emit == "Auto" |
| 270 | assert reason is None |
| 271 | |
| 272 | op_row = QueueRow( |
| 273 | phase_label="**LAND**", |
| 274 | model="Operator + Auto", |
| 275 | status="**NEXT**", |
| 276 | deliverable="docs/PHASE-FRV.md", |
| 277 | raw_line="", |
| 278 | ) |
| 279 | emit, reason, is_b, advisory = decide_split_emission(op_row, tmp_path, config=config) |
| 280 | assert emit == "Operator + Auto" |
| 281 | |
| 282 | |
| 283 | def test_gate_scan_mechanical_vs_substantive(tmp_path: Path) -> None: |
| 284 | write_config(tmp_path, "config-git-only.yaml") |
| 285 | docs = tmp_path / "docs" |
| 286 | docs.mkdir(parents=True, exist_ok=True) |
| 287 | art = docs / "archive" / "phases" |
| 288 | art.mkdir(parents=True) |
| 289 | phase = art / "PHASE-FRV.md" |
| 290 | phase.write_text( |
| 291 | "```yaml\nphase: FRV\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n" |
| 292 | "review_stamp:\n verdict: pass\n```\n", |
| 293 | encoding="utf-8", |
| 294 | ) |
| 295 | roadmap = ( |
| 296 | "# Roadmap\n\n## Build queue\n\n| Phase | Model | Status | Deliverable |\n" |
| 297 | "| --- | --- | --- | --- |\n" |
| 298 | "| **FRV-b Build** | Auto | **WIP** | docs/archive/phases/PHASE-FRV.md |\n" |
| 299 | ) |
| 300 | (docs / "ROADMAP.md").write_text(roadmap, encoding="utf-8") |
| 301 | (docs / "OVERSEER-HANDOVER.md").write_text( |
| 302 | "| **ID** | **FRV-b Build** |\n", encoding="utf-8" |
| 303 | ) |
| 304 | config = load_config(tmp_path / ".overseer" / "config.yaml") |
| 305 | # ensure gates remind |
| 306 | if not getattr(config.governance_gates, "remind", True): |
| 307 | pass |
| 308 | result = scan_governance_gates(config, tmp_path, roadmap_text=roadmap, handover_text="| **ID** | **FRV-b Build** |\n") |
| 309 | pending = [p for p in result.pending if p.gate_id == "freeze_review"] |
| 310 | assert pending |
| 311 | assert "mechanical_only" in pending[0].message |
File History
1 commit
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4
docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit.
Human
3 days ago