aaronrene / overseer-kit public
test_frv_unit.py python
302 lines 11.2 KB
Raw
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4 docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit. Human 4 days ago
1 """Unit tests for FRV freeze-review verdict integrity (§FRV.12)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6
7 import pytest
8
9 from adapters.config import load_config
10 from tools.freeze_authorization import ACCEPT_LEGACY_VERDICT_STAMP
11 from tools.freeze_authorization.resolve import (
12 FreezeAuthorization,
13 freeze_authorization_state,
14 resolve_stamp_record,
15 )
16 from tools.freeze_reviewer.findings import derive_verdict
17 from tools.freeze_reviewer.providers.base import ApiReviewProvider, LocalReviewProvider
18 from tools.freeze_reviewer.stamp import build_stamp, merge_stamp_mapping
19 from tools.freeze_reviewer.types import Finding, ReviewStamp, ReviewerSettings, STAMP_KEY_ORDER
20 from tools.governance_hygiene.next_regen import (
21 ADVISORY_FORGED_SUBSTANTIVE_GATE,
22 ADVISORY_LEDGER_CHAIN_BROKEN,
23 ADVISORY_MECHANICAL_ONLY,
24 ADVISORY_OPERATOR_BLOCK,
25 ADVISORY_OPERATOR_BLOCK_MALFORMED,
26 ADVISORY_UNREADABLE_GATE,
27 REASON_FREEZE_NOT_SUBSTANTIVE,
28 compact_step_id,
29 )
30 from tools.honesty.types import ENTRY_KINDS, FREEZE_VERDICTS
31 from tools.honesty.validate import EntryValidationError, find_matching_freeze_review, validate_append_body
32
33
34 DIGEST = "sha256:" + ("a" * 64)
35
36
37 def _stamp_kwargs(**overrides):
38 base = dict(
39 reviewed_at="2026-09-12T00:00:00Z",
40 mechanical_verdict="pass",
41 reviewer_mode="agent",
42 reviewer_model=None,
43 reviewer_provider="local",
44 kit_version="0.1.0",
45 artifact_digest=DIGEST,
46 gate="mechanical",
47 produced_by="checklist_engine",
48 provider_kind="rule_engine",
49 checklist_ids=["C1", "C2"],
50 checklist_source="builtin",
51 findings_count=0,
52 override_applied=False,
53 )
54 base.update(overrides)
55 return base
56
57
58 def test_to_mapping_fourteen_keys_no_verdict() -> None:
59 mapping = ReviewStamp(**_stamp_kwargs()).to_mapping()
60 assert list(mapping.keys()) == list(STAMP_KEY_ORDER)
61 assert len(mapping) == 14
62 assert "verdict" not in mapping
63 assert mapping["gate"] == "mechanical"
64
65
66 def test_mechanical_verdict_from_derive() -> None:
67 assert derive_verdict([], human_escalation=["security"]) == "pass"
68 findings = [
69 Finding(check="C1", severity="MAJOR", category="completeness", path="a.md", line=1, message="x").with_citation()
70 ]
71 assert derive_verdict(findings, human_escalation=["security"]) == "findings"
72 blockers = [
73 Finding(check="C4", severity="BLOCKER", category="security", path="a.md", line=1, message="x").with_citation()
74 ]
75 assert derive_verdict(blockers, human_escalation=["security"]) == "blocked"
76
77
78 def test_resolver_six_branches() -> None:
79 assert resolve_stamp_record({"gate": "mechanical", "mechanical_verdict": "pass"}).kind == "mechanical"
80 forged = resolve_stamp_record({"gate": "substantive", "mechanical_verdict": "pass"})
81 assert forged.kind == "forged_substantive"
82 assert forged.advisory == "forged_substantive_gate"
83 unread = resolve_stamp_record({"gate": 1})
84 assert unread.kind == "unreadable"
85 assert unread.advisory == "unreadable_gate"
86 partial = resolve_stamp_record({"mechanical_verdict": "findings"})
87 assert partial.kind == "mechanical"
88 assert partial.verdict == "findings"
89 assert ACCEPT_LEGACY_VERDICT_STAMP is True
90 legacy = resolve_stamp_record({"verdict": "pass"})
91 assert legacy.kind == "mechanical"
92 assert legacy.verdict == "pass"
93 assert resolve_stamp_record({}).kind == "none"
94 assert resolve_stamp_record(None).kind == "none"
95
96
97 def test_producer_identity_four_pairs() -> None:
98 local = LocalReviewProvider()
99 assert local.producer_identity() == ("checklist_engine", "rule_engine")
100 scripted = LocalReviewProvider(scripted_findings=[])
101 assert scripted.producer_identity() == ("scripted_provider", "rule_engine")
102 api_scripted = ApiReviewProvider(scripted_findings=[])
103 assert api_scripted.producer_identity() == ("scripted_provider", "rule_engine")
104 api = ApiReviewProvider()
105 assert api.producer_identity() == ("api_model", "model_api")
106
107
108 def test_build_stamp_nulls_model_for_rule_engine(tmp_path: Path) -> None:
109 path = tmp_path / "f.yaml"
110 path.write_text("phase: X\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n", encoding="utf-8")
111 from tools.freeze_reviewer.artifact import parse_artifact
112
113 parsed = parse_artifact(path, rel_path="f.yaml")
114 stamp = build_stamp(
115 parsed,
116 reviewer=ReviewerSettings("agent", "thinking-high", "local", "human"),
117 kit_version="0.1.0",
118 produced_by="checklist_engine",
119 provider_kind="rule_engine",
120 checklist_ids=["C1"],
121 checklist_source="builtin",
122 findings_count=0,
123 )
124 assert stamp.reviewer_model is None
125 assert "verdict" not in stamp.to_mapping()
126 api_stamp = build_stamp(
127 parsed,
128 reviewer=ReviewerSettings("agent", "thinking-high", "api", "human"),
129 kit_version="0.1.0",
130 produced_by="api_model",
131 provider_kind="model_api",
132 checklist_ids=["C1"],
133 checklist_source="operator_file",
134 findings_count=3,
135 )
136 assert api_stamp.reviewer_model == "thinking-high"
137 assert api_stamp.checklist_source == "operator_file"
138 assert api_stamp.findings_count == 3
139
140
141 def test_merge_keeps_unknown_drops_verdict() -> None:
142 existing = {"verdict": "findings", "operator_note": "keep", "extra": {"nested": 1}}
143 new = ReviewStamp(**_stamp_kwargs()).to_mapping()
144 merged = merge_stamp_mapping(existing, new)
145 assert "verdict" not in merged
146 assert merged["operator_note"] == "keep"
147 assert merged["extra"] == {"nested": 1}
148 assert list(merged.keys())[:14] == list(STAMP_KEY_ORDER)
149
150
151 def test_freeze_review_entry_kind_and_validation() -> None:
152 assert "freeze_review" in ENTRY_KINDS
153 assert FREEZE_VERDICTS == frozenset({"pass", "findings", "blocked"})
154 body = {
155 "actor_role": "verifier",
156 "actor_session_id": "review-1",
157 "phase_id": "FRV-b",
158 "frozen_spec": "docs/archive/phases/PHASE-FRV.md",
159 "round": 1,
160 "gate": "substantive",
161 "freeze_verdict": "pass",
162 "artifact_digest": DIGEST,
163 "reviewer_model": "thinking-high",
164 }
165 assert validate_append_body(kind="freeze_review", body=body)["gate"] == "substantive"
166
167 for bad in (
168 {"gate": "mechanical"},
169 {"freeze_verdict": "ok"},
170 {"artifact_digest": "sha256:ABC"},
171 {"artifact_digest": "sha256:" + ("a" * 63)},
172 {"artifact_digest": "md5:" + ("a" * 64)},
173 {"round": 0},
174 {"round": "1"},
175 {"phase_id": ""},
176 {"frozen_spec": ""},
177 {"findings_count": -1},
178 {"checklist_ids": []},
179 {"producer_session_id": "review-1"},
180 ):
181 mutated = dict(body)
182 mutated.update(bad)
183 with pytest.raises(EntryValidationError) as exc:
184 validate_append_body(kind="freeze_review", body=mutated)
185 assert exc.value.exit_code == 2
186
187 with pytest.raises(EntryValidationError) as exc:
188 validate_append_body(kind="freeze_review", body={**body, "actor_role": "producer"})
189 assert exc.value.exit_code == 23
190
191 for key in ("gate", "freeze_verdict", "artifact_digest", "checklist_ids", "findings_count"):
192 with pytest.raises(EntryValidationError) as exc:
193 validate_append_body(kind="genesis", body={key: "x"})
194 assert exc.value.exit_code == 2
195
196
197 def test_find_matching_freeze_review_rules() -> None:
198 good = {
199 "kind": "freeze_review",
200 "actor_role": "verifier",
201 "gate": "substantive",
202 "freeze_verdict": "pass",
203 "phase_id": "FRV-b",
204 "frozen_spec": "docs/x.md",
205 "artifact_digest": DIGEST,
206 "actor_session_id": "a",
207 "entry_hash": "h1",
208 }
209 later = {**good, "entry_hash": "h2"}
210 assert find_matching_freeze_review(
211 [good, later], phase_id="FRV-b", frozen_spec="docs/x.md", artifact_digest=DIGEST
212 )["entry_hash"] == "h2"
213 assert (
214 find_matching_freeze_review(
215 [{**good, "freeze_verdict": "findings"}],
216 phase_id="FRV-b",
217 frozen_spec="docs/x.md",
218 artifact_digest=DIGEST,
219 )
220 is None
221 )
222 assert (
223 find_matching_freeze_review(
224 [{**good, "gate": "mechanical"}],
225 phase_id="FRV-b",
226 frozen_spec="docs/x.md",
227 artifact_digest=DIGEST,
228 )
229 is None
230 )
231 assert (
232 find_matching_freeze_review(
233 [good],
234 phase_id="FRV-b",
235 frozen_spec="docs/x.md",
236 artifact_digest="sha256:" + ("b" * 64),
237 )
238 is None
239 )
240 same_session = {**good, "producer_session_id": "a"}
241 assert (
242 find_matching_freeze_review(
243 [same_session], phase_id="FRV-b", frozen_spec="docs/x.md", artifact_digest=DIGEST
244 )
245 is None
246 )
247
248
249 def test_advisory_and_reason_constants() -> None:
250 assert REASON_FREEZE_NOT_SUBSTANTIVE == "freeze_not_substantive"
251 assert ADVISORY_MECHANICAL_ONLY == "mechanical_only"
252 assert ADVISORY_OPERATOR_BLOCK == "operator_block"
253 assert ADVISORY_OPERATOR_BLOCK_MALFORMED == "operator_block_malformed"
254 assert ADVISORY_LEDGER_CHAIN_BROKEN == "ledger_chain_broken"
255 assert ADVISORY_FORGED_SUBSTANTIVE_GATE == "forged_substantive_gate"
256 assert ADVISORY_UNREADABLE_GATE == "unreadable_gate"
257
258
259 def test_compact_step_id_retains_suffix() -> None:
260 assert compact_step_id("**FRV-a Freeze**") == "FRV-a"
261 assert compact_step_id("**FRV-b Build**") == "FRV-b"
262 assert compact_step_id("**FRV-a Freeze**") != compact_step_id("**FRV-b Build**")
263
264
265 def test_freeze_authorization_states(tmp_path: Path) -> None:
266 config = load_config(Path(__file__).resolve().parents[1] / "fixtures" / "config-git-only.yaml")
267 docs = tmp_path / "docs"
268 docs.mkdir()
269 art = docs / "PHASE-FRV.md"
270 art.write_text(
271 "```yaml\nphase: FRV\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n"
272 "review_stamp:\n verdict: pass\n```\n",
273 encoding="utf-8",
274 )
275 auth = freeze_authorization_state(tmp_path, art, phase_id="FRV-b", config=config)
276 assert isinstance(auth, FreezeAuthorization)
277 assert auth.state == "mechanical_only"
278
279 art.write_text(
280 "```yaml\nphase: FRV\nauto_may_start: false\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n"
281 "review_stamp:\n verdict: pass\n```\n",
282 encoding="utf-8",
283 )
284 assert freeze_authorization_state(tmp_path, art, phase_id="FRV-b", config=config).state == "blocked_by_operator"
285
286 art.write_text(
287 "```yaml\nphase: FRV\nauto_may_start: maybe\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n```\n",
288 encoding="utf-8",
289 )
290 blocked = freeze_authorization_state(tmp_path, art, phase_id="FRV-b", config=config)
291 assert blocked.state == "blocked_by_operator"
292 assert blocked.advisory == "operator_block_malformed"
293
294 art.write_text(
295 "```yaml\nphase: FRV\noutputs:\n - id: a\n path: docs/a.md\n frozen: true\n"
296 "review_stamp:\n verdict: findings\n```\n",
297 encoding="utf-8",
298 )
299 assert freeze_authorization_state(tmp_path, art, phase_id="FRV-b", config=config).state == "non_pass"
300
301 art.write_text("# no stamp\n", encoding="utf-8")
302 assert freeze_authorization_state(tmp_path, art, phase_id="FRV-b", config=config).state == "absent"
File History 1 commit
sha256:8461d44b77376fbf06fa7c3e085d309e3010fd8d5886d63c63e69ce118811ad4 docs: record AFF-b feature-tip SHAs after AFF-b-ISR commit. Human 4 days ago