aaronrene / overseer-kit public
test_isr_unit.py python
276 lines 8.5 KB
Raw
sha256:c07f2f34a0db9f43fe866f157d1935322008545ff8940c06c7c921eb219c55ab NXP-b DONE: independent BV-r2 pass + ISR (SD-17) Human minor ⚠ breaking 20 hours ago
1 """Unit tests for ISR ledger kind + Mode D resolution (§ISR.11)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6 from typing import get_args
7
8 import pytest
9 import yaml
10
11 from adapters.config import HONESTY_KEYS, HonestyConfig, load_config
12 from tests.fixtures.isr import load_isr_entry
13 from tests.support import seed_honesty_repo
14 from tools.honesty.status import HonestyStatusOptions, _resolve_mode
15 from tools.honesty.types import ENTRY_KINDS, ISR_VERDICTS, HonestyErrorToken
16 from tools.honesty.validate import (
17 EntryValidationError,
18 find_matching_independent_second_review,
19 validate_append_body,
20 )
21
22
23 def _minimal(**overrides) -> dict:
24 body = load_isr_entry("isr-pass.json")
25 body.update(overrides)
26 return body
27
28
29 def test_isr_in_entry_kinds() -> None:
30 assert "independent_second_review" in ENTRY_KINDS
31 assert ISR_VERDICTS == frozenset({"pass", "findings", "blocked"})
32
33
34 def test_validate_accepts_minimal_isr() -> None:
35 validated = validate_append_body(kind="independent_second_review", body=_minimal())
36 assert validated["kind"] == "independent_second_review"
37 assert validated["isr_verdict"] == "pass"
38
39
40 def test_same_session_ids_exit_2() -> None:
41 body = _minimal(actor_session_id="same", producer_session_id="same")
42 with pytest.raises(EntryValidationError) as exc:
43 validate_append_body(kind="independent_second_review", body=body)
44 assert exc.value.exit_code == 2
45
46
47 def test_empty_producer_session_exit_2() -> None:
48 body = _minimal(producer_session_id="")
49 with pytest.raises(EntryValidationError) as exc:
50 validate_append_body(kind="independent_second_review", body=body)
51 assert exc.value.exit_code == 2
52
53
54 def test_missing_producer_session_exit_2() -> None:
55 body = _minimal()
56 del body["producer_session_id"]
57 with pytest.raises(EntryValidationError) as exc:
58 validate_append_body(kind="independent_second_review", body=body)
59 assert exc.value.exit_code == 2
60
61
62 def test_non_verifier_exit_23() -> None:
63 body = _minimal(actor_role="producer")
64 with pytest.raises(EntryValidationError) as exc:
65 validate_append_body(kind="independent_second_review", body=body)
66 assert exc.value.exit_code == 23
67
68
69 def test_bad_isr_verdict_exit_2() -> None:
70 body = _minimal(isr_verdict="ok")
71 with pytest.raises(EntryValidationError) as exc:
72 validate_append_body(kind="independent_second_review", body=body)
73 assert exc.value.exit_code == 2
74
75
76 def test_round_lt_1_exit_2() -> None:
77 body = _minimal(round=0)
78 with pytest.raises(EntryValidationError) as exc:
79 validate_append_body(kind="independent_second_review", body=body)
80 assert exc.value.exit_code == 2
81
82
83 def test_both_agent_ids_equal_exit_2() -> None:
84 body = _minimal(producer_agent_id="agent-x", verifier_agent_id="agent-x")
85 with pytest.raises(EntryValidationError) as exc:
86 validate_append_body(kind="independent_second_review", body=body)
87 assert exc.value.exit_code == 2
88
89
90 def test_one_agent_id_only_accept() -> None:
91 validated = validate_append_body(
92 kind="independent_second_review",
93 body=_minimal(producer_agent_id="only-producer"),
94 )
95 assert validated["producer_agent_id"] == "only-producer"
96
97
98 def test_genesis_forbid_isr_keys() -> None:
99 for key in (
100 "isr_verdict",
101 "producer_session_id",
102 "producer_agent_id",
103 "verifier_agent_id",
104 "bound_verification_evidence_hash",
105 ):
106 with pytest.raises(EntryValidationError) as exc:
107 validate_append_body(kind="genesis", body={key: "x"})
108 assert exc.value.exit_code == 2
109
110
111 def test_frozen_spec_opaque_non_empty_no_must_exist() -> None:
112 validated = validate_append_body(
113 kind="independent_second_review",
114 body=_minimal(frozen_spec="docs/does-not-need-to-exist.md"),
115 )
116 assert validated["frozen_spec"] == "docs/does-not-need-to-exist.md"
117
118
119 def test_require_isr_config_parse(repo_root: Path) -> None:
120 seed_honesty_repo(repo_root)
121 cfg_path = repo_root / ".overseer" / "config.yaml"
122 data = yaml.safe_load(cfg_path.read_text(encoding="utf-8"))
123 assert "require_independent_second_reviewer" not in data["honesty"]
124 config = load_config(cfg_path)
125 assert config.honesty.require_independent_second_reviewer == "require"
126
127 for mode in ("off", "warn", "require"):
128 data["honesty"]["require_independent_second_reviewer"] = mode
129 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
130 assert load_config(cfg_path).honesty.require_independent_second_reviewer == mode
131
132 data["honesty"]["require_independent_second_reviewer"] = "maybe"
133 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
134 with pytest.raises(Exception, match="require_independent_second_reviewer"):
135 load_config(cfg_path)
136
137
138 def test_require_isr_in_honesty_keys() -> None:
139 assert "require_independent_second_reviewer" in HONESTY_KEYS
140
141
142 def test_honesty_config_field_default_require() -> None:
143 assert HonestyConfig().require_independent_second_reviewer == "require"
144
145
146 def test_error_token_includes_missing_isr() -> None:
147 assert "missing_independent_second_review" in get_args(HonestyErrorToken)
148
149
150 def test_resolve_mode_d_invariants() -> None:
151 # (1) Mode D + frozen-spec
152 assert (
153 _resolve_mode(
154 HonestyStatusOptions(
155 hook=None,
156 artifact=None,
157 independent_second_review="ISR-b",
158 frozen_spec="docs/x.md",
159 )
160 )
161 == "mode_d"
162 )
163 # (2) Mode D + producer-session
164 assert (
165 _resolve_mode(
166 HonestyStatusOptions(
167 hook=None,
168 artifact=None,
169 independent_second_review="ISR-b",
170 producer_session="builder-chat-1",
171 )
172 )
173 == "mode_d"
174 )
175 # (3) Mode D + both optionals
176 assert (
177 _resolve_mode(
178 HonestyStatusOptions(
179 hook=None,
180 artifact=None,
181 independent_second_review="ISR-b",
182 producer_session="builder-chat-1",
183 frozen_spec="docs/x.md",
184 )
185 )
186 == "mode_d"
187 )
188 # (4) Mode D alone
189 assert (
190 _resolve_mode(
191 HonestyStatusOptions(hook=None, artifact=None, independent_second_review="ISR-b")
192 )
193 == "mode_d"
194 )
195 # (5) producer alone → usage
196 assert (
197 _resolve_mode(HonestyStatusOptions(hook=None, artifact=None, producer_session="x"))
198 is None
199 )
200 # (6) Mode B + producer → usage
201 assert (
202 _resolve_mode(
203 HonestyStatusOptions(
204 hook=None,
205 artifact=None,
206 verification_evidence="p",
207 producer_session="x",
208 )
209 )
210 is None
211 )
212 # (7) Mode C + producer → usage
213 assert (
214 _resolve_mode(
215 HonestyStatusOptions(
216 hook=None,
217 artifact=None,
218 deploy_health="p",
219 producer_session="x",
220 )
221 )
222 is None
223 )
224 # (8) Mode A + Mode D → usage
225 assert (
226 _resolve_mode(
227 HonestyStatusOptions(
228 hook="board_done",
229 artifact="a.txt",
230 independent_second_review="ISR-b",
231 )
232 )
233 is None
234 )
235
236
237 def test_find_matching_isr_last_wins_and_pin() -> None:
238 findings = load_isr_entry("isr-findings.json")
239 findings["entry_hash"] = "f" * 64
240 first_pass = load_isr_entry("isr-pass.json")
241 first_pass["entry_hash"] = "a" * 64
242 first_pass["round"] = 1
243 second_pass = load_isr_entry("isr-pass.json")
244 second_pass["entry_hash"] = "b" * 64
245 second_pass["round"] = 2
246 second_pass["actor_session_id"] = "verifier-chat-3"
247 entries = [findings, first_pass, second_pass]
248
249 assert (
250 find_matching_independent_second_review(
251 entries, phase_id="ISR-b", frozen_spec=None, producer_session=None
252 )["entry_hash"]
253 == "b" * 64
254 )
255 assert (
256 find_matching_independent_second_review(
257 [findings], phase_id="ISR-b", frozen_spec=None, producer_session=None
258 )
259 is None
260 )
261 pinned = find_matching_independent_second_review(
262 entries,
263 phase_id="ISR-b",
264 frozen_spec=None,
265 producer_session="builder-chat-1",
266 )
267 assert pinned is not None
268 assert (
269 find_matching_independent_second_review(
270 entries,
271 phase_id="ISR-b",
272 frozen_spec=None,
273 producer_session="other-builder",
274 )
275 is None
276 )
File History 1 commit
sha256:c07f2f34a0db9f43fe866f157d1935322008545ff8940c06c7c921eb219c55ab NXP-b DONE: independent BV-r2 pass + ISR (SD-17) Human minor ⚠ 20 hours ago