aaronrene / overseer-kit public
test_p_evidence_unit.py python
139 lines 5.3 KB
Raw
sha256:8d7f41aafae41deee70035a92f93602ef1722a290153597451e5932af503a42c docs: queue board-identity follow-ups so they survive the session Human 11 hours ago
1 """Unit tests for P-evidence verification_evidence schema (§PE.10)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6
7 import pytest
8 import yaml
9
10 from adapters.config import HONESTY_KEYS, load_config
11 from tests.fixtures.p_evidence import load_p_evidence_entry
12 from tests.support import seed_honesty_repo
13 from tools.honesty.types import BV_VERDICTS, ENTRY_KINDS, VERIFICATION_ARTIFACT_TYPES
14 from tools.honesty.validate import EntryValidationError, validate_append_body, validate_verification_artifacts
15
16
17 def _minimal_body(**overrides) -> dict:
18 body = load_p_evidence_entry("verification-evidence-pass.json")
19 body.update(overrides)
20 return body
21
22
23 def test_verification_evidence_in_entry_kinds() -> None:
24 assert "verification_evidence" in ENTRY_KINDS
25 assert VERIFICATION_ARTIFACT_TYPES == frozenset({"test_output", "deploy_health", "screenshot"})
26 assert BV_VERDICTS == frozenset({"pass", "findings", "blocked"})
27
28
29 def test_validate_accepts_minimal_verification_evidence() -> None:
30 validated = validate_append_body(kind="verification_evidence", body=_minimal_body())
31 assert validated["kind"] == "verification_evidence"
32 assert validated["artifacts"][0]["type"] == "test_output"
33
34
35 def test_unknown_artifact_type_exit_2() -> None:
36 body = _minimal_body(artifacts=[{"type": "video", "sha256": "a" * 64}])
37 with pytest.raises(EntryValidationError) as exc:
38 validate_append_body(kind="verification_evidence", body=body)
39 assert exc.value.exit_code == 2
40
41
42 def test_uppercase_sha256_exit_2() -> None:
43 body = _minimal_body(
44 artifacts=[{"type": "test_output", "sha256": "B" * 64}],
45 )
46 with pytest.raises(EntryValidationError) as exc:
47 validate_append_body(kind="verification_evidence", body=body)
48 assert exc.value.exit_code == 2
49
50
51 def test_short_sha256_exit_2() -> None:
52 body = _minimal_body(artifacts=[{"type": "test_output", "sha256": "abc"}])
53 with pytest.raises(EntryValidationError) as exc:
54 validate_append_body(kind="verification_evidence", body=body)
55 assert exc.value.exit_code == 2
56
57
58 def test_deploy_health_missing_ref_exit_2() -> None:
59 with pytest.raises(EntryValidationError) as exc:
60 validate_verification_artifacts([{"type": "deploy_health", "sha256": "a" * 64}])
61 assert exc.value.exit_code == 2
62
63
64 def test_screenshot_missing_ref_exit_2() -> None:
65 with pytest.raises(EntryValidationError) as exc:
66 validate_verification_artifacts([{"type": "screenshot", "sha256": "a" * 64}])
67 assert exc.value.exit_code == 2
68
69
70 def test_empty_artifacts_exit_24() -> None:
71 body = _minimal_body(artifacts=[])
72 with pytest.raises(EntryValidationError) as exc:
73 validate_append_body(kind="verification_evidence", body=body)
74 assert exc.value.exit_code == 24
75
76
77 def test_non_verifier_actor_exit_23() -> None:
78 body = _minimal_body(actor_role="producer")
79 with pytest.raises(EntryValidationError) as exc:
80 validate_append_body(kind="verification_evidence", body=body)
81 assert exc.value.exit_code == 23
82
83
84 def test_bad_bv_verdict_exit_2() -> None:
85 body = _minimal_body(bv_verdict="PASS")
86 with pytest.raises(EntryValidationError) as exc:
87 validate_append_body(kind="verification_evidence", body=body)
88 assert exc.value.exit_code == 2
89
90
91 def test_round_less_than_one_exit_2() -> None:
92 body = _minimal_body(round=0)
93 with pytest.raises(EntryValidationError) as exc:
94 validate_append_body(kind="verification_evidence", body=body)
95 assert exc.value.exit_code == 2
96
97
98 def test_genesis_forbids_verification_evidence_keys() -> None:
99 for key in ("phase_id", "frozen_spec", "round", "bv_verdict", "artifacts", "subject_sha256"):
100 with pytest.raises(EntryValidationError) as exc:
101 validate_append_body(kind="genesis", body={key: "x"})
102 assert exc.value.exit_code == 2
103
104
105 def test_frozen_spec_opaque_without_file_existence() -> None:
106 body = _minimal_body(frozen_spec="docs/DOES-NOT-EXIST.md")
107 validated = validate_append_body(kind="verification_evidence", body=body)
108 assert validated["frozen_spec"] == "docs/DOES-NOT-EXIST.md"
109
110
111 def test_require_verification_evidence_config_parse(repo_root: Path) -> None:
112 seed_honesty_repo(repo_root)
113 cfg_path = repo_root / ".overseer" / "config.yaml"
114 data = yaml.safe_load(cfg_path.read_text(encoding="utf-8"))
115 assert "require_verification_evidence" not in data["honesty"]
116 config = load_config(cfg_path)
117 assert config.honesty.require_verification_evidence == "off"
118
119 for mode in ("off", "warn", "require"):
120 data["honesty"]["require_verification_evidence"] = mode
121 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
122 assert load_config(cfg_path).honesty.require_verification_evidence == mode
123
124 data["honesty"]["require_verification_evidence"] = "maybe"
125 cfg_path.write_text(yaml.safe_dump(data), encoding="utf-8")
126 with pytest.raises(Exception, match="require_verification_evidence"):
127 load_config(cfg_path)
128
129
130 def test_require_verification_evidence_in_honesty_keys() -> None:
131 assert "require_verification_evidence" in HONESTY_KEYS
132
133
134 def test_honesty_error_token_includes_missing_verification_evidence() -> None:
135 from typing import get_args
136
137 from tools.honesty.types import HonestyErrorToken
138
139 assert "missing_verification_evidence" in get_args(HonestyErrorToken)
File History 1 commit
sha256:8d7f41aafae41deee70035a92f93602ef1722a290153597451e5932af503a42c docs: queue board-identity follow-ups so they survive the session Human 11 hours ago