aaronrene / overseer-kit public
test_p_evidence_e2e.py python
130 lines 4.6 KB
Raw
sha256:0e9549ec7b463911bc08b7d586dc320b1ac9b1f5c943ee7e3865dcc6cb0f6f83 chore(governance): sync handover+roadmap to 84db8c8 (drift:… Human 1 day ago
1 """End-to-end P-evidence CLI tests (§PE.10)."""
2
3 from __future__ import annotations
4
5 import json
6 from pathlib import Path
7
8 import yaml
9
10 from cli.kit_root import kit_root
11 from tests.fixtures.p_evidence import (
12 FIXTURES,
13 copy_p_evidence_entries,
14 load_p_evidence_entry,
15 seed_p_evidence_repo,
16 )
17 from tests.support import git_status_runner, muse_mirror_status_runner, run_cli, seed_muse_substrate
18
19
20 def _append_evidence(tmp_path: Path, body_name: str) -> int:
21 copy_p_evidence_entries(tmp_path)
22 payload = tmp_path / "payload.json"
23 payload.write_text(json.dumps(load_p_evidence_entry(body_name)), encoding="utf-8")
24 return run_cli(
25 ["ledger", "append", "--kind", "verification_evidence", "--file", "payload.json"],
26 cwd=tmp_path,
27 runner=git_status_runner(),
28 kit=kit_root(),
29 )
30
31
32 def test_e2e_append_show_and_mode_b_match(tmp_path: Path, capsys) -> None:
33 seed_p_evidence_repo(tmp_path, require_verification_evidence="require")
34 assert _append_evidence(tmp_path, "verification-evidence-pass.json") == 0
35 show_code = run_cli(
36 ["ledger", "show", "--last", "3"],
37 cwd=tmp_path,
38 runner=git_status_runner(),
39 kit=kit_root(),
40 )
41 assert show_code == 0
42 out = capsys.readouterr().out
43 assert "verification_evidence" in out
44
45 code = run_cli(
46 [
47 "honesty-status",
48 "--verification-evidence",
49 "Track P / P-evidence",
50 "--frozen-spec",
51 "docs/archive/phases/PHASE-TRACK-P-P-EVIDENCE.md",
52 "--json",
53 ],
54 cwd=tmp_path,
55 runner=git_status_runner(),
56 kit=kit_root(),
57 json_mode=True,
58 )
59 assert code == 0
60 payload = json.loads(capsys.readouterr().out)
61 assert payload["verification_evidence"]["matched_entry_hash"] is not None
62
63
64 def test_e2e_last_wins_findings_then_pass(tmp_path: Path, capsys) -> None:
65 seed_p_evidence_repo(tmp_path, require_verification_evidence="require")
66 assert _append_evidence(tmp_path, "verification-evidence-findings.json") == 0
67 body = load_p_evidence_entry("verification-evidence-pass.json")
68 body["round"] = 2
69 second = tmp_path / "second.json"
70 second.write_text(json.dumps(body), encoding="utf-8")
71 assert run_cli(
72 ["ledger", "append", "--kind", "verification_evidence", "--file", "second.json"],
73 cwd=tmp_path,
74 runner=git_status_runner(),
75 kit=kit_root(),
76 ) == 0
77 code = run_cli(
78 [
79 "honesty-status",
80 "--verification-evidence",
81 "Track P / P-evidence",
82 "--json",
83 ],
84 cwd=tmp_path,
85 runner=git_status_runner(),
86 kit=kit_root(),
87 json_mode=True,
88 )
89 assert code == 0
90 payload = json.loads(capsys.readouterr().out)
91 ledger = (tmp_path / ".overseer" / "honesty" / "VERDICT-LEDGER.jsonl").read_text(encoding="utf-8")
92 last_line = json.loads(ledger.strip().splitlines()[-1])
93 assert last_line["bv_verdict"] == "pass"
94 assert payload["verification_evidence"]["matched_entry_hash"] == last_line["entry_hash"]
95
96
97 def test_skill_normative_pass_body_includes_test_output() -> None:
98 fixture = json.loads((FIXTURES / "skill-pass-body.json").read_text(encoding="utf-8"))
99 body = load_p_evidence_entry("verification-evidence-pass.json")
100 assert body["bv_verdict"] == fixture["bv_verdict"]
101 types = {item["type"] for item in body["artifacts"]}
102 assert "test_output" in types
103 for required in fixture["required_artifact_types_for_pass"]:
104 assert required in types
105
106
107 def test_e2e_git_only_and_muse_regime_unsigned(tmp_path: Path) -> None:
108 for regime in ("config-git-only.yaml", "config-muse-git-mirror.yaml"):
109 repo = tmp_path / regime
110 repo.mkdir()
111 seed_p_evidence_repo(repo, require_verification_evidence="require", regime_config=regime)
112 if "muse" in regime:
113 seed_muse_substrate(repo)
114 copy_p_evidence_entries(repo)
115 payload = repo / "payload.json"
116 payload.write_text(json.dumps(load_p_evidence_entry("verification-evidence-pass.json")), encoding="utf-8")
117 runner = muse_mirror_status_runner(repo) if "muse" in regime else git_status_runner()
118 assert run_cli(
119 ["ledger", "append", "--kind", "verification_evidence", "--file", "payload.json"],
120 cwd=repo,
121 runner=runner,
122 kit=kit_root(),
123 ) == 0
124 assert run_cli(
125 ["honesty-status", "--verification-evidence", "Track P / P-evidence", "--json"],
126 cwd=repo,
127 runner=runner,
128 kit=kit_root(),
129 json_mode=True,
130 ) == 0
File History 1 commit
sha256:6abcf1fa82a7a621ccbc945f19acdba5bc0db54569599404a1452fb4a096a199 fix(ISR): default require_independent_second_reviewer to require Human minor ⚠ 1 day ago