aaronrene / overseer-kit public
test_checklist_engine.py python
61 lines 2.2 KB
Raw
sha256:4671b7f787ddbe63ced31c895b688c77ab495653b65a730b423329f26b3c1439 feat: K1-P1 complete — agent provenance, build-verification… Sonnet 4.6 patch 54 days ago
1 """Unit tests for ChecklistEngine semantics (K5b-r F1)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6
7 from tools.freeze_reviewer.checklist import builtin_checklist
8 from tools.freeze_reviewer.providers.base import ChecklistEngine
9
10
11 def test_normative_escalation_vocabulary_is_not_a_finding() -> None:
12 """Discussing security / tier-3 / billing must not auto-escalate (§K5.5 / §6.3)."""
13 text = """
14 # Freeze contract
15
16 ```yaml
17 phase: K5a
18 outputs:
19 - id: contract
20 path: docs/CONTRACT.md
21 frozen: true
22 ```
23
24 This document defines security, injection, irreversible deletes, real money billing,
25 live model spend, gates_tier3, and merge to main. Human escalation is rare and meaningful.
26 It includes a seven-tier test matrix and requires file+line citations.
27 """
28 findings = ChecklistEngine().evaluate(
29 artifact_text=text,
30 artifact_path="docs/CONTRACT.md",
31 checklist=builtin_checklist(),
32 )
33 escalation_cats = {"security", "irreversible", "real_money", "gates_tier3"}
34 assert not any(item.category in escalation_cats for item in findings)
35 assert not any(item.check in {"C4", "C5", "C6", "C7"} for item in findings)
36
37
38 def test_absolute_path_still_emits_c4_security() -> None:
39 text = (
40 "# x\n\n```yaml\nphase: K\noutputs:\n - id: a\n path: docs/a.md\n"
41 " frozen: true\n```\n\nPath: /Users/operator/secret/repo\n"
42 "seven-tier matrix and file+line discipline.\n"
43 )
44 findings = ChecklistEngine().evaluate(
45 artifact_text=text,
46 artifact_path="docs/x.md",
47 checklist=builtin_checklist(),
48 )
49 assert any(item.check == "C4" and item.category == "security" for item in findings)
50
51
52 def test_k5_contract_dogfood_has_no_false_escalation() -> None:
53 root = Path(__file__).resolve().parents[2]
54 text = (root / "docs" / "PHASE-K5-FREEZE-REVIEWER-CONTRACT.md").read_text(encoding="utf-8")
55 findings = ChecklistEngine().evaluate(
56 artifact_text=text,
57 artifact_path="docs/PHASE-K5-FREEZE-REVIEWER-CONTRACT.md",
58 checklist=builtin_checklist(),
59 )
60 escalation_cats = {"security", "irreversible", "real_money", "gates_tier3"}
61 assert not any(item.category in escalation_cats for item in findings)
File History 1 commit
sha256:4671b7f787ddbe63ced31c895b688c77ab495653b65a730b423329f26b3c1439 feat: K1-P1 complete — agent provenance, build-verification… Sonnet 4.6 patch 54 days ago