aaronrene / overseer-kit public
test_cost_awareness_unit.py python
183 lines 5.7 KB
Raw
sha256:0e9549ec7b463911bc08b7d586dc320b1ac9b1f5c943ee7e3865dcc6cb0f6f83 chore(governance): sync handover+roadmap to 84db8c8 (drift:… Human 2 days ago
1 """Unit tests for Track P / P-cost cost awareness (§PC.9)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6
7 import pytest
8 import yaml
9
10 from adapters.config import load_config
11 from adapters.errors import ConfigError
12 from cli.kit_root import kit_root
13 from tests.support import FIXTURES
14 from tools.cost_awareness.derive import COST_CLASS_ORDER, derive_cost_view
15 from tools.cost_awareness.normalize import gate_for_phase, normalize_phase_tier
16 from tools.governance_gates.types import PendingGate
17 from tools.model_routing.labels import (
18 COST_CLASS_VALUES,
19 RoutingPolicyError,
20 load_model_tier_cost_bands,
21 validate_model_tier_entry,
22 )
23
24
25 def test_cost_class_optional_valid_values() -> None:
26 for band in COST_CLASS_VALUES:
27 tier_id = validate_model_tier_entry(
28 {
29 "id": "fast",
30 "display": "Fast",
31 "meaning": "quick",
32 "cost_class": band,
33 },
34 index=0,
35 path="policy/model-labels.yaml",
36 )
37 assert tier_id == "fast"
38
39
40 def test_cost_class_rejects_unknown_value() -> None:
41 with pytest.raises(ConfigError, match="outside frozen vocabulary"):
42 validate_model_tier_entry(
43 {
44 "id": "fast",
45 "display": "Fast",
46 "meaning": "quick",
47 "cost_class": "unknown",
48 },
49 index=0,
50 path="policy/model-labels.yaml",
51 )
52
53
54 def test_cost_class_rejects_non_string() -> None:
55 with pytest.raises(ConfigError, match="must be a string"):
56 validate_model_tier_entry(
57 {
58 "id": "fast",
59 "display": "Fast",
60 "meaning": "quick",
61 "cost_class": 42,
62 },
63 index=0,
64 path="policy/model-labels.yaml",
65 )
66
67
68 def test_cost_class_recognized_key_not_unknown() -> None:
69 validate_model_tier_entry(
70 {
71 "id": "fast",
72 "display": "Fast",
73 "meaning": "quick",
74 "cost_class": "low",
75 "cursor_model_hint": "hint",
76 },
77 index=0,
78 path="policy/model-labels.yaml",
79 )
80
81
82 def test_cost_class_fail_closed_exit_32(tmp_path: Path) -> None:
83 labels = tmp_path / "policy" / "model-labels.yaml"
84 labels.parent.mkdir(parents=True)
85 labels.write_text(
86 yaml.safe_dump(
87 {
88 "model_tiers": [
89 {
90 "id": "fast",
91 "display": "Fast",
92 "meaning": "quick",
93 "cost_class": "pricey",
94 }
95 ]
96 }
97 ),
98 encoding="utf-8",
99 )
100 load_model_tier_cost_bands.cache_clear()
101 with pytest.raises(RoutingPolicyError) as exc:
102 load_model_tier_cost_bands(tmp_path, fail_closed=True)
103 assert exc.value.exit_code == 32
104 load_model_tier_cost_bands.cache_clear()
105
106
107 def test_paid_derivation_all_bands() -> None:
108 bands = {
109 "deep-reasoning": "high",
110 "standard": "moderate",
111 "fast": "low",
112 "local-offline": "free",
113 }
114 assert derive_cost_view("deep-reasoning", bands) == ("high", True)
115 assert derive_cost_view("standard", bands) == ("moderate", True)
116 assert derive_cost_view("fast", bands) == ("low", True)
117 assert derive_cost_view("local-offline", bands) == ("free", False)
118
119
120 def test_paid_derivation_human_unpaid() -> None:
121 assert derive_cost_view("human", {}) == ("free", False)
122
123
124 def test_paid_derivation_absent_band_conservative() -> None:
125 assert derive_cost_view("standard", {"standard": None}) == ("unknown", True)
126
127
128 def test_cost_class_ordinal_order() -> None:
129 assert COST_CLASS_ORDER["free"] < COST_CLASS_ORDER["low"]
130 assert COST_CLASS_ORDER["low"] < COST_CLASS_ORDER["moderate"]
131 assert COST_CLASS_ORDER["moderate"] < COST_CLASS_ORDER["high"]
132
133
134 def test_cost_awareness_config_defaults() -> None:
135 config = load_config(FIXTURES / "config-git-only.yaml")
136 assert config.cost_awareness.enabled is False
137 assert config.cost_awareness.surfaces == frozenset({"status", "governance-sync"})
138
139
140 def test_cost_awareness_unknown_surface_exit_2(tmp_path: Path) -> None:
141 write = FIXTURES / "config-git-only.yaml"
142 cfg = tmp_path / "config.yaml"
143 data = yaml.safe_load(write.read_text(encoding="utf-8"))
144 data["cost_awareness"] = {"enabled": True, "surfaces": ["handover-paste"]}
145 cfg.write_text(yaml.safe_dump(data), encoding="utf-8")
146 with pytest.raises(ConfigError, match="status\\|governance-sync"):
147 load_config(cfg)
148
149
150 def test_phase_model_label_normalization() -> None:
151 label_ids = frozenset({"thinking", "auto"})
152 assert normalize_phase_tier("Thinking", label_ids=label_ids) == "thinking"
153 assert normalize_phase_tier("Auto", label_ids=label_ids) == "auto"
154 assert normalize_phase_tier("Operator + Auto", label_ids=label_ids) is None
155
156
157 def test_pending_gate_mapping() -> None:
158 pending = (
159 PendingGate(
160 gate_id="build_verification",
161 phase_id="Demo Auto",
162 artifact=None,
163 message="msg",
164 invoke="invoke",
165 ),
166 PendingGate(
167 gate_id="freeze_review",
168 phase_id="Demo Thinking",
169 artifact="docs/x.md",
170 message="msg",
171 invoke="invoke",
172 ),
173 )
174 assert gate_for_phase(pending, "Demo Thinking") == "freeze_review"
175 assert gate_for_phase(pending, "Demo Auto") == "build_verification"
176 assert gate_for_phase(pending, "Other") is None
177
178
179 def test_derivation_is_pure_no_io() -> None:
180 bands = {"standard": "moderate"}
181 first = derive_cost_view("standard", bands)
182 second = derive_cost_view("standard", bands)
183 assert first == second == ("moderate", True)
File History 1 commit
sha256:6abcf1fa82a7a621ccbc945f19acdba5bc0db54569599404a1452fb4a096a199 fix(ISR): default require_independent_second_reviewer to require Human minor ⚠ 2 days ago