aaronrene / overseer-kit public
test_cost_awareness_integration.py python
153 lines 4.7 KB
Raw
sha256:0e9549ec7b463911bc08b7d586dc320b1ac9b1f5c943ee7e3865dcc6cb0f6f83 chore(governance): sync handover+roadmap to 84db8c8 (drift:… Human 2 days ago
1 """Integration tests for Track P / P-cost CLI (§PC.9)."""
2
3 from __future__ import annotations
4
5 import json
6 from pathlib import Path
7
8 import pytest
9 import yaml
10
11 from cli.kit_root import kit_root
12 from tests.fixtures.cost_awareness import seed_cost_awareness_repo
13 from tests.fixtures.model_routing import seed_routing_repo
14 from tests.support import FIXTURES, git_status_runner, run_cli
15 from tools.model_routing.labels import RoutingPolicyError
16
17
18 def _write_lock(repo_root: Path) -> None:
19 (repo_root / ".overseer" / "version.lock").write_text(
20 "lock_version: 1\nkit_version: 0.1.0\nconfig_version: 1\n"
21 "footprint_digest: sha256:0\ninstalled_at: '2026-01-01'\n"
22 "synced_at: '2026-01-01'\nfootprint: []\n",
23 encoding="utf-8",
24 )
25
26
27 def test_route_emits_cost_annotation(tmp_path: Path, capsys) -> None:
28 seed_routing_repo(tmp_path)
29 code = run_cli(
30 ["route", "--phase-tier", "auto", "--json"],
31 cwd=tmp_path,
32 runner=git_status_runner(),
33 kit=kit_root(),
34 json_mode=True,
35 )
36 assert code == 0
37 payload = json.loads(capsys.readouterr().out)
38 assert payload["route_id"] == "auto-build"
39 assert payload["model_tier"] == "standard"
40 assert payload["cost_class"] == "moderate"
41 assert payload["paid_step_before_spend"] is True
42
43
44 def test_route_resolution_unchanged_with_cost_fields(tmp_path: Path, capsys) -> None:
45 seed_routing_repo(tmp_path)
46 run_cli(
47 ["route", "--position", "overseer", "--json"],
48 cwd=tmp_path,
49 runner=git_status_runner(),
50 kit=kit_root(),
51 json_mode=True,
52 )
53 payload = json.loads(capsys.readouterr().out)
54 assert payload["route_id"] == "overseer-ruling"
55 assert payload["model_tier"] == "deep-reasoning"
56 assert payload["fallback"] == ["deep-reasoning", "human"]
57
58
59 def test_route_malformed_cost_class_exit_32(tmp_path: Path, monkeypatch) -> None:
60 seed_routing_repo(tmp_path)
61
62 def _raise(*args, **kwargs):
63 raise RoutingPolicyError("bad cost_class", exit_code=32)
64
65 monkeypatch.setattr("cli.commands.route.load_model_tier_cost_bands", _raise)
66 assert (
67 run_cli(
68 ["route", "--phase-tier", "auto"],
69 cwd=tmp_path,
70 runner=git_status_runner(),
71 kit=kit_root(),
72 )
73 == 32
74 )
75
76
77 def test_status_disabled_byte_identical_no_cost_key(tmp_path: Path, capsys) -> None:
78 seed_cost_awareness_repo(tmp_path, enabled=False)
79 _write_lock(tmp_path)
80 code = run_cli(
81 ["status", "--json"],
82 cwd=tmp_path,
83 runner=git_status_runner(),
84 kit=kit_root(),
85 json_mode=True,
86 )
87 assert code == 0
88 payload = json.loads(capsys.readouterr().out)
89 assert "cost_awareness" not in payload
90
91
92 def test_status_enabled_emits_cost_surface(tmp_path: Path, capsys) -> None:
93 from tests.fixtures.cost_awareness import seed_cost_e2e_repo
94
95 seed_cost_e2e_repo(tmp_path)
96 code = run_cli(
97 ["status", "--json"],
98 cwd=tmp_path,
99 runner=git_status_runner(),
100 kit=kit_root(),
101 json_mode=True,
102 )
103 assert code == 0
104 payload = json.loads(capsys.readouterr().out)
105 assert payload["cost_awareness"]["enabled"] is True
106 assert payload["cost_awareness"]["slices"]
107
108
109 def test_status_missing_policy_exit_31(tmp_path: Path) -> None:
110 from tests.support import write_config
111
112 write_config(tmp_path, "config-git-only.yaml")
113 cfg_path = tmp_path / ".overseer" / "config.yaml"
114 data = yaml.safe_load(cfg_path.read_text(encoding="utf-8"))
115 data["cost_awareness"] = {"enabled": True}
116 cfg_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
117 _write_lock(tmp_path)
118 assert (
119 run_cli(
120 ["status"],
121 cwd=tmp_path,
122 runner=git_status_runner(),
123 kit=kit_root(),
124 )
125 == 31
126 )
127
128
129 def test_status_invalid_cost_metadata_warning_only(tmp_path: Path, capsys, monkeypatch) -> None:
130 seed_cost_awareness_repo(tmp_path, enabled=True)
131 _write_lock(tmp_path)
132 from tests.fixtures.model_routing import copy_default_routing_policy
133
134 copy_default_routing_policy(tmp_path)
135
136 def _raise(*args, **kwargs):
137 raise RoutingPolicyError("bad cost_class", exit_code=32)
138
139 monkeypatch.setattr(
140 "tools.cost_awareness.surface.load_model_tier_cost_bands",
141 _raise,
142 )
143 code = run_cli(
144 ["status", "--json"],
145 cwd=tmp_path,
146 runner=git_status_runner(),
147 kit=kit_root(),
148 json_mode=True,
149 )
150 assert code == 0
151 payload = json.loads(capsys.readouterr().out)
152 assert payload["cost_awareness"]["invalid"] is True
153 assert any("cost_awareness: invalid" in warning for warning in payload["warnings"])
File History 1 commit
sha256:6abcf1fa82a7a621ccbc945f19acdba5bc0db54569599404a1452fb4a096a199 fix(ISR): default require_independent_second_reviewer to require Human minor ⚠ 2 days ago