aaronrene / overseer-kit public
test_model_routing_unit.py python
171 lines 5.3 KB
Raw
sha256:a78e7e5a8740e03315f325d19edeb3aa1b306b3337d04abbaa9a9e0f3bbeb7a1 docs: MuseHub-first before ISR #74 — staging solidify NEXT Human 11 hours ago
1 """Unit tests for Track P / P-route model routing (§PR.8)."""
2
3 from __future__ import annotations
4
5 from pathlib import Path
6
7 import pytest
8 import yaml
9
10 from adapters.errors import ConfigError
11 from cli.kit_root import kit_root
12 from tests.fixtures.model_routing import minimal_valid_policy_yaml, write_routing_policy
13 from tools.model_routing.labels import (
14 load_model_tier_ids,
15 validate_model_tier_entry,
16 validate_model_tiers_document,
17 )
18 from tools.model_routing.policy import RoutingPolicyError, load_routing_policy_text, parse_routing_policy
19 from tools.model_routing.resolve import resolve_route, selector_matches
20 from tools.model_routing.types import RouteSelector
21
22
23 def test_model_tiers_required_fields() -> None:
24 with pytest.raises(ConfigError, match="display"):
25 validate_model_tier_entry({"id": "fast"}, index=0, path="policy/model-labels.yaml")
26
27
28 def test_model_tiers_unique_kebab_ids() -> None:
29 raw = {
30 "model_tiers": [
31 {"id": "fast", "display": "Fast", "meaning": "quick"},
32 {"id": "fast", "display": "Fast2", "meaning": "dup"},
33 ]
34 }
35 with pytest.raises(ConfigError, match="unique"):
36 validate_model_tiers_document(raw, path="policy/model-labels.yaml")
37
38
39 def test_model_tiers_reject_vendor_slug_id() -> None:
40 with pytest.raises(ConfigError, match="vendor slug"):
41 validate_model_tier_entry(
42 {"id": "gpt-fast", "display": "Bad", "meaning": "bad"},
43 index=0,
44 path="policy/model-labels.yaml",
45 )
46
47
48 def test_kit_model_tiers_load() -> None:
49 ids = load_model_tier_ids(kit_root())
50 assert ids == frozenset({"deep-reasoning", "standard", "fast", "local-offline"})
51
52
53 def test_routing_policy_requires_defaults() -> None:
54 text = "version: 1\nroutes: []\n"
55 with pytest.raises(RoutingPolicyError, match="defaults"):
56 load_routing_policy_text(text, kit_root=kit_root(), citation="policy.yaml")
57
58
59 def test_routing_policy_rejects_unknown_when_key() -> None:
60 text = """
61 version: 1
62 defaults:
63 model_tier: standard
64 fallback: [standard, human]
65 routes:
66 - id: bad
67 when: { extra: x }
68 model_tier: standard
69 fallback: [standard, human]
70 """
71 with pytest.raises(RoutingPolicyError, match="unknown"):
72 load_routing_policy_text(text, kit_root=kit_root(), citation="policy.yaml")
73
74
75 def test_routing_policy_fallback_must_start_with_model_tier() -> None:
76 text = """
77 version: 1
78 defaults:
79 model_tier: standard
80 fallback: [fast, human]
81 """
82 with pytest.raises(RoutingPolicyError, match="fallback\\[0\\]"):
83 load_routing_policy_text(text, kit_root=kit_root(), citation="policy.yaml")
84
85
86 def test_routing_policy_fallback_must_end_with_human() -> None:
87 text = """
88 version: 1
89 defaults:
90 model_tier: standard
91 fallback: [standard, fast]
92 """
93 with pytest.raises(RoutingPolicyError, match="terminate"):
94 load_routing_policy_text(text, kit_root=kit_root(), citation="policy.yaml")
95
96
97 def test_routing_policy_rejects_unknown_model_tier() -> None:
98 text = """
99 version: 1
100 defaults:
101 model_tier: unknown-tier
102 fallback: [unknown-tier, human]
103 """
104 with pytest.raises(RoutingPolicyError, match="not in model_tiers"):
105 load_routing_policy_text(text, kit_root=kit_root(), citation="policy.yaml")
106
107
108 def test_routing_policy_duplicate_route_id() -> None:
109 text = """
110 version: 1
111 defaults:
112 model_tier: standard
113 fallback: [standard, human]
114 routes:
115 - id: dup
116 when: {}
117 model_tier: standard
118 fallback: [standard, human]
119 - id: dup
120 when: {}
121 model_tier: fast
122 fallback: [fast, human]
123 """
124 with pytest.raises(RoutingPolicyError, match="duplicate"):
125 load_routing_policy_text(text, kit_root=kit_root(), citation="policy.yaml")
126
127
128 def test_selector_wildcard_and_first_match() -> None:
129 policy = parse_routing_policy(
130 yaml.safe_load(minimal_valid_policy_yaml()),
131 kit_root=kit_root(),
132 citation="policy/model-routing.yaml",
133 )
134 assert not selector_matches(RouteSelector(position="overseer"), RouteSelector())
135 assert selector_matches(
136 RouteSelector(position="overseer"),
137 RouteSelector(position="overseer"),
138 )
139 decision = resolve_route(policy, RouteSelector(position="overseer"))
140 assert decision.route_id == "overseer-ruling"
141 assert decision.model_tier == "deep-reasoning"
142
143
144 def test_defaults_fallthrough() -> None:
145 policy = parse_routing_policy(
146 yaml.safe_load(minimal_valid_policy_yaml()),
147 kit_root=kit_root(),
148 citation="policy/model-routing.yaml",
149 )
150 decision = resolve_route(policy, RouteSelector(position="unknown-seat"))
151 assert decision.route_id == "defaults"
152 assert decision.model_tier == "standard"
153 assert decision.fallback == ("standard", "human")
154
155
156 def test_resolution_is_pure_no_io(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
157 policy = parse_routing_policy(
158 yaml.safe_load(minimal_valid_policy_yaml()),
159 kit_root=kit_root(),
160 citation="policy/model-routing.yaml",
161 )
162 query = RouteSelector(phase_tier="auto")
163
164 def _boom(*_args, **_kwargs):
165 raise AssertionError("resolution must not perform I/O")
166
167 monkeypatch.setattr(Path, "read_text", _boom)
168 first = resolve_route(policy, query)
169 second = resolve_route(policy, query)
170 assert first == second
171 assert first.route_id == "auto-build"
File History 1 commit
sha256:a78e7e5a8740e03315f325d19edeb3aa1b306b3337d04abbaa9a9e0f3bbeb7a1 docs: MuseHub-first before ISR #74 — staging solidify NEXT Human 11 hours ago