aaronrene / overseer-kit public
test_model_routing_stress.py python
75 lines 2.3 KB
Raw
sha256:0e9549ec7b463911bc08b7d586dc320b1ac9b1f5c943ee7e3865dcc6cb0f6f83 chore(governance): sync handover+roadmap to 84db8c8 (drift:… Human 2 days ago
1 """Stress tests for Track P / P-route (§PR.8)."""
2
3 from __future__ import annotations
4
5 import time
6
7 import yaml
8
9 from cli.kit_root import kit_root
10 from tools.model_routing.policy import load_routing_policy_text, parse_routing_policy
11 from tools.model_routing.resolve import resolve_route
12 from tools.model_routing.types import RouteSelector
13
14 BOUND_MS = 500
15
16
17 def _large_policy_yaml(route_count: int = 250) -> str:
18 lines = [
19 "version: 1",
20 "defaults:",
21 " model_tier: standard",
22 " fallback: [standard, human]",
23 "routes:",
24 ]
25 for index in range(route_count):
26 lines.extend(
27 [
28 f" - id: route-{index}",
29 f" when: {{ position: seat-{index % 17}, gate: default }}",
30 " model_tier: fast",
31 " fallback: [fast, standard, human]",
32 ]
33 )
34 lines.extend(
35 [
36 " - id: terminal-match",
37 " when: { position: target-seat }",
38 " model_tier: deep-reasoning",
39 " fallback: [deep-reasoning, human]",
40 ]
41 )
42 return "\n".join(lines) + "\n"
43
44
45 def test_large_policy_resolves_within_bound() -> None:
46 text = _large_policy_yaml()
47 policy = parse_routing_policy(
48 yaml.safe_load(text),
49 kit_root=kit_root(),
50 citation="large.yaml",
51 )
52 query = RouteSelector(position="target-seat")
53 start = time.perf_counter()
54 decision = resolve_route(policy, query)
55 elapsed_ms = (time.perf_counter() - start) * 1000
56 assert decision.route_id == "terminal-match"
57 assert elapsed_ms < BOUND_MS
58
59
60 def test_large_policy_validate_within_bound() -> None:
61 text = _large_policy_yaml()
62 start = time.perf_counter()
63 policy = load_routing_policy_text(text, kit_root=kit_root(), citation="large.yaml")
64 elapsed_ms = (time.perf_counter() - start) * 1000
65 assert policy.routes[-1].id == "terminal-match"
66 assert elapsed_ms < BOUND_MS
67
68
69 def test_match_result_order_independent_under_fixed_policy() -> None:
70 text = _large_policy_yaml(route_count=40)
71 policy = load_routing_policy_text(text, kit_root=kit_root(), citation="large.yaml")
72 query = RouteSelector(position="seat-3", gate="default")
73 first = resolve_route(policy, query)
74 second = resolve_route(policy, query)
75 assert first == second
File History 1 commit
sha256:6abcf1fa82a7a621ccbc945f19acdba5bc0db54569599404a1452fb4a096a199 fix(ISR): default require_independent_second_reviewer to require Human minor ⚠ 2 days ago