gabriel / muse public
test_contract_supercharge.py python
395 lines 15.1 KB
Raw
sha256:81ae324db5ad375fbfe4834c6fcb378312cafad3cc92dec5d3e5c427306621a2 fix: remove commit_exists filter from have anchors — server… Sonnet 4.6 patch 21 days ago
1 """Supercharge tests for ``muse code contract`` — agent-usability gaps.
2
3 The existing TestContract suite in test_code_commands.py covers correctness,
4 JSON schema, all field keys, parameter schema, history schema, stability values,
5 arg observations, and error paths. This file targets only the gaps those
6 tests leave open:
7
8 Coverage matrix
9 ---------------
10 - --json / -j: -j alias works identically to --json
11 - exit_code: JSON output includes exit_code = 0 on success
12 - duration_ms: JSON output includes non-negative float duration_ms
13 - TypedDicts: _ContractJson gains exit_code/duration_ms annotations
14 - Docstrings: run() docstring mentions exit_code and duration_ms
15 - ANSI: JSON output never contains terminal escape sequences
16 - Performance: duration_ms stays under 2000 ms for a small repo
17 """
18
19 from __future__ import annotations
20 from collections.abc import Mapping
21
22 import json
23 import os
24 import pathlib
25 import textwrap
26
27 import pytest
28
29 from tests.cli_test_helper import CliRunner, InvokeResult
30
31 runner = CliRunner()
32
33 _ADDR = "billing.py::compute_total"
34
35
36 # ---------------------------------------------------------------------------
37 # Helpers
38 # ---------------------------------------------------------------------------
39
40
41 def _env(root: pathlib.Path) -> Mapping[str, str]:
42 return {"MUSE_REPO_ROOT": str(root)}
43
44
45 def _run(root: pathlib.Path, *args: str) -> InvokeResult:
46 return runner.invoke(None, list(args), env=_env(root))
47
48
49 # ---------------------------------------------------------------------------
50 # Fixture — multi-commit repo with a real call graph + test assertions
51 # ---------------------------------------------------------------------------
52
53
54 @pytest.fixture()
55 def contract_repo(
56 tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
57 ) -> pathlib.Path:
58 """Repo exercising every dimension of ``muse code contract``.
59
60 Layout::
61
62 billing.py — compute_total(items, currency="USD") → float
63 services.py — place_order() calls compute_total → stored
64 audit.py — run_audit() calls compute_total → discarded
65 tests/test_billing.py — test functions with assertions
66
67 Commit history::
68
69 1. readme.txt seed commit
70 2. billing.py added — compute_total created
71 3. callers + tests added
72 4. billing.py body rewrite (PATCH)
73 5. billing.py currency param added (MINOR)
74 """
75 monkeypatch.chdir(tmp_path)
76
77 r = _run(tmp_path, "init", "--domain", "code")
78 assert r.exit_code == 0, r.output
79
80 # commit 1 — seed
81 (tmp_path / "readme.txt").write_text("# contract test repo\n")
82 r = _run(tmp_path, "code", "add", ".")
83 assert r.exit_code == 0, r.output
84 r = _run(tmp_path, "commit", "-m", "seed: initial readme")
85 assert r.exit_code == 0, r.output
86
87 # commit 2 — add compute_total
88 (tmp_path / "billing.py").write_text(textwrap.dedent("""\
89 def compute_total(items):
90 return sum(i["price"] for i in items)
91 """))
92 r = _run(tmp_path, "code", "add", ".")
93 assert r.exit_code == 0, r.output
94 r = _run(tmp_path, "commit", "-m", "feat: add compute_total")
95 assert r.exit_code == 0, r.output
96
97 # commit 3 — callers + tests
98 os.makedirs(tmp_path / "tests", exist_ok=True)
99 (tmp_path / "services.py").write_text(textwrap.dedent("""\
100 from billing import compute_total
101
102 def place_order(items):
103 total = compute_total(items, currency="EUR")
104 return total
105 """))
106 (tmp_path / "audit.py").write_text(textwrap.dedent("""\
107 from billing import compute_total
108
109 def run_audit(items):
110 compute_total(items)
111 """))
112 (tmp_path / "tests" / "test_billing.py").write_text(textwrap.dedent("""\
113 from billing import compute_total
114
115 def test_compute_total_basic():
116 result = compute_total([{"price": 10}, {"price": 5}])
117 assert result == 15
118 assert result > 0
119 assert isinstance(result, (int, float))
120
121 def test_compute_total_empty():
122 result = compute_total([])
123 assert result == 0
124 """))
125 r = _run(tmp_path, "code", "add", ".")
126 assert r.exit_code == 0, r.output
127 r = _run(tmp_path, "commit", "-m", "feat: add callers and tests")
128 assert r.exit_code == 0, r.output
129
130 # commit 4 — body rewrite (PATCH)
131 (tmp_path / "billing.py").write_text(textwrap.dedent("""\
132 def compute_total(items):
133 total = 0.0
134 for item in items:
135 total += float(item["price"])
136 return total
137 """))
138 r = _run(tmp_path, "code", "add", ".")
139 assert r.exit_code == 0, r.output
140 r = _run(tmp_path, "commit", "-m", "perf: vectorise compute_total")
141 assert r.exit_code == 0, r.output
142
143 # commit 5 — add currency param (MINOR)
144 (tmp_path / "billing.py").write_text(textwrap.dedent("""\
145 def compute_total(items, currency="USD"):
146 total = 0.0
147 for item in items:
148 total += float(item["price"])
149 return total
150 """))
151 r = _run(tmp_path, "code", "add", ".")
152 assert r.exit_code == 0, r.output
153 r = _run(tmp_path, "commit", "-m", "feat: add optional currency param")
154 assert r.exit_code == 0, r.output
155
156 return tmp_path
157
158
159 # ---------------------------------------------------------------------------
160 # TestJsonAlias — -j works identically to --json
161 # ---------------------------------------------------------------------------
162
163
164 class TestJsonAlias:
165 """-j shorthand must behave identically to --json."""
166
167 def test_j_alias_exits_zero(self, contract_repo: pathlib.Path) -> None:
168 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
169 assert r.exit_code == 0, r.output
170
171 def test_j_alias_valid_json(self, contract_repo: pathlib.Path) -> None:
172 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
173 json.loads(r.output) # must not raise
174
175 def test_j_alias_has_address_key(self, contract_repo: pathlib.Path) -> None:
176 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
177 assert "address" in json.loads(r.output)
178
179 def test_j_alias_has_ops_key(self, contract_repo: pathlib.Path) -> None:
180 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
181 data = json.loads(r.output)
182 assert "stability" in data
183
184 def test_j_alias_same_top_level_keys_as_json_flag(
185 self, contract_repo: pathlib.Path
186 ) -> None:
187 r1 = _run(contract_repo, "code", "contract", _ADDR, "--json")
188 r2 = _run(contract_repo, "code", "contract", _ADDR, "-j")
189 d1 = json.loads(r1.output)
190 d2 = json.loads(r2.output)
191 d1.pop("duration_ms", None)
192 d2.pop("duration_ms", None)
193 assert set(d1.keys()) == set(d2.keys())
194
195 def test_j_alias_address_matches(self, contract_repo: pathlib.Path) -> None:
196 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
197 assert json.loads(r.output)["address"] == _ADDR
198
199 def test_j_alias_has_history_key(self, contract_repo: pathlib.Path) -> None:
200 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
201 assert "history" in json.loads(r.output)
202
203
204 # ---------------------------------------------------------------------------
205 # TestDurationMs — JSON output must include duration_ms
206 # ---------------------------------------------------------------------------
207
208
209 class TestDurationMs:
210 """JSON output must include a non-negative float duration_ms."""
211
212 def test_json_has_duration_ms(self, contract_repo: pathlib.Path) -> None:
213 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
214 assert "duration_ms" in json.loads(r.output)
215
216 def test_json_duration_ms_nonnegative(self, contract_repo: pathlib.Path) -> None:
217 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
218 assert json.loads(r.output)["duration_ms"] >= 0
219
220 def test_json_duration_ms_is_float(self, contract_repo: pathlib.Path) -> None:
221 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
222 assert isinstance(json.loads(r.output)["duration_ms"], float)
223
224 def test_j_alias_duration_ms_present(self, contract_repo: pathlib.Path) -> None:
225 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
226 assert "duration_ms" in json.loads(r.output)
227
228 def test_duration_ms_with_max_commits_1(self, contract_repo: pathlib.Path) -> None:
229 """duration_ms present even with --max-commits 1."""
230 r = _run(contract_repo, "code", "contract", _ADDR, "--json", "--max-commits", "1")
231 data = json.loads(r.output)
232 assert "duration_ms" in data
233 assert data["duration_ms"] >= 0
234
235 def test_duration_ms_not_zero_for_real_work(self, contract_repo: pathlib.Path) -> None:
236 """Non-trivial analysis should take measurable time."""
237 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
238 # Allow 0.0 only in very fast CI — just confirm type and sign
239 assert isinstance(json.loads(r.output)["duration_ms"], float)
240
241
242 # ---------------------------------------------------------------------------
243 # TestExitCode — JSON includes exit_code = 0 on success
244 # ---------------------------------------------------------------------------
245
246
247 class TestExitCode:
248 """JSON exit_code must be 0 on success."""
249
250 def test_json_has_exit_code(self, contract_repo: pathlib.Path) -> None:
251 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
252 assert "exit_code" in json.loads(r.output)
253
254 def test_json_exit_code_zero(self, contract_repo: pathlib.Path) -> None:
255 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
256 assert r.exit_code == 0
257 assert json.loads(r.output)["exit_code"] == 0
258
259 def test_json_exit_code_is_int(self, contract_repo: pathlib.Path) -> None:
260 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
261 assert isinstance(json.loads(r.output)["exit_code"], int)
262
263 def test_j_alias_exit_code_present(self, contract_repo: pathlib.Path) -> None:
264 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
265 assert "exit_code" in json.loads(r.output)
266
267 def test_exit_code_mirrors_process_exit(self, contract_repo: pathlib.Path) -> None:
268 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
269 assert json.loads(r.output)["exit_code"] == r.exit_code
270
271 def test_exit_code_zero_with_max_commits(self, contract_repo: pathlib.Path) -> None:
272 r = _run(contract_repo, "code", "contract", _ADDR, "--json", "--max-commits", "3")
273 assert r.exit_code == 0
274 assert json.loads(r.output)["exit_code"] == 0
275
276 def test_exit_code_not_present_in_error_path(
277 self, contract_repo: pathlib.Path
278 ) -> None:
279 """Error paths raise SystemExit before JSON emits — no JSON to check."""
280 r = _run(contract_repo, "code", "contract", "billing.py::nonexistent", "--json")
281 assert r.exit_code != 0
282
283
284 # ---------------------------------------------------------------------------
285 # TestTypedDicts — _ContractJson carries the new fields
286 # ---------------------------------------------------------------------------
287
288
289 class TestTypedDicts:
290 """_ContractJson must carry exit_code and duration_ms annotations."""
291
292 def test_contract_json_typeddict_exists(self) -> None:
293 from muse.cli.commands.contract import _ContractJson # noqa: F401
294
295 def test_has_exit_code_annotation(self) -> None:
296 from muse.cli.commands.contract import _ContractJson
297 assert "exit_code" in _ContractJson.__annotations__
298
299 def test_has_duration_ms_annotation(self) -> None:
300 from muse.cli.commands.contract import _ContractJson
301 assert "duration_ms" in _ContractJson.__annotations__
302
303 def test_retains_address_annotation(self) -> None:
304 from muse.cli.commands.contract import _ContractJson
305 assert "address" in _ContractJson.__annotations__
306
307 def test_retains_history_annotation(self) -> None:
308 from muse.cli.commands.contract import _ContractJson
309 assert "history" in _ContractJson.__annotations__
310
311 def test_retains_stability_annotation(self) -> None:
312 from muse.cli.commands.contract import _ContractJson
313 assert "stability" in _ContractJson.__annotations__
314
315 def test_retains_warnings_annotation(self) -> None:
316 from muse.cli.commands.contract import _ContractJson
317 assert "warnings" in _ContractJson.__annotations__
318
319
320 # ---------------------------------------------------------------------------
321 # TestAnsiSanitization — no escape codes in JSON output
322 # ---------------------------------------------------------------------------
323
324
325 class TestAnsiSanitization:
326 """No ANSI escape sequences anywhere in the JSON output."""
327
328 def test_json_output_no_ansi(self, contract_repo: pathlib.Path) -> None:
329 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
330 assert "\x1b" not in r.output
331
332 def test_j_alias_output_no_ansi(self, contract_repo: pathlib.Path) -> None:
333 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
334 assert "\x1b" not in r.output
335
336 def test_json_output_no_ansi_with_max_commits(
337 self, contract_repo: pathlib.Path
338 ) -> None:
339 r = _run(
340 contract_repo, "code", "contract", _ADDR,
341 "--json", "--max-commits", "2",
342 )
343 assert "\x1b" not in r.output
344
345
346 # ---------------------------------------------------------------------------
347 # TestPerformance — duration_ms under 2000 ms for a small repo
348 # ---------------------------------------------------------------------------
349
350
351 class TestPerformance:
352 """duration_ms must stay under 2000 ms for small repos."""
353
354 def test_json_duration_under_2000ms(self, contract_repo: pathlib.Path) -> None:
355 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
356 assert json.loads(r.output)["duration_ms"] < 2000
357
358 def test_j_alias_duration_under_2000ms(self, contract_repo: pathlib.Path) -> None:
359 r = _run(contract_repo, "code", "contract", _ADDR, "-j")
360 assert json.loads(r.output)["duration_ms"] < 2000
361
362 def test_duration_ms_is_float_not_int(self, contract_repo: pathlib.Path) -> None:
363 r = _run(contract_repo, "code", "contract", _ADDR, "--json")
364 assert isinstance(json.loads(r.output)["duration_ms"], float)
365
366
367 # ---------------------------------------------------------------------------
368 # TestRegisterFlags
369 # ---------------------------------------------------------------------------
370
371
372 import argparse as _argparse
373
374
375 class TestRegisterFlags:
376 """register() wires --json / -j correctly."""
377
378 def _parse(self, *args: str) -> _argparse.Namespace:
379 from muse.cli.commands.contract import register
380 p = _argparse.ArgumentParser()
381 sub = p.add_subparsers()
382 register(sub)
383 return p.parse_args(["contract", *args])
384
385 def test_default_json_out_is_false(self) -> None:
386 ns = self._parse("src/billing.py::compute_total")
387 assert ns.json_out is False
388
389 def test_json_flag_sets_json_out(self) -> None:
390 ns = self._parse("--json", "src/billing.py::compute_total")
391 assert ns.json_out is True
392
393 def test_j_shorthand_sets_json_out(self) -> None:
394 ns = self._parse("-j", "src/billing.py::compute_total")
395 assert ns.json_out is True
File History 4 commits
sha256:81ae324db5ad375fbfe4834c6fcb378312cafad3cc92dec5d3e5c427306621a2 fix: remove commit_exists filter from have anchors — server… Sonnet 4.6 patch 21 days ago
sha256:36c3cb3e76619d4c30a6d9bf81b5ec4ff148e30dcfed913e3114ca7b43b81c7e fix: rename objects→blobs in push client and all stale test… Sonnet 4.6 patch 23 days ago
sha256:c06a9b9b9fee26c68ea725b44d54b2c0a171301ce9de746d5b656617b4463a9a fix: repair four test failures from post-migration audit Sonnet 4.6 patch 29 days ago
sha256:1900655993c83c4107067375548a7be823e471d2515830842f1a12cba4bd3cdf fix: unified object store migration — idempotent writes, JS… Sonnet 4.6 minor ⚠ 29 days ago