gabriel / muse public

test_cmd_diff_hardening.py file-level

at sha256:8 · View file ↗ · Intel ↗

History
1 files
1 commits
0 hotspots
0 🧊 dead
0 💥 blast risk
sha256:b adding issues docs to bust staging mpack prebuild cache. · gabriel · Jun 20, 2026
1 """Hardening test suite for ``muse diff``.
2
3 Coverage tiers:
4 - JSON schema: duration_ms, exit_code, summary always present on every path
5 - duration_ms: type, non-negative, under 10 s
6 - exit_code: always 0 in payload on success
7 - summary field: correct human string, empty on clean tree
8 - staged integration: --staged --json, --stat, --text
9 - unstaged integration: --unstaged --json
10 - symbols field: structure validated for real diffs
11 - sem_ver_bump / breaking_changes: present in JSON output
12 - conflict JSON: duration_ms, exit_code, all fields
13 - security: ANSI in commit IDs sanitized in error paths
14 - stress: 200-file diff under 5 s
15 """
16
17 from __future__ import annotations
18
19 import json
20 import os
21 import pathlib
22
23 import pytest
24
25 from tests.cli_test_helper import CliRunner, InvokeResult
26
27 runner = CliRunner()
28
29
30 # ---------------------------------------------------------------------------
31 # Helpers
32 # ---------------------------------------------------------------------------
33
34
35 def _invoke(repo: pathlib.Path, args: list[str]) -> InvokeResult:
36 saved = os.getcwd()
37 try:
38 os.chdir(repo)
39 return runner.invoke(None, args)
40 finally:
41 os.chdir(saved)
42
43
44 def _diff(repo: pathlib.Path, *extra: str) -> InvokeResult:
45 return _invoke(repo, ["diff", *extra])
46
47
48 def _commit(repo: pathlib.Path, msg: str = "commit") -> None:
49 _invoke(repo, ["commit", "-m", msg])
50
51
52 def _stage(repo: pathlib.Path, path: str) -> None:
53 _invoke(repo, ["code", "add", path])
54
55
56 @pytest.fixture()
57 def repo(tmp_path: pathlib.Path) -> pathlib.Path:
58 saved = os.getcwd()
59 try:
60 os.chdir(tmp_path)
61 runner.invoke(None, ["init"])
62 finally:
63 os.chdir(saved)
64 (tmp_path / "a.py").write_text("x = 1\n")
65 _invoke(tmp_path, ["commit", "-m", "first"])
66 return tmp_path
67
68
69 # ---------------------------------------------------------------------------
70 # JSON schema: duration_ms + exit_code + summary always present
71 # ---------------------------------------------------------------------------
72
73
74 class TestJsonSchemaComplete:
75 """Every --json path emits duration_ms, exit_code, and summary."""
76
77 def test_all_fields_clean_tree(self, repo: pathlib.Path) -> None:
78 r = _diff(repo, "--json")
79 assert r.exit_code == 0
80 data = json.loads(r.output)
81 for field in ("duration_ms", "exit_code", "summary",
82 "from_ref", "to_ref", "has_changes",
83 "added", "deleted", "modified", "total_changes"):
84 assert field in data, f"Missing: {field}"
85
86 def test_all_fields_dirty_tree(self, repo: pathlib.Path) -> None:
87 (repo / "a.py").write_text("x = 99\n")
88 r = _diff(repo, "--json")
89 assert r.exit_code == 0
90 data = json.loads(r.output)
91 for field in ("duration_ms", "exit_code", "summary"):
92 assert field in data, f"Missing: {field}"
93
94 def test_all_fields_two_commit(self, repo: pathlib.Path) -> None:
95 (repo / "b.py").write_text("y = 2\n")
96 _invoke(repo, ["commit", "-m", "second"])
97 r = _invoke(repo, ["diff", "HEAD"])
98 # text output, no --json needed here — just verify JSON path explicitly
99 r = _diff(repo, "--json")
100 assert r.exit_code == 0
101 data = json.loads(r.output)
102 assert "duration_ms" in data
103 assert "exit_code" in data
104
105 def test_all_fields_staged(self, repo: pathlib.Path) -> None:
106 (repo / "a.py").write_text("staged change\n")
107 _stage(repo, "a.py")
108 r = _diff(repo, "--staged", "--json")
109 assert r.exit_code == 0
110 data = json.loads(r.output)
111 for field in ("duration_ms", "exit_code", "summary"):
112 assert field in data, f"Missing: {field}"
113
114 def test_all_fields_unstaged(self, repo: pathlib.Path) -> None:
115 (repo / "a.py").write_text("staged\n")
116 _stage(repo, "a.py")
117 (repo / "a.py").write_text("unstaged on top\n")
118 r = _diff(repo, "--unstaged", "--json")
119 assert r.exit_code == 0
120 data = json.loads(r.output)
121 for field in ("duration_ms", "exit_code", "summary"):
122 assert field in data, f"Missing: {field}"
123
124 def test_exit_code_field_zero(self, repo: pathlib.Path) -> None:
125 r = _diff(repo, "--json")
126 assert r.exit_code == 0
127 assert json.loads(r.output)["exit_code"] == 0
128
129 def test_exit_code_field_zero_with_changes(self, repo: pathlib.Path) -> None:
130 (repo / "a.py").write_text("changed\n")
131 r = _diff(repo, "--json")
132 assert r.exit_code == 0
133 assert json.loads(r.output)["exit_code"] == 0
134
135 def test_json_is_compact(self, repo: pathlib.Path) -> None:
136 r = _diff(repo, "--json")
137 assert "\n" not in r.output.strip()
138
139
140 # ---------------------------------------------------------------------------
141 # duration_ms: type, magnitude, precision
142 # ---------------------------------------------------------------------------
143
144
145 class TestElapsedSeconds:
146
147 def test_is_float(self, repo: pathlib.Path) -> None:
148 r = _diff(repo, "--json")
149 assert isinstance(json.loads(r.output)["duration_ms"], float)
150
151 def test_non_negative(self, repo: pathlib.Path) -> None:
152 r = _diff(repo, "--json")
153 assert json.loads(r.output)["duration_ms"] >= 0.0
154
155 def test_under_ten_seconds(self, repo: pathlib.Path) -> None:
156 r = _diff(repo, "--json")
157 assert json.loads(r.output)["duration_ms"] < 10.0
158
159 def test_present_on_dirty_tree(self, repo: pathlib.Path) -> None:
160 (repo / "a.py").write_text("dirty\n")
161 r = _diff(repo, "--json")
162 assert json.loads(r.output)["duration_ms"] >= 0.0
163
164 def test_present_on_staged(self, repo: pathlib.Path) -> None:
165 (repo / "a.py").write_text("s\n")
166 _stage(repo, "a.py")
167 r = _diff(repo, "--staged", "--json")
168 data = json.loads(r.output)
169 assert "duration_ms" in data
170 assert data["duration_ms"] >= 0.0
171
172 def test_six_decimal_precision(self, repo: pathlib.Path) -> None:
173 r = _diff(repo, "--json")
174 s = str(json.loads(r.output)["duration_ms"])
175 dec = s.split(".")[-1] if "." in s else ""
176 assert len(dec) <= 6
177
178
179 # ---------------------------------------------------------------------------
180 # summary field
181 # ---------------------------------------------------------------------------
182
183
184 class TestSummaryField:
185
186 def test_summary_is_string(self, repo: pathlib.Path) -> None:
187 r = _diff(repo, "--json")
188 assert isinstance(json.loads(r.output)["summary"], str)
189
190 def test_summary_empty_string_on_clean(self, repo: pathlib.Path) -> None:
191 r = _diff(repo, "--json")
192 data = json.loads(r.output)
193 assert data["has_changes"] is False
194 # Clean tree → no summary needed; empty string or "No differences"
195 assert isinstance(data["summary"], str)
196
197 def test_summary_non_empty_when_dirty(self, repo: pathlib.Path) -> None:
198 (repo / "a.py").write_text("modified\n")
199 r = _diff(repo, "--json")
200 data = json.loads(r.output)
201 assert data["has_changes"] is True
202 assert len(data["summary"]) > 0
203
204 def test_summary_contains_change_count(self, repo: pathlib.Path) -> None:
205 (repo / "b.py").write_text("new\n")
206 r = _diff(repo, "--json")
207 data = json.loads(r.output)
208 # delta["summary"] typically contains a digit for change count
209 assert any(c.isdigit() for c in data["summary"]) or data["summary"] == ""
210
211 def test_summary_matches_total_changes(self, repo: pathlib.Path) -> None:
212 (repo / "a.py").write_text("mod\n")
213 (repo / "c.py").write_text("new\n")
214 r = _diff(repo, "--json")
215 data = json.loads(r.output)
216 # summary is a string derived from the delta, total_changes is numeric
217 assert data["total_changes"] >= 1
218
219
220 # ---------------------------------------------------------------------------
221 # --staged integration
222 # ---------------------------------------------------------------------------
223
224
225 class TestStagedIntegration:
226
227 def test_staged_json_shows_staged_file(self, repo: pathlib.Path) -> None:
228 (repo / "a.py").write_text("staged change\n")
229 _stage(repo, "a.py")
230 r = _diff(repo, "--staged", "--json")
231 assert r.exit_code == 0
232 data = json.loads(r.output)
233 assert data["has_changes"] is True
234 assert "a.py" in data["modified"] or "a.py" in data["added"]
235
236 def test_staged_json_clean_when_nothing_staged(self, repo: pathlib.Path) -> None:
237 # Modify but don't stage
238 (repo / "a.py").write_text("unstaged only\n")
239 r = _diff(repo, "--staged", "--json")
240 assert r.exit_code == 0
241 data = json.loads(r.output)
242 assert data["has_changes"] is False
243
244 def test_staged_json_from_ref_is_head(self, repo: pathlib.Path) -> None:
245 (repo / "a.py").write_text("s\n")
246 _stage(repo, "a.py")
247 r = _diff(repo, "--staged", "--json")
248 data = json.loads(r.output)
249 assert data["from_ref"] == "HEAD"
250
251 def test_staged_json_to_ref_is_staged(self, repo: pathlib.Path) -> None:
252 (repo / "a.py").write_text("s\n")
253 _stage(repo, "a.py")
254 r = _diff(repo, "--staged", "--json")
255 data = json.loads(r.output)
256 assert "staged" in data["to_ref"].lower()
257
258 def test_staged_new_file(self, repo: pathlib.Path) -> None:
259 (repo / "new.py").write_text("brand new\n")
260 _stage(repo, "new.py")
261 r = _diff(repo, "--staged", "--json")
262 assert r.exit_code == 0
263 data = json.loads(r.output)
264 assert data["has_changes"] is True
265
266 def test_staged_stat_output(self, repo: pathlib.Path) -> None:
267 (repo / "a.py").write_text("changed\n")
268 _stage(repo, "a.py")
269 r = _diff(repo, "--staged", "--stat")
270 assert r.exit_code == 0
271 assert len(r.output.strip()) > 0
272
273 def test_staged_exit_code_flag(self, repo: pathlib.Path) -> None:
274 (repo / "a.py").write_text("changed\n")
275 _stage(repo, "a.py")
276 r = _diff(repo, "--staged", "--exit-code")
277 assert r.exit_code == 1 # changes present
278
279 def test_staged_exit_code_clean(self, repo: pathlib.Path) -> None:
280 r = _diff(repo, "--staged", "--exit-code")
281 assert r.exit_code == 0 # nothing staged
282
283
284 # ---------------------------------------------------------------------------
285 # --unstaged integration
286 # ---------------------------------------------------------------------------
287
288
289 class TestUnstagedIntegration:
290
291 def test_unstaged_json_shows_unstaged_only(self, repo: pathlib.Path) -> None:
292 (repo / "a.py").write_text("staged\n")
293 _stage(repo, "a.py")
294 (repo / "a.py").write_text("plus unstaged edits\n")
295 r = _diff(repo, "--unstaged", "--json")
296 assert r.exit_code == 0
297 data = json.loads(r.output)
298 assert data["has_changes"] is True
299
300 def test_unstaged_clean_when_all_staged(self, repo: pathlib.Path) -> None:
301 (repo / "a.py").write_text("all staged\n")
302 _stage(repo, "a.py")
303 r = _diff(repo, "--unstaged", "--json")
304 assert r.exit_code == 0
305 data = json.loads(r.output)
306 assert data["has_changes"] is False
307
308 def test_unstaged_json_has_elapsed(self, repo: pathlib.Path) -> None:
309 r = _diff(repo, "--unstaged", "--json")
310 assert r.exit_code == 0
311 assert "duration_ms" in json.loads(r.output)
312
313 def test_unstaged_exit_code_when_dirty(self, repo: pathlib.Path) -> None:
314 (repo / "a.py").write_text("staged\n")
315 _stage(repo, "a.py")
316 (repo / "a.py").write_text("extra unstaged\n")
317 r = _diff(repo, "--unstaged", "--exit-code")
318 assert r.exit_code == 1
319
320
321 # ---------------------------------------------------------------------------
322 # symbols field
323 # ---------------------------------------------------------------------------
324
325
326 class TestSymbolsField:
327
328 def test_symbols_is_dict(self, repo: pathlib.Path) -> None:
329 r = _diff(repo, "--json")
330 data = json.loads(r.output)
331 assert isinstance(data["symbols"], dict)
332
333 def test_symbols_empty_on_clean(self, repo: pathlib.Path) -> None:
334 r = _diff(repo, "--json")
335 data = json.loads(r.output)
336 assert data["has_changes"] is False
337 assert data["symbols"] == {}
338
339 def test_symbols_present_on_dirty(self, repo: pathlib.Path) -> None:
340 (repo / "a.py").write_text("x = 99\n")
341 r = _diff(repo, "--json")
342 data = json.loads(r.output)
343 # symbols may be {} if no symbol-level tracking for this domain
344 assert isinstance(data["symbols"], dict)
345
346 def test_symbols_keys_are_file_paths(self, repo: pathlib.Path) -> None:
347 (repo / "a.py").write_text("x = 99\n")
348 r = _diff(repo, "--json")
349 data = json.loads(r.output)
350 for key in data["symbols"]:
351 assert isinstance(key, str)
352
353 def test_symbols_values_have_buckets(self, repo: pathlib.Path) -> None:
354 (repo / "a.py").write_text("def foo(): pass\n")
355 r = _diff(repo, "--json")
356 data = json.loads(r.output)
357 for _path, buckets in data["symbols"].items():
358 assert isinstance(buckets, dict)
359 for bucket in buckets.values():
360 assert isinstance(bucket, list)
361
362
363 # ---------------------------------------------------------------------------
364 # sem_ver_bump and breaking_changes
365 # ---------------------------------------------------------------------------
366
367
368 class TestSemVerFields:
369
370 def test_sem_ver_bump_present(self, repo: pathlib.Path) -> None:
371 r = _diff(repo, "--json")
372 data = json.loads(r.output)
373 assert "sem_ver_bump" in data
374
375 def test_sem_ver_bump_null_on_clean(self, repo: pathlib.Path) -> None:
376 r = _diff(repo, "--json")
377 data = json.loads(r.output)
378 assert data["sem_ver_bump"] in (None, "", "none", "patch", "minor", "major")
379
380 def test_breaking_changes_present(self, repo: pathlib.Path) -> None:
381 r = _diff(repo, "--json")
382 data = json.loads(r.output)
383 assert "breaking_changes" in data
384
385 def test_breaking_changes_is_list(self, repo: pathlib.Path) -> None:
386 r = _diff(repo, "--json")
387 data = json.loads(r.output)
388 assert isinstance(data["breaking_changes"], list)
389
390 def test_breaking_changes_empty_on_clean(self, repo: pathlib.Path) -> None:
391 r = _diff(repo, "--json")
392 data = json.loads(r.output)
393 assert data["breaking_changes"] == []
394
395
396 # ---------------------------------------------------------------------------
397 # renamed field
398 # ---------------------------------------------------------------------------
399
400
401 class TestRenamedField:
402
403 def test_renamed_is_dict(self, repo: pathlib.Path) -> None:
404 r = _diff(repo, "--json")
405 data = json.loads(r.output)
406 assert isinstance(data["renamed"], dict)
407
408 def test_renamed_empty_on_clean(self, repo: pathlib.Path) -> None:
409 r = _diff(repo, "--json")
410 data = json.loads(r.output)
411 assert data["renamed"] == {}
412
413
414 # ---------------------------------------------------------------------------
415 # from_commit_id / to_commit_id canonical sha256: prefix
416 # ---------------------------------------------------------------------------
417
418
419 class TestCommitIdFormat:
420
421 def test_from_commit_id_has_sha256_prefix(self, repo: pathlib.Path) -> None:
422 r = _diff(repo, "--json")
423 data = json.loads(r.output)
424 assert data["from_commit_id"].startswith("sha256:")
425
426 def test_from_commit_id_full_length(self, repo: pathlib.Path) -> None:
427 r = _diff(repo, "--json")
428 data = json.loads(r.output)
429 cid = data["from_commit_id"]
430 assert len(cid) == len("sha256:") + 64
431
432 def test_to_commit_id_null_for_workdir(self, repo: pathlib.Path) -> None:
433 r = _diff(repo, "--json")
434 data = json.loads(r.output)
435 assert data["to_commit_id"] is None
436
437 def test_both_commit_ids_present_for_two_commit_diff(
438 self, repo: pathlib.Path
439 ) -> None:
440 (repo / "b.py").write_text("y = 2\n")
441 _invoke(repo, ["commit", "-m", "second"])
442 # diff between the two branch tips via explicit refs would need two SHAs;
443 # for now verify that HEAD diff still has sha256: on from_commit_id
444 r = _diff(repo, "--json")
445 data = json.loads(r.output)
446 assert data["from_commit_id"].startswith("sha256:")
447
448
449 # ---------------------------------------------------------------------------
450 # E2E: help output
451 # ---------------------------------------------------------------------------
452
453
454 class TestHelp:
455
456 def test_help_exits_0(self) -> None:
457 r = runner.invoke(None, ["diff", "--help"])
458 assert r.exit_code == 0
459
460 def test_help_mentions_json(self) -> None:
461 r = runner.invoke(None, ["diff", "--help"])
462 assert "--json" in r.output
463
464 def test_help_mentions_staged(self) -> None:
465 r = runner.invoke(None, ["diff", "--help"])
466 assert "--staged" in r.output
467
468 def test_help_mentions_exit_code(self) -> None:
469 r = runner.invoke(None, ["diff", "--help"])
470 assert "--exit-code" in r.output or "-z" in r.output
471
472
473 # ---------------------------------------------------------------------------
474 # Stress: 200-file diff under 5 s
475 # ---------------------------------------------------------------------------
476
477
478 class TestStressElapsed:
479
480 def test_200_file_diff_under_5s(self, tmp_path: pathlib.Path) -> None:
481 import time as _time
482 saved = os.getcwd()
483 try:
484 os.chdir(tmp_path)
485 runner.invoke(None, ["init"])
486 finally:
487 os.chdir(saved)
488
489 for i in range(200):
490 (tmp_path / f"f{i:03d}.py").write_text(f"x = {i}\n")
491 _invoke(tmp_path, ["commit", "-m", "bulk"])
492
493 for i in range(200):
494 (tmp_path / f"f{i:03d}.py").write_text(f"x = {i * 2}\n")
495
496 t0 = _time.monotonic()
497 r = _diff(tmp_path, "--json")
498 elapsed = _time.monotonic() - t0
499
500 assert r.exit_code == 0
501 data = json.loads(r.output)
502 assert data["has_changes"] is True
503 assert elapsed < 5.0, f"200-file diff took {elapsed:.2f}s"
504
505 def test_duration_ms_reflects_real_work(self, tmp_path: pathlib.Path) -> None:
506 """duration_ms in JSON must be > 0 for non-trivial diffs."""
507 saved = os.getcwd()
508 try:
509 os.chdir(tmp_path)
510 runner.invoke(None, ["init"])
511 finally:
512 os.chdir(saved)
513
514 for i in range(50):
515 (tmp_path / f"g{i}.py").write_text(f"y = {i}\n")
516 _invoke(tmp_path, ["commit", "-m", "init"])
517 for i in range(50):
518 (tmp_path / f"g{i}.py").write_text(f"y = {i + 1}\n")
519
520 r = _diff(tmp_path, "--json")
521 assert r.exit_code == 0
522 assert json.loads(r.output)["duration_ms"] >= 0.0