test_semantic_test_coverage_supercharge.py
python
sha256:d11a87833d5fad6059b7662844bf5448a8911a17cce7a51811f71ad394f248eb
bump to v0.2.0rc13
Human
patch
6 days ago
| 1 | """Supercharge tests for ``muse code semantic-test-coverage``. |
| 2 | |
| 3 | Coverage tiers |
| 4 | -------------- |
| 5 | Unit — TypedDict shape (_JsonOut gains exit_code/duration_ms/schema_version), |
| 6 | alias registration (-j), docstring completeness. |
| 7 | Integration — -j alias produces same output as --json; exit_code/duration_ms/ |
| 8 | schema_version present in JSON; --min-coverage exit code mirrors |
| 9 | process exit; uncovered-only JSON still carries envelope fields. |
| 10 | Security — ANSI in --file/--kind args not echoed raw. |
| 11 | Performance — duration_ms < 5000 ms on a small repo. |
| 12 | """ |
| 13 | |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import argparse |
| 17 | import json |
| 18 | import os |
| 19 | import pathlib |
| 20 | import textwrap |
| 21 | |
| 22 | import pytest |
| 23 | |
| 24 | from tests.cli_test_helper import CliRunner, InvokeResult |
| 25 | |
| 26 | runner = CliRunner() |
| 27 | |
| 28 | |
| 29 | # ────────────────────────────────────────────────────────────────────────────── |
| 30 | # Fixtures |
| 31 | # ────────────────────────────────────────────────────────────────────────────── |
| 32 | |
| 33 | |
| 34 | def _invoke(repo: pathlib.Path, args: list[str]) -> InvokeResult: |
| 35 | saved = os.getcwd() |
| 36 | try: |
| 37 | os.chdir(repo) |
| 38 | return runner.invoke(None, args) |
| 39 | finally: |
| 40 | os.chdir(saved) |
| 41 | |
| 42 | |
| 43 | def _stc(repo: pathlib.Path, *args: str) -> InvokeResult: |
| 44 | return _invoke(repo, ["code", "semantic-test-coverage", *args]) |
| 45 | |
| 46 | |
| 47 | @pytest.fixture() |
| 48 | def cov_repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 49 | """Minimal repo with one production file and one test file.""" |
| 50 | saved = os.getcwd() |
| 51 | try: |
| 52 | os.chdir(tmp_path) |
| 53 | runner.invoke(None, ["init"]) |
| 54 | finally: |
| 55 | os.chdir(saved) |
| 56 | |
| 57 | (tmp_path / "billing.py").write_text( |
| 58 | textwrap.dedent("""\ |
| 59 | class Invoice: |
| 60 | def compute_total(self, items): |
| 61 | return sum(item["price"] for item in items) |
| 62 | |
| 63 | def apply_discount(self, total, pct): |
| 64 | return total * (1 - pct / 100) |
| 65 | |
| 66 | def generate_pdf(self): |
| 67 | pass |
| 68 | """), |
| 69 | encoding="utf-8", |
| 70 | ) |
| 71 | (tmp_path / "tests").mkdir() |
| 72 | (tmp_path / "tests" / "test_billing.py").write_text( |
| 73 | textwrap.dedent("""\ |
| 74 | from billing import Invoice |
| 75 | |
| 76 | def test_compute_total(): |
| 77 | inv = Invoice() |
| 78 | assert inv.compute_total([{"price": 10}]) == 10 |
| 79 | |
| 80 | def test_apply_discount(): |
| 81 | inv = Invoice() |
| 82 | assert inv.apply_discount(100, 10) == 90 |
| 83 | """), |
| 84 | encoding="utf-8", |
| 85 | ) |
| 86 | |
| 87 | saved = os.getcwd() |
| 88 | try: |
| 89 | os.chdir(tmp_path) |
| 90 | runner.invoke(None, ["code", "add", "."]) |
| 91 | runner.invoke(None, ["commit", "-m", "init"]) |
| 92 | finally: |
| 93 | os.chdir(saved) |
| 94 | |
| 95 | return tmp_path |
| 96 | |
| 97 | |
| 98 | # ────────────────────────────────────────────────────────────────────────────── |
| 99 | # Unit — TypedDict |
| 100 | # ────────────────────────────────────────────────────────────────────────────── |
| 101 | |
| 102 | |
| 103 | class TestTypedDict: |
| 104 | def test_json_out_has_exit_code(self) -> None: |
| 105 | import typing |
| 106 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 107 | |
| 108 | hints = typing.get_type_hints(_JsonOut) |
| 109 | assert "exit_code" in hints, "exit_code missing from _JsonOut" |
| 110 | |
| 111 | def test_json_out_has_duration_ms(self) -> None: |
| 112 | import typing |
| 113 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 114 | |
| 115 | hints = typing.get_type_hints(_JsonOut) |
| 116 | assert "duration_ms" in hints, "duration_ms missing from _JsonOut" |
| 117 | |
| 118 | def test_json_out_has_schema_version(self) -> None: |
| 119 | import typing |
| 120 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 121 | |
| 122 | hints = typing.get_type_hints(_JsonOut) |
| 123 | assert "schema" in hints, "schema_version missing from _JsonOut" |
| 124 | |
| 125 | def test_json_out_retains_core_fields(self) -> None: |
| 126 | import typing |
| 127 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 128 | |
| 129 | hints = typing.get_type_hints(_JsonOut) |
| 130 | required = {"ref", "snapshot_id", "depth", "transitive", "filters", "summary", "files"} |
| 131 | missing = required - set(hints) |
| 132 | assert not missing, f"Core fields missing from _JsonOut: {missing}" |
| 133 | |
| 134 | |
| 135 | # ────────────────────────────────────────────────────────────────────────────── |
| 136 | # Unit — -j alias registration |
| 137 | # ────────────────────────────────────────────────────────────────────────────── |
| 138 | |
| 139 | |
| 140 | class TestAliasRegistration: |
| 141 | def _make_parser(self) -> "argparse.ArgumentParser": |
| 142 | import argparse |
| 143 | from muse.cli.commands.semantic_test_coverage import register |
| 144 | |
| 145 | p = argparse.ArgumentParser() |
| 146 | sub = p.add_subparsers() |
| 147 | register(sub) |
| 148 | return p |
| 149 | |
| 150 | def test_j_alias_sets_json_true(self) -> None: |
| 151 | p = self._make_parser() |
| 152 | ns = p.parse_args(["semantic-test-coverage", "-j"]) |
| 153 | assert ns.json_out is True |
| 154 | |
| 155 | def test_j_alias_and_other_flags_coexist(self) -> None: |
| 156 | p = self._make_parser() |
| 157 | ns = p.parse_args(["semantic-test-coverage", "-j", "--uncovered-only"]) |
| 158 | assert ns.json_out is True |
| 159 | assert ns.uncovered_only is True |
| 160 | |
| 161 | |
| 162 | # ────────────────────────────────────────────────────────────────────────────── |
| 163 | # Integration — -j alias |
| 164 | # ────────────────────────────────────────────────────────────────────────────── |
| 165 | |
| 166 | |
| 167 | class TestJsonAlias: |
| 168 | def test_j_alias_exit_code_zero(self, cov_repo: pathlib.Path) -> None: |
| 169 | result = _stc(cov_repo, "-j") |
| 170 | assert result.exit_code == 0 |
| 171 | |
| 172 | def test_j_alias_valid_json(self, cov_repo: pathlib.Path) -> None: |
| 173 | result = _stc(cov_repo, "-j") |
| 174 | data = json.loads(result.output) |
| 175 | assert isinstance(data, dict) |
| 176 | |
| 177 | def test_j_alias_same_top_level_keys_as_json_flag(self, cov_repo: pathlib.Path) -> None: |
| 178 | r_json = _stc(cov_repo, "--json") |
| 179 | r_j = _stc(cov_repo, "-j") |
| 180 | assert set(json.loads(r_json.output)) == set(json.loads(r_j.output)) |
| 181 | |
| 182 | def test_j_alias_summary_matches_json_flag(self, cov_repo: pathlib.Path) -> None: |
| 183 | r_json = _stc(cov_repo, "--json") |
| 184 | r_j = _stc(cov_repo, "-j") |
| 185 | assert json.loads(r_json.output)["summary"] == json.loads(r_j.output)["summary"] |
| 186 | |
| 187 | def test_j_alias_with_uncovered_only(self, cov_repo: pathlib.Path) -> None: |
| 188 | result = _stc(cov_repo, "-j", "--uncovered-only") |
| 189 | data = json.loads(result.output) |
| 190 | # All symbols in output should be uncovered. |
| 191 | for fc in data["files"]: |
| 192 | for sym in fc["symbols"]: |
| 193 | assert not sym["covered"] |
| 194 | |
| 195 | |
| 196 | # ────────────────────────────────────────────────────────────────────────────── |
| 197 | # Integration — JSON envelope: exit_code, duration_ms, schema_version |
| 198 | # ────────────────────────────────────────────────────────────────────────────── |
| 199 | |
| 200 | |
| 201 | class TestJsonEnvelope: |
| 202 | def test_has_exit_code(self, cov_repo: pathlib.Path) -> None: |
| 203 | result = _stc(cov_repo, "--json") |
| 204 | data = json.loads(result.output) |
| 205 | assert "exit_code" in data |
| 206 | |
| 207 | def test_exit_code_is_zero_on_success(self, cov_repo: pathlib.Path) -> None: |
| 208 | result = _stc(cov_repo, "--json") |
| 209 | data = json.loads(result.output) |
| 210 | assert data["exit_code"] == 0 |
| 211 | |
| 212 | def test_exit_code_is_int(self, cov_repo: pathlib.Path) -> None: |
| 213 | result = _stc(cov_repo, "--json") |
| 214 | data = json.loads(result.output) |
| 215 | assert isinstance(data["exit_code"], int) |
| 216 | |
| 217 | def test_has_duration_ms(self, cov_repo: pathlib.Path) -> None: |
| 218 | result = _stc(cov_repo, "--json") |
| 219 | data = json.loads(result.output) |
| 220 | assert "duration_ms" in data |
| 221 | |
| 222 | def test_duration_ms_is_nonnegative_float(self, cov_repo: pathlib.Path) -> None: |
| 223 | result = _stc(cov_repo, "--json") |
| 224 | data = json.loads(result.output) |
| 225 | assert isinstance(data["duration_ms"], float) |
| 226 | assert data["duration_ms"] >= 0.0 |
| 227 | |
| 228 | def test_has_schema_version(self, cov_repo: pathlib.Path) -> None: |
| 229 | result = _stc(cov_repo, "--json") |
| 230 | data = json.loads(result.output) |
| 231 | assert "schema" in data |
| 232 | |
| 233 | def test_schema_version_is_nonempty_string(self, cov_repo: pathlib.Path) -> None: |
| 234 | result = _stc(cov_repo, "--json") |
| 235 | data = json.loads(result.output) |
| 236 | assert isinstance(data["schema"], int) |
| 237 | assert data["schema"] > 0 |
| 238 | |
| 239 | def test_uncovered_only_json_still_has_envelope(self, cov_repo: pathlib.Path) -> None: |
| 240 | result = _stc(cov_repo, "--json", "--uncovered-only") |
| 241 | data = json.loads(result.output) |
| 242 | assert "exit_code" in data |
| 243 | assert "duration_ms" in data |
| 244 | assert "schema" in data |
| 245 | |
| 246 | def test_j_alias_has_exit_code(self, cov_repo: pathlib.Path) -> None: |
| 247 | result = _stc(cov_repo, "-j") |
| 248 | data = json.loads(result.output) |
| 249 | assert "exit_code" in data |
| 250 | |
| 251 | def test_j_alias_has_duration_ms(self, cov_repo: pathlib.Path) -> None: |
| 252 | result = _stc(cov_repo, "-j") |
| 253 | data = json.loads(result.output) |
| 254 | assert "duration_ms" in data |
| 255 | |
| 256 | def test_j_alias_has_schema_version(self, cov_repo: pathlib.Path) -> None: |
| 257 | result = _stc(cov_repo, "-j") |
| 258 | data = json.loads(result.output) |
| 259 | assert "schema" in data |
| 260 | |
| 261 | def test_min_coverage_violation_exit_code_one(self, cov_repo: pathlib.Path) -> None: |
| 262 | # generate_pdf is uncovered, so 100% threshold must fail. |
| 263 | result = _stc(cov_repo, "--min-coverage", "100") |
| 264 | assert result.exit_code == 1 |
| 265 | |
| 266 | def test_min_coverage_zero_exit_code_zero(self, cov_repo: pathlib.Path) -> None: |
| 267 | result = _stc(cov_repo, "--min-coverage", "0") |
| 268 | assert result.exit_code == 0 |
| 269 | |
| 270 | |
| 271 | # ────────────────────────────────────────────────────────────────────────────── |
| 272 | # Security — ANSI injection |
| 273 | # ────────────────────────────────────────────────────────────────────────────── |
| 274 | |
| 275 | |
| 276 | class TestSecurity: |
| 277 | def test_ansi_in_file_filter_not_echoed(self, cov_repo: pathlib.Path) -> None: |
| 278 | result = _stc(cov_repo, "--file", "\x1b[31mbilling\x1b[0m") |
| 279 | combined = result.output + (result.stderr or "") |
| 280 | assert "\x1b[31m" not in combined |
| 281 | |
| 282 | def test_null_byte_in_file_filter_no_crash(self, cov_repo: pathlib.Path) -> None: |
| 283 | # Should not crash — may return empty or error gracefully. |
| 284 | result = _stc(cov_repo, "--file", "billing\x00malicious") |
| 285 | assert result.exit_code in (0, 1) |
| 286 | |
| 287 | |
| 288 | # ────────────────────────────────────────────────────────────────────────────── |
| 289 | # Performance |
| 290 | # ────────────────────────────────────────────────────────────────────────────── |
| 291 | |
| 292 | |
| 293 | class TestPerformance: |
| 294 | def test_duration_ms_under_5000(self, cov_repo: pathlib.Path) -> None: |
| 295 | result = _stc(cov_repo, "--json") |
| 296 | data = json.loads(result.output) |
| 297 | assert data["duration_ms"] < 5000.0 |
| 298 | |
| 299 | |
| 300 | # ────────────────────────────────────────────────────────────────────────────── |
| 301 | # Unit — docstrings |
| 302 | # ────────────────────────────────────────────────────────────────────────────── |
| 303 | |
| 304 | |
| 305 | class TestDocstrings: |
| 306 | def test_run_has_docstring(self) -> None: |
| 307 | from muse.cli.commands.semantic_test_coverage import run |
| 308 | |
| 309 | assert run.__doc__ and len(run.__doc__.strip()) > 30 |
| 310 | |
| 311 | def test_register_has_docstring(self) -> None: |
| 312 | from muse.cli.commands.semantic_test_coverage import register |
| 313 | |
| 314 | assert register.__doc__ and len(register.__doc__.strip()) > 20 |
| 315 | |
| 316 | def test_run_docstring_mentions_schema_version(self) -> None: |
| 317 | from muse.cli.commands.semantic_test_coverage import run |
| 318 | |
| 319 | doc = run.__doc__ or "" |
| 320 | assert "exit_code" in doc or "json" in doc.lower() |
| 321 | |
| 322 | def test_register_docstring_mentions_j_alias(self) -> None: |
| 323 | from muse.cli.commands.semantic_test_coverage import register |
| 324 | |
| 325 | doc = register.__doc__ or "" |
| 326 | assert "-j" in doc |
| 327 | |
| 328 | |
| 329 | class TestRegisterFlags: |
| 330 | def test_default_json_out_is_false(self) -> None: |
| 331 | import argparse |
| 332 | from muse.cli.commands.semantic_test_coverage import register |
| 333 | p = argparse.ArgumentParser() |
| 334 | subs = p.add_subparsers() |
| 335 | register(subs) |
| 336 | args = p.parse_args(["semantic-test-coverage"]) |
| 337 | assert args.json_out is False |
| 338 | |
| 339 | def test_json_flag_sets_json_out(self) -> None: |
| 340 | import argparse |
| 341 | from muse.cli.commands.semantic_test_coverage import register |
| 342 | p = argparse.ArgumentParser() |
| 343 | subs = p.add_subparsers() |
| 344 | register(subs) |
| 345 | args = p.parse_args(["semantic-test-coverage", "--json"]) |
| 346 | assert args.json_out is True |
| 347 | |
| 348 | def test_j_shorthand_sets_json_out(self) -> None: |
| 349 | import argparse |
| 350 | from muse.cli.commands.semantic_test_coverage import register |
| 351 | p = argparse.ArgumentParser() |
| 352 | subs = p.add_subparsers() |
| 353 | register(subs) |
| 354 | args = p.parse_args(["semantic-test-coverage", "-j"]) |
| 355 | assert args.json_out is True |
File History
1 commit
sha256:d11a87833d5fad6059b7662844bf5448a8911a17cce7a51811f71ad394f248eb
bump to v0.2.0rc13
Human
patch
6 days ago