test_core_test_runner.py
python
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa
feat: Muse — version control for the agent era
Human
73 days ago
| 1 | """Tests for muse.core.test_runner — subprocess pytest adapter. |
| 2 | |
| 3 | Coverage: |
| 4 | - _build_env strips non-allowed env vars. |
| 5 | - _check_json_report returns a bool. |
| 6 | - _parse_text_output extracts pass/fail counts from pytest text output. |
| 7 | - _partition splits lists into roughly equal parts. |
| 8 | - run_tests executes tests and returns RunResult with correct structure. |
| 9 | - run_tests with a failing test returns exit_code != 0. |
| 10 | - run_tests with no targets runs pytest discovery. |
| 11 | - run_tests respects timeout_s (TimeoutExpired → timed_out=True). |
| 12 | - run_tests with workers > 1 partitions correctly. |
| 13 | - RunResult has all required fields. |
| 14 | """ |
| 15 | |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import os |
| 19 | import pathlib |
| 20 | import sys |
| 21 | |
| 22 | import pytest |
| 23 | |
| 24 | from muse.core.test_runner import ( |
| 25 | RunConfig, |
| 26 | RunResult, |
| 27 | CaseResult, |
| 28 | _build_env, |
| 29 | _check_json_report, |
| 30 | _parse_text_output, |
| 31 | _partition, |
| 32 | run_tests, |
| 33 | ) |
| 34 | |
| 35 | |
| 36 | # --------------------------------------------------------------------------- |
| 37 | # Unit tests — _build_env |
| 38 | # --------------------------------------------------------------------------- |
| 39 | |
| 40 | |
| 41 | class TestBuildEnv: |
| 42 | def test_strips_sensitive_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: |
| 43 | """SECRET_TOKEN is not forwarded unless explicitly in allowlist.""" |
| 44 | monkeypatch.setenv("SECRET_TOKEN", "top-secret") |
| 45 | env = _build_env([]) |
| 46 | assert "SECRET_TOKEN" not in env |
| 47 | |
| 48 | def test_mandatory_vars_forwarded(self, monkeypatch: pytest.MonkeyPatch) -> None: |
| 49 | """PATH is always forwarded (mandatory).""" |
| 50 | monkeypatch.setenv("PATH", "/usr/bin") |
| 51 | env = _build_env([]) |
| 52 | assert "PATH" in env |
| 53 | assert env["PATH"] == "/usr/bin" |
| 54 | |
| 55 | def test_allowlist_forwarded(self, monkeypatch: pytest.MonkeyPatch) -> None: |
| 56 | """Explicitly allowed vars are forwarded.""" |
| 57 | monkeypatch.setenv("MY_CUSTOM_VAR", "value") |
| 58 | env = _build_env(["MY_CUSTOM_VAR"]) |
| 59 | assert "MY_CUSTOM_VAR" in env |
| 60 | assert env["MY_CUSTOM_VAR"] == "value" |
| 61 | |
| 62 | def test_allowlist_missing_var_skipped( |
| 63 | self, monkeypatch: pytest.MonkeyPatch |
| 64 | ) -> None: |
| 65 | """Allowlisted var that doesn't exist in env is silently skipped.""" |
| 66 | monkeypatch.delenv("NOT_SET_VAR", raising=False) |
| 67 | env = _build_env(["NOT_SET_VAR"]) |
| 68 | assert "NOT_SET_VAR" not in env |
| 69 | |
| 70 | def test_returns_dict_str_str(self) -> None: |
| 71 | """_build_env returns dict[str, str].""" |
| 72 | env = _build_env([]) |
| 73 | assert isinstance(env, dict) |
| 74 | for k, v in env.items(): |
| 75 | assert isinstance(k, str) |
| 76 | assert isinstance(v, str) |
| 77 | |
| 78 | |
| 79 | # --------------------------------------------------------------------------- |
| 80 | # Unit tests — _check_json_report |
| 81 | # --------------------------------------------------------------------------- |
| 82 | |
| 83 | |
| 84 | class TestCheckJsonReport: |
| 85 | def test_returns_bool(self) -> None: |
| 86 | result = _check_json_report() |
| 87 | assert isinstance(result, bool) |
| 88 | |
| 89 | |
| 90 | # --------------------------------------------------------------------------- |
| 91 | # Unit tests — _parse_text_output |
| 92 | # --------------------------------------------------------------------------- |
| 93 | |
| 94 | |
| 95 | class TestParseTextOutput: |
| 96 | def test_parse_all_passed(self) -> None: |
| 97 | stdout = "3 passed in 0.12s" |
| 98 | counts = _parse_text_output(stdout) |
| 99 | assert counts["passed"] == 3 |
| 100 | assert counts["failed"] == 0 |
| 101 | assert counts["total"] == 3 |
| 102 | |
| 103 | def test_parse_mixed(self) -> None: |
| 104 | stdout = "2 passed, 1 failed in 0.55s" |
| 105 | counts = _parse_text_output(stdout) |
| 106 | assert counts["passed"] == 2 |
| 107 | assert counts["failed"] == 1 |
| 108 | assert counts["total"] == 3 |
| 109 | |
| 110 | def test_parse_error(self) -> None: |
| 111 | stdout = "1 passed, 1 error in 0.1s" |
| 112 | counts = _parse_text_output(stdout) |
| 113 | assert counts["errored"] == 1 |
| 114 | |
| 115 | def test_parse_empty(self) -> None: |
| 116 | counts = _parse_text_output("") |
| 117 | assert counts["total"] == 0 |
| 118 | |
| 119 | |
| 120 | # --------------------------------------------------------------------------- |
| 121 | # Unit tests — _partition |
| 122 | # --------------------------------------------------------------------------- |
| 123 | |
| 124 | |
| 125 | class TestPartition: |
| 126 | def test_single_partition(self) -> None: |
| 127 | result = _partition(["a", "b", "c"], 1) |
| 128 | assert result == [["a", "b", "c"]] |
| 129 | |
| 130 | def test_three_partitions(self) -> None: |
| 131 | items = ["a", "b", "c", "d", "e", "f"] |
| 132 | parts = _partition(items, 3) |
| 133 | assert len(parts) == 3 |
| 134 | flat = [x for p in parts for x in p] |
| 135 | assert sorted(flat) == sorted(items) |
| 136 | |
| 137 | def test_empty_list(self) -> None: |
| 138 | assert _partition([], 4) == [[]] |
| 139 | |
| 140 | def test_more_workers_than_items(self) -> None: |
| 141 | """When workers > len(items), we still get at least 1 partition.""" |
| 142 | parts = _partition(["a"], 10) |
| 143 | assert len(parts) >= 1 |
| 144 | assert parts[0] == ["a"] |
| 145 | |
| 146 | |
| 147 | # --------------------------------------------------------------------------- |
| 148 | # Integration tests — run_tests |
| 149 | # --------------------------------------------------------------------------- |
| 150 | |
| 151 | |
| 152 | @pytest.fixture() |
| 153 | def passing_test_file(tmp_path: pathlib.Path) -> pathlib.Path: |
| 154 | """Create a minimal passing test file.""" |
| 155 | f = tmp_path / "test_pass.py" |
| 156 | f.write_text( |
| 157 | "def test_always_passes() -> None:\n assert True\n" |
| 158 | ) |
| 159 | return f |
| 160 | |
| 161 | |
| 162 | @pytest.fixture() |
| 163 | def failing_test_file(tmp_path: pathlib.Path) -> pathlib.Path: |
| 164 | """Create a minimal failing test file.""" |
| 165 | f = tmp_path / "test_fail.py" |
| 166 | f.write_text( |
| 167 | "def test_always_fails() -> None:\n assert False, 'intentional'\n" |
| 168 | ) |
| 169 | return f |
| 170 | |
| 171 | |
| 172 | class TestRunTests: |
| 173 | def test_run_passing_test( |
| 174 | self, passing_test_file: pathlib.Path, tmp_path: pathlib.Path |
| 175 | ) -> None: |
| 176 | """run_tests with a passing test returns exit_code=0.""" |
| 177 | config = RunConfig( |
| 178 | targets=[str(passing_test_file)], |
| 179 | workers=1, |
| 180 | timeout_s=30.0, |
| 181 | extra_args=[], |
| 182 | env_allowlist=[], |
| 183 | cwd=tmp_path, |
| 184 | ) |
| 185 | result = run_tests(config) |
| 186 | assert result["exit_code"] == 0 |
| 187 | assert result["passed"] >= 1 |
| 188 | assert result["timed_out"] is False |
| 189 | assert isinstance(result["run_id"], str) |
| 190 | assert isinstance(result["duration_ms"], float) |
| 191 | assert result["duration_ms"] > 0 |
| 192 | |
| 193 | def test_run_failing_test( |
| 194 | self, failing_test_file: pathlib.Path, tmp_path: pathlib.Path |
| 195 | ) -> None: |
| 196 | """run_tests with a failing test returns exit_code != 0.""" |
| 197 | config = RunConfig( |
| 198 | targets=[str(failing_test_file)], |
| 199 | workers=1, |
| 200 | timeout_s=30.0, |
| 201 | extra_args=[], |
| 202 | env_allowlist=[], |
| 203 | cwd=tmp_path, |
| 204 | ) |
| 205 | result = run_tests(config) |
| 206 | assert result["exit_code"] != 0 |
| 207 | assert result["failed"] >= 1 |
| 208 | |
| 209 | def test_result_structure( |
| 210 | self, passing_test_file: pathlib.Path, tmp_path: pathlib.Path |
| 211 | ) -> None: |
| 212 | """RunResult has all required fields with correct types.""" |
| 213 | config = RunConfig( |
| 214 | targets=[str(passing_test_file)], |
| 215 | workers=1, |
| 216 | timeout_s=30.0, |
| 217 | extra_args=[], |
| 218 | env_allowlist=[], |
| 219 | cwd=tmp_path, |
| 220 | ) |
| 221 | result = run_tests(config) |
| 222 | assert isinstance(result["run_id"], str) |
| 223 | assert isinstance(result["targets"], list) |
| 224 | assert isinstance(result["exit_code"], int) |
| 225 | assert isinstance(result["duration_ms"], float) |
| 226 | assert isinstance(result["results"], list) |
| 227 | assert isinstance(result["total"], int) |
| 228 | assert isinstance(result["passed"], int) |
| 229 | assert isinstance(result["failed"], int) |
| 230 | assert isinstance(result["errored"], int) |
| 231 | assert isinstance(result["skipped"], int) |
| 232 | assert isinstance(result["timed_out"], bool) |
| 233 | assert isinstance(result["json_report_available"], bool) |
| 234 | assert isinstance(result["stdout"], str) |
| 235 | assert isinstance(result["stderr"], str) |
| 236 | |
| 237 | def test_progress_callback_called( |
| 238 | self, passing_test_file: pathlib.Path, tmp_path: pathlib.Path |
| 239 | ) -> None: |
| 240 | """progress_cb is invoked once per test case result.""" |
| 241 | seen: list[CaseResult] = [] |
| 242 | |
| 243 | def cb(r: CaseResult) -> None: |
| 244 | seen.append(r) |
| 245 | |
| 246 | config = RunConfig( |
| 247 | targets=[str(passing_test_file)], |
| 248 | workers=1, |
| 249 | timeout_s=30.0, |
| 250 | extra_args=[], |
| 251 | env_allowlist=[], |
| 252 | cwd=tmp_path, |
| 253 | ) |
| 254 | run_tests(config, progress_cb=cb) |
| 255 | # If json report is available, progress_cb should have been called. |
| 256 | # If not available, we can't assert count — just assert it's a list. |
| 257 | assert isinstance(seen, list) |
| 258 | |
| 259 | def test_run_with_workers_2( |
| 260 | self, tmp_path: pathlib.Path |
| 261 | ) -> None: |
| 262 | """workers=2 runs tests in parallel partitions without error.""" |
| 263 | # Write two test files. |
| 264 | (tmp_path / "test_one.py").write_text( |
| 265 | "def test_one() -> None:\n assert True\n" |
| 266 | ) |
| 267 | (tmp_path / "test_two.py").write_text( |
| 268 | "def test_two() -> None:\n assert True\n" |
| 269 | ) |
| 270 | config = RunConfig( |
| 271 | targets=[ |
| 272 | str(tmp_path / "test_one.py"), |
| 273 | str(tmp_path / "test_two.py"), |
| 274 | ], |
| 275 | workers=2, |
| 276 | timeout_s=30.0, |
| 277 | extra_args=[], |
| 278 | env_allowlist=[], |
| 279 | cwd=tmp_path, |
| 280 | ) |
| 281 | result = run_tests(config) |
| 282 | assert result["exit_code"] == 0 |
| 283 | assert result["timed_out"] is False |
| 284 | |
| 285 | def test_timeout_returns_timed_out_flag( |
| 286 | self, tmp_path: pathlib.Path |
| 287 | ) -> None: |
| 288 | """A test that sleeps longer than timeout_s is killed and timed_out=True.""" |
| 289 | slow_test = tmp_path / "test_slow.py" |
| 290 | slow_test.write_text( |
| 291 | "import time\n" |
| 292 | "def test_slow() -> None:\n" |
| 293 | " time.sleep(30)\n" |
| 294 | ) |
| 295 | config = RunConfig( |
| 296 | targets=[str(slow_test)], |
| 297 | workers=1, |
| 298 | timeout_s=0.1, # 100 ms — definitely not enough |
| 299 | extra_args=[], |
| 300 | env_allowlist=[], |
| 301 | cwd=tmp_path, |
| 302 | ) |
| 303 | result = run_tests(config) |
| 304 | assert result["timed_out"] is True |
| 305 | |
| 306 | def test_run_id_is_unique_per_call( |
| 307 | self, passing_test_file: pathlib.Path, tmp_path: pathlib.Path |
| 308 | ) -> None: |
| 309 | """Each run_tests call produces a distinct run_id.""" |
| 310 | config = RunConfig( |
| 311 | targets=[str(passing_test_file)], |
| 312 | workers=1, |
| 313 | timeout_s=30.0, |
| 314 | extra_args=[], |
| 315 | env_allowlist=[], |
| 316 | cwd=tmp_path, |
| 317 | ) |
| 318 | r1 = run_tests(config) |
| 319 | r2 = run_tests(config) |
| 320 | assert r1["run_id"] != r2["run_id"] |
File History
1 commit
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa
feat: Muse — version control for the agent era
Human
73 days ago