"""CI gate runner for ``muse code test --ci``. Reads ``.muse/ci.toml`` and executes each configured gate (lint, type-check, tests, custom shell commands) in order. Returns a fully-typed :class:`CiRunResult` suitable for JSON serialisation and programmatic inspection. ``.muse/ci.toml`` schema ------------------------ :: version = 1 [settings] test_budget_s = 300 # max seconds for the test gate workers = 4 # parallel pytest partitions env_allowlist = ["CI", "MUSE_TEST_ENV"] # extra env vars for subprocesses [[gate]] name = "type-check" command = ["python", "-m", "mypy", "muse/"] timeout_s = 120 required = true [[gate]] name = "tests" command = ["python", "-m", "pytest", "tests/", "-q"] timeout_s = 300 required = true [[gate]] name = "typing-audit" command = ["python", "tools/typing_audit.py", "--dirs", "muse/", "tests/", "--max-any", "0"] timeout_s = 60 required = true [[gate]] name = "custom-lint" command = ["sh", "-c", "ruff check muse/"] timeout_s = 30 required = false # failure is a warning, not a blocker Built-in defaults ----------------- When ``.muse/ci.toml`` does not exist, a minimal default configuration is used that runs mypy and pytest sequentially. Security -------- * All commands are executed as ``list[str]`` with ``shell=False``. * The subprocess environment is sanitised identically to :mod:`muse.core.test_runner`. * Command strings are never passed to a shell interpreter. * No user-supplied data is ``eval``-ed or ``exec``-ed. """ from __future__ import annotations import logging import os import pathlib import subprocess import sys import time from collections.abc import Sequence from typing import NotRequired, TypedDict import tomllib from muse.core.store import MsgpackValue, MsgpackDict, Metadata, _int_val, _str_list, _str_val logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Public type definitions # --------------------------------------------------------------------------- class CiGate(TypedDict): """A single CI gate (one external command).""" name: str """Human-readable gate name shown in output.""" command: list[str] """Command to execute as a subprocess list (never passed to shell).""" timeout_s: float """Maximum wall-clock time for this gate in seconds. ``0`` = unlimited.""" required: bool """When False, a failure is reported as a warning but does not block subsequent gates or mark the overall CI run as failed.""" class CiSettings(TypedDict): """Global settings section of ``.muse/ci.toml``.""" test_budget_s: float """Total seconds budget for the test gate specifically.""" workers: int """Parallel pytest subprocess partitions for the test gate.""" env_allowlist: list[str] """Environment variable names forwarded to gate subprocesses.""" class CiConfig(TypedDict): """Parsed ``.muse/ci.toml`` configuration.""" version: int """Config schema version (currently 1).""" settings: CiSettings """Global settings.""" gates: list[CiGate] """Ordered list of gates to execute.""" class GateResult(TypedDict): """Outcome for a single CI gate.""" name: str """Gate name.""" command: list[str] """Command that was executed.""" exit_code: int """Subprocess exit code.""" duration_ms: float """Wall-clock execution time in milliseconds.""" stdout: str """Captured standard output.""" stderr: str """Captured standard error.""" required: bool """Whether this gate was required.""" passed: bool """``True`` when the gate exited with code 0.""" timed_out: bool """``True`` when the gate was killed due to ``timeout_s``.""" warning: NotRequired[str] """Present when a non-required gate failed (explains advisory status).""" class CiRunResult(TypedDict): """Aggregated result of a full CI run.""" passed: bool """Overall CI status: ``True`` only when all *required* gates passed.""" gates: list[GateResult] """Results for every gate in order of execution.""" total_duration_ms: float """Wall-clock time for all gates combined.""" timestamp: str """ISO 8601 UTC timestamp of when the CI run began.""" # --------------------------------------------------------------------------- # Default configuration # --------------------------------------------------------------------------- _DEFAULT_GATES: list[CiGate] = [ CiGate( name="type-check", command=[sys.executable, "-m", "mypy", "muse/"], timeout_s=120.0, required=True, ), CiGate( name="typing-audit", command=[ sys.executable, "tools/typing_audit.py", "--dirs", "muse/", "tests/", "--max-any", "0", ], timeout_s=60.0, required=True, ), CiGate( name="tests", command=[sys.executable, "-m", "pytest", "tests/", "-q"], timeout_s=300.0, required=True, ), ] _DEFAULT_SETTINGS = CiSettings( test_budget_s=300.0, workers=1, env_allowlist=[], ) _DEFAULT_CONFIG = CiConfig( version=1, settings=_DEFAULT_SETTINGS, gates=_DEFAULT_GATES, ) _CI_FILE = ".muse/ci.toml" # --------------------------------------------------------------------------- # TOML loading # --------------------------------------------------------------------------- def _ci_path(root: pathlib.Path) -> pathlib.Path: return root / _CI_FILE def _parse_gate(raw: MsgpackValue, index: int) -> CiGate: """Parse one ``[[gate]]`` entry from the TOML document.""" if not isinstance(raw, dict): raise ValueError(f"gate[{index}] must be a TOML table, got {type(raw).__name__}") raw_dict: MsgpackDict = raw name = _str_val(raw_dict, "name", f"gate-{index}") command_raw = raw_dict.get("command") timeout_raw = raw_dict.get("timeout_s", 0.0) required_raw = raw_dict.get("required", True) if not isinstance(command_raw, list) or not all(isinstance(c, str) for c in command_raw): raise ValueError( f"gate[{index}] 'command' must be a list of strings, got {command_raw!r}" ) command: list[str] = [c for c in command_raw if isinstance(c, str)] if not command: raise ValueError(f"gate[{index}] 'command' must not be empty") timeout_s = ( float(timeout_raw) if isinstance(timeout_raw, (int, float)) and not isinstance(timeout_raw, bool) else 0.0 ) required = bool(required_raw) if isinstance(required_raw, bool) else True return CiGate( name=name, command=command, timeout_s=timeout_s, required=required, ) def _parse_settings(raw: MsgpackValue) -> CiSettings: """Parse the ``[settings]`` section from the TOML document.""" if not isinstance(raw, dict): return _DEFAULT_SETTINGS raw_dict: MsgpackDict = raw budget_raw = raw_dict.get("test_budget_s", 300.0) budget = ( float(budget_raw) if isinstance(budget_raw, (int, float)) and not isinstance(budget_raw, bool) else 300.0 ) return CiSettings( test_budget_s=budget, workers=_int_val(raw_dict, "workers", 1), env_allowlist=_str_list(raw_dict, "env_allowlist"), ) def load_ci_config(root: pathlib.Path) -> CiConfig: """Load and parse ``.muse/ci.toml``. Returns the built-in default configuration when the file does not exist. Args: root: Repository root directory. Raises: ValueError: If the TOML file exists but cannot be parsed or contains invalid gate definitions. """ path = _ci_path(root) if not path.exists(): logger.debug("ci: no %s found — using default gates", _CI_FILE) return _DEFAULT_CONFIG try: raw_bytes = path.read_bytes() # tomllib.loads returns a Python object tree equivalent to MsgpackValue # (str, int, float, bool, list, dict — no bytes or None at top level). # We annotate as MsgpackValue after the isinstance guard below. parsed = tomllib.loads(raw_bytes.decode()) except Exception as exc: raise ValueError(f"Failed to parse {_CI_FILE}: {exc}") from exc if not isinstance(parsed, dict): raise ValueError(f"{_CI_FILE} must be a TOML table at the top level") doc: MsgpackDict = parsed # safe: TOML dict ⊆ MsgpackValue version = _int_val(doc, "version", 1) settings = _parse_settings(doc.get("settings", {})) raw_gates = doc.get("gate", []) if not isinstance(raw_gates, list): raise ValueError(f"{_CI_FILE} 'gate' must be an array of tables") gates: list[CiGate] = [_parse_gate(g, i) for i, g in enumerate(raw_gates)] return CiConfig(version=version, settings=settings, gates=gates) # --------------------------------------------------------------------------- # Gate execution # --------------------------------------------------------------------------- def _mandatory_env() -> Metadata: """Return the minimal environment variables Python needs to start.""" _MANDATORY = frozenset( { "PATH", "HOME", "PYTHONPATH", "VIRTUAL_ENV", "PYTHONHOME", "PYTHONDONTWRITEBYTECODE", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "LC_CTYPE", } ) return {k: v for k, v in os.environ.items() if k in _MANDATORY} def _build_gate_env(allowlist: Sequence[str]) -> Metadata: """Build the subprocess environment for a gate (mandatory + allowlist).""" env = _mandatory_env() for key in allowlist: if key in os.environ: env[key] = os.environ[key] return env def _run_gate(gate: CiGate, env: Metadata, cwd: str) -> GateResult: """Execute a single CI gate subprocess and return its result.""" timeout: float | None = gate["timeout_s"] if gate["timeout_s"] > 0 else None timed_out = False exit_code = 0 stdout = "" stderr = "" t0 = time.monotonic() try: proc = subprocess.run( gate["command"], capture_output=True, text=True, timeout=timeout, env=env, cwd=cwd, shell=False, ) exit_code = proc.returncode stdout = proc.stdout or "" stderr = proc.stderr or "" except subprocess.TimeoutExpired as exc: timed_out = True exit_code = 124 stdout = ( (exc.stdout or b"").decode(errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "") ) stderr = ( (exc.stderr or b"").decode(errors="replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "") ) logger.warning("⚠️ ci: gate %r timed out after %.1f s", gate["name"], gate["timeout_s"]) except OSError as exc: exit_code = 127 stderr = f"Failed to launch command: {exc}" logger.error("❌ ci: gate %r failed to launch: %s", gate["name"], exc) duration_ms = (time.monotonic() - t0) * 1000.0 passed = exit_code == 0 and not timed_out result = GateResult( name=gate["name"], command=gate["command"], exit_code=exit_code, duration_ms=duration_ms, stdout=stdout, stderr=stderr, required=gate["required"], passed=passed, timed_out=timed_out, ) if not passed and not gate["required"]: result["warning"] = ( f"Non-required gate '{gate['name']}' failed (exit {exit_code})" ) return result # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- def run_ci( root: pathlib.Path, config: CiConfig, ) -> CiRunResult: """Execute all gates in *config* sequentially and return a :class:`CiRunResult`. Gates are executed in order. When a *required* gate fails, subsequent gates are still executed to collect the full failure picture — the run is only stopped immediately on timeout of a required gate to avoid wasting budget. Args: root: Repository root (used as the subprocess working directory). config: Parsed CI configuration (from :func:`load_ci_config`). Returns: :class:`CiRunResult` with per-gate outcomes and an overall pass/fail. """ from muse.core.test_history import iso_now timestamp = iso_now() env = _build_gate_env(config["settings"]["env_allowlist"]) cwd = str(root) t_start = time.monotonic() gate_results: list[GateResult] = [] overall_passed = True for gate in config["gates"]: logger.info("🔷 ci: running gate %r: %s", gate["name"], " ".join(gate["command"])) result = _run_gate(gate, env, cwd) gate_results.append(result) if not result["passed"] and gate["required"]: overall_passed = False logger.error( "❌ ci: required gate %r failed (exit %d)", gate["name"], result["exit_code"], ) if result["timed_out"]: logger.error("❌ ci: aborting remaining gates due to timeout") break elif not result["passed"] and not gate["required"]: logger.warning( "⚠️ ci: non-required gate %r failed (exit %d) — continuing", gate["name"], result["exit_code"], ) else: logger.info("✅ ci: gate %r passed", gate["name"]) total_duration_ms = (time.monotonic() - t_start) * 1000.0 return CiRunResult( passed=overall_passed, gates=gate_results, total_duration_ms=total_duration_ms, timestamp=timestamp, )