ci.py
python
sha256:37af4ab271a64f11d7bf14b4871ebaaf5dd68fb0247521007c6b034d35aa01be
Merge branch 'fix/hub-user-read-update-path' into dev
Human
9 days ago
| 1 | """CI gate runner for ``muse code test --ci``. |
| 2 | |
| 3 | Reads ``.muse/ci.toml`` and executes each configured gate (lint, type-check, |
| 4 | tests, custom shell commands) in order. Returns a fully-typed |
| 5 | :class:`CiRunResult` suitable for JSON serialisation and programmatic |
| 6 | inspection. |
| 7 | |
| 8 | ``.muse/ci.toml`` schema |
| 9 | ------------------------ |
| 10 | :: |
| 11 | |
| 12 | version = 1 |
| 13 | |
| 14 | [settings] |
| 15 | test_budget_s = 300 # max seconds for the test gate |
| 16 | workers = 4 # parallel pytest partitions |
| 17 | env_allowlist = ["CI", "MUSE_TEST_ENV"] # extra env vars for subprocesses |
| 18 | |
| 19 | [[gate]] |
| 20 | name = "type-check" |
| 21 | command = ["python", "-m", "mypy", "muse/"] |
| 22 | timeout_s = 120 |
| 23 | required = true |
| 24 | |
| 25 | [[gate]] |
| 26 | name = "tests" |
| 27 | command = ["python", "-m", "pytest", "tests/", "-q"] |
| 28 | timeout_s = 300 |
| 29 | required = true |
| 30 | |
| 31 | [[gate]] |
| 32 | name = "typing-audit" |
| 33 | command = ["python", "tools/typing_audit.py", "--dirs", "muse/", "tests/", "--max-any", "0"] |
| 34 | timeout_s = 60 |
| 35 | required = true |
| 36 | |
| 37 | [[gate]] |
| 38 | name = "custom-lint" |
| 39 | command = ["sh", "-c", "ruff check muse/"] |
| 40 | timeout_s = 30 |
| 41 | required = false # failure is a warning, not a blocker |
| 42 | |
| 43 | Built-in defaults |
| 44 | ----------------- |
| 45 | When ``.muse/ci.toml`` does not exist, a minimal default configuration is |
| 46 | used that runs mypy and pytest sequentially. |
| 47 | |
| 48 | Security |
| 49 | -------- |
| 50 | * All commands are executed as ``list[str]`` with ``shell=False``. |
| 51 | * The subprocess environment is sanitised identically to :mod:`muse.core.test_runner`. |
| 52 | * Command strings are never passed to a shell interpreter. |
| 53 | * No user-supplied data is ``eval``-ed or ``exec``-ed. |
| 54 | """ |
| 55 | |
| 56 | import logging |
| 57 | import os |
| 58 | import pathlib |
| 59 | import subprocess |
| 60 | import sys |
| 61 | import time |
| 62 | from collections.abc import Sequence |
| 63 | from typing import NotRequired, TypedDict |
| 64 | |
| 65 | import tomllib |
| 66 | |
| 67 | from muse.core.paths import ci_toml_path |
| 68 | |
| 69 | from muse.core.types import ( |
| 70 | Metadata, |
| 71 | MsgpackDict, |
| 72 | MsgpackValue, |
| 73 | ) |
| 74 | from muse.core.record_helpers import ( |
| 75 | _int_val, |
| 76 | _str_list, |
| 77 | _str_val, |
| 78 | ) |
| 79 | from muse.core.timing import start_timer |
| 80 | |
| 81 | logger = logging.getLogger(__name__) |
| 82 | |
| 83 | # --------------------------------------------------------------------------- |
| 84 | # Public type definitions |
| 85 | # --------------------------------------------------------------------------- |
| 86 | |
| 87 | class CiGate(TypedDict): |
| 88 | """A single CI gate (one external command).""" |
| 89 | |
| 90 | name: str |
| 91 | """Human-readable gate name shown in output.""" |
| 92 | |
| 93 | command: list[str] |
| 94 | """Command to execute as a subprocess list (never passed to shell).""" |
| 95 | |
| 96 | timeout_s: float |
| 97 | """Maximum wall-clock time for this gate in seconds. ``0`` = unlimited.""" |
| 98 | |
| 99 | required: bool |
| 100 | """When False, a failure is reported as a warning but does not block |
| 101 | subsequent gates or mark the overall CI run as failed.""" |
| 102 | |
| 103 | class CiSettings(TypedDict): |
| 104 | """Global settings section of ``.muse/ci.toml``.""" |
| 105 | |
| 106 | test_budget_s: float |
| 107 | """Total seconds budget for the test gate specifically.""" |
| 108 | |
| 109 | workers: int |
| 110 | """Parallel pytest subprocess partitions for the test gate.""" |
| 111 | |
| 112 | env_allowlist: list[str] |
| 113 | """Environment variable names forwarded to gate subprocesses.""" |
| 114 | |
| 115 | class CiConfig(TypedDict): |
| 116 | """Parsed ``.muse/ci.toml`` configuration.""" |
| 117 | |
| 118 | version: int |
| 119 | """Config schema version (currently 1).""" |
| 120 | |
| 121 | settings: CiSettings |
| 122 | """Global settings.""" |
| 123 | |
| 124 | gates: list[CiGate] |
| 125 | """Ordered list of gates to execute.""" |
| 126 | |
| 127 | class GateResult(TypedDict): |
| 128 | """Outcome for a single CI gate.""" |
| 129 | |
| 130 | name: str |
| 131 | """Gate name.""" |
| 132 | |
| 133 | command: list[str] |
| 134 | """Command that was executed.""" |
| 135 | |
| 136 | exit_code: int |
| 137 | """Subprocess exit code.""" |
| 138 | |
| 139 | duration_ms: float |
| 140 | """Wall-clock execution time in milliseconds.""" |
| 141 | |
| 142 | stdout: str |
| 143 | """Captured standard output.""" |
| 144 | |
| 145 | stderr: str |
| 146 | """Captured standard error.""" |
| 147 | |
| 148 | required: bool |
| 149 | """Whether this gate was required.""" |
| 150 | |
| 151 | passed: bool |
| 152 | """``True`` when the gate exited with code 0.""" |
| 153 | |
| 154 | timed_out: bool |
| 155 | """``True`` when the gate was killed due to ``timeout_s``.""" |
| 156 | |
| 157 | warning: NotRequired[str] |
| 158 | """Present when a non-required gate failed (explains advisory status).""" |
| 159 | |
| 160 | class CiRunResult(TypedDict): |
| 161 | """Aggregated result of a full CI run.""" |
| 162 | |
| 163 | passed: bool |
| 164 | """Overall CI status: ``True`` only when all *required* gates passed.""" |
| 165 | |
| 166 | gates: list[GateResult] |
| 167 | """Results for every gate in order of execution.""" |
| 168 | |
| 169 | total_duration_ms: float |
| 170 | """Wall-clock time for all gates combined.""" |
| 171 | |
| 172 | timestamp: str |
| 173 | """ISO 8601 UTC timestamp of when the CI run began.""" |
| 174 | |
| 175 | # --------------------------------------------------------------------------- |
| 176 | # Default configuration |
| 177 | # --------------------------------------------------------------------------- |
| 178 | |
| 179 | _DEFAULT_GATES: list[CiGate] = [ |
| 180 | CiGate( |
| 181 | name="type-check", |
| 182 | command=[sys.executable, "-m", "mypy", "muse/"], |
| 183 | timeout_s=120.0, |
| 184 | required=True, |
| 185 | ), |
| 186 | CiGate( |
| 187 | name="typing-audit", |
| 188 | command=[ |
| 189 | sys.executable, |
| 190 | "tools/typing_audit.py", |
| 191 | "--dirs", |
| 192 | "muse/", |
| 193 | "tests/", |
| 194 | "--max-any", |
| 195 | "0", |
| 196 | ], |
| 197 | timeout_s=60.0, |
| 198 | required=True, |
| 199 | ), |
| 200 | CiGate( |
| 201 | name="tests", |
| 202 | command=[sys.executable, "-m", "pytest", "tests/", "-q"], |
| 203 | timeout_s=300.0, |
| 204 | required=True, |
| 205 | ), |
| 206 | ] |
| 207 | |
| 208 | _DEFAULT_SETTINGS = CiSettings( |
| 209 | test_budget_s=300.0, |
| 210 | workers=1, |
| 211 | env_allowlist=[], |
| 212 | ) |
| 213 | |
| 214 | _DEFAULT_CONFIG = CiConfig( |
| 215 | version=1, |
| 216 | settings=_DEFAULT_SETTINGS, |
| 217 | gates=_DEFAULT_GATES, |
| 218 | ) |
| 219 | |
| 220 | # --------------------------------------------------------------------------- |
| 221 | # TOML loading |
| 222 | # --------------------------------------------------------------------------- |
| 223 | |
| 224 | def _parse_gate(raw: MsgpackValue, index: int) -> CiGate: |
| 225 | """Parse one ``[[gate]]`` entry from the TOML document.""" |
| 226 | if not isinstance(raw, dict): |
| 227 | raise ValueError(f"gate[{index}] must be a TOML table, got {type(raw).__name__}") |
| 228 | |
| 229 | raw_dict: MsgpackDict = raw |
| 230 | name = _str_val(raw_dict, "name", f"gate-{index}") |
| 231 | command_raw = raw_dict.get("command") |
| 232 | timeout_raw = raw_dict.get("timeout_s", 0.0) |
| 233 | required_raw = raw_dict.get("required", True) |
| 234 | |
| 235 | if not isinstance(command_raw, list) or not all(isinstance(c, str) for c in command_raw): |
| 236 | raise ValueError( |
| 237 | f"gate[{index}] 'command' must be a list of strings, got {command_raw!r}" |
| 238 | ) |
| 239 | command: list[str] = [c for c in command_raw if isinstance(c, str)] |
| 240 | if not command: |
| 241 | raise ValueError(f"gate[{index}] 'command' must not be empty") |
| 242 | |
| 243 | timeout_s = ( |
| 244 | float(timeout_raw) |
| 245 | if isinstance(timeout_raw, (int, float)) and not isinstance(timeout_raw, bool) |
| 246 | else 0.0 |
| 247 | ) |
| 248 | required = bool(required_raw) if isinstance(required_raw, bool) else True |
| 249 | |
| 250 | return CiGate( |
| 251 | name=name, |
| 252 | command=command, |
| 253 | timeout_s=timeout_s, |
| 254 | required=required, |
| 255 | ) |
| 256 | |
| 257 | def _parse_settings(raw: MsgpackValue) -> CiSettings: |
| 258 | """Parse the ``[settings]`` section from the TOML document.""" |
| 259 | if not isinstance(raw, dict): |
| 260 | return _DEFAULT_SETTINGS |
| 261 | raw_dict: MsgpackDict = raw |
| 262 | budget_raw = raw_dict.get("test_budget_s", 300.0) |
| 263 | budget = ( |
| 264 | float(budget_raw) |
| 265 | if isinstance(budget_raw, (int, float)) and not isinstance(budget_raw, bool) |
| 266 | else 300.0 |
| 267 | ) |
| 268 | return CiSettings( |
| 269 | test_budget_s=budget, |
| 270 | workers=_int_val(raw_dict, "workers", 1), |
| 271 | env_allowlist=_str_list(raw_dict, "env_allowlist"), |
| 272 | ) |
| 273 | |
| 274 | def load_ci_config(root: pathlib.Path) -> CiConfig: |
| 275 | """Load and parse ``.muse/ci.toml``. |
| 276 | |
| 277 | Returns the built-in default configuration when the file does not exist. |
| 278 | |
| 279 | Args: |
| 280 | root: Repository root directory. |
| 281 | |
| 282 | Raises: |
| 283 | ValueError: If the TOML file exists but cannot be parsed or contains |
| 284 | invalid gate definitions. |
| 285 | """ |
| 286 | path = ci_toml_path(root) |
| 287 | if not path.exists(): |
| 288 | logger.debug("ci: no %s found — using default gates", ".muse/ci.toml") |
| 289 | return _DEFAULT_CONFIG |
| 290 | |
| 291 | try: |
| 292 | raw_bytes = path.read_bytes() |
| 293 | # tomllib.loads returns a Python object tree equivalent to MsgpackValue |
| 294 | # (str, int, float, bool, list, dict — no bytes or None at top level). |
| 295 | # We annotate as MsgpackValue after the isinstance guard below. |
| 296 | parsed = tomllib.loads(raw_bytes.decode()) |
| 297 | except Exception as exc: |
| 298 | raise ValueError(f"Failed to parse {".muse/ci.toml"}: {exc}") from exc |
| 299 | |
| 300 | if not isinstance(parsed, dict): |
| 301 | raise ValueError(f"{".muse/ci.toml"} must be a TOML table at the top level") |
| 302 | |
| 303 | doc: MsgpackDict = parsed # safe: TOML dict ⊆ MsgpackValue |
| 304 | version = _int_val(doc, "version", 1) |
| 305 | settings = _parse_settings(doc.get("settings", {})) |
| 306 | |
| 307 | raw_gates = doc.get("gate", []) |
| 308 | if not isinstance(raw_gates, list): |
| 309 | raise ValueError(f"{".muse/ci.toml"} 'gate' must be an array of tables") |
| 310 | |
| 311 | gates: list[CiGate] = [_parse_gate(g, i) for i, g in enumerate(raw_gates)] |
| 312 | |
| 313 | return CiConfig(version=version, settings=settings, gates=gates) |
| 314 | |
| 315 | # --------------------------------------------------------------------------- |
| 316 | # Gate execution |
| 317 | # --------------------------------------------------------------------------- |
| 318 | |
| 319 | def _mandatory_env() -> Metadata: |
| 320 | """Return the minimal environment variables Python needs to start.""" |
| 321 | _MANDATORY = frozenset( |
| 322 | { |
| 323 | "PATH", |
| 324 | "HOME", |
| 325 | "PYTHONPATH", |
| 326 | "VIRTUAL_ENV", |
| 327 | "PYTHONHOME", |
| 328 | "PYTHONDONTWRITEBYTECODE", |
| 329 | "TMPDIR", |
| 330 | "TMP", |
| 331 | "TEMP", |
| 332 | "LANG", |
| 333 | "LC_ALL", |
| 334 | "LC_CTYPE", |
| 335 | } |
| 336 | ) |
| 337 | return {k: v for k, v in os.environ.items() if k in _MANDATORY} |
| 338 | |
| 339 | def _build_gate_env(allowlist: Sequence[str]) -> Metadata: |
| 340 | """Build the subprocess environment for a gate (mandatory + allowlist).""" |
| 341 | env = _mandatory_env() |
| 342 | for key in allowlist: |
| 343 | if key in os.environ: |
| 344 | env[key] = os.environ[key] |
| 345 | return env |
| 346 | |
| 347 | def _run_gate(gate: CiGate, env: Metadata, cwd: str) -> GateResult: |
| 348 | """Execute a single CI gate subprocess and return its result.""" |
| 349 | timeout: float | None = gate["timeout_s"] if gate["timeout_s"] > 0 else None |
| 350 | timed_out = False |
| 351 | exit_code = 0 |
| 352 | stdout = "" |
| 353 | stderr = "" |
| 354 | |
| 355 | elapsed = start_timer() |
| 356 | try: |
| 357 | proc = subprocess.run( |
| 358 | gate["command"], |
| 359 | capture_output=True, |
| 360 | text=True, |
| 361 | timeout=timeout, |
| 362 | env=env, |
| 363 | cwd=cwd, |
| 364 | shell=False, |
| 365 | ) |
| 366 | exit_code = proc.returncode |
| 367 | stdout = proc.stdout or "" |
| 368 | stderr = proc.stderr or "" |
| 369 | except subprocess.TimeoutExpired as exc: |
| 370 | timed_out = True |
| 371 | exit_code = 124 |
| 372 | stdout = ( |
| 373 | (exc.stdout or b"").decode(errors="replace") |
| 374 | if isinstance(exc.stdout, bytes) |
| 375 | else (exc.stdout or "") |
| 376 | ) |
| 377 | stderr = ( |
| 378 | (exc.stderr or b"").decode(errors="replace") |
| 379 | if isinstance(exc.stderr, bytes) |
| 380 | else (exc.stderr or "") |
| 381 | ) |
| 382 | logger.warning("⚠️ ci: gate %r timed out after %.1f s", gate["name"], gate["timeout_s"]) |
| 383 | except OSError as exc: |
| 384 | exit_code = 127 |
| 385 | stderr = f"Failed to launch command: {exc}" |
| 386 | logger.error("❌ ci: gate %r failed to launch: %s", gate["name"], exc) |
| 387 | |
| 388 | passed = exit_code == 0 and not timed_out |
| 389 | |
| 390 | result = GateResult( |
| 391 | name=gate["name"], |
| 392 | command=gate["command"], |
| 393 | exit_code=exit_code, |
| 394 | duration_ms=elapsed(), |
| 395 | stdout=stdout, |
| 396 | stderr=stderr, |
| 397 | required=gate["required"], |
| 398 | passed=passed, |
| 399 | timed_out=timed_out, |
| 400 | ) |
| 401 | if not passed and not gate["required"]: |
| 402 | result["warning"] = ( |
| 403 | f"Non-required gate '{gate['name']}' failed (exit {exit_code})" |
| 404 | ) |
| 405 | return result |
| 406 | |
| 407 | # --------------------------------------------------------------------------- |
| 408 | # Public API |
| 409 | # --------------------------------------------------------------------------- |
| 410 | |
| 411 | def run_ci( |
| 412 | root: pathlib.Path, |
| 413 | config: CiConfig, |
| 414 | ) -> CiRunResult: |
| 415 | """Execute all gates in *config* sequentially and return a :class:`CiRunResult`. |
| 416 | |
| 417 | Gates are executed in order. When a *required* gate fails, subsequent |
| 418 | gates are still executed to collect the full failure picture — the run |
| 419 | is only stopped immediately on timeout of a required gate to avoid |
| 420 | wasting budget. |
| 421 | |
| 422 | Args: |
| 423 | root: Repository root (used as the subprocess working directory). |
| 424 | config: Parsed CI configuration (from :func:`load_ci_config`). |
| 425 | |
| 426 | Returns: |
| 427 | :class:`CiRunResult` with per-gate outcomes and an overall pass/fail. |
| 428 | """ |
| 429 | from muse.core.test_history import iso_now |
| 430 | |
| 431 | timestamp = iso_now() |
| 432 | env = _build_gate_env(config["settings"]["env_allowlist"]) |
| 433 | cwd = str(root) |
| 434 | |
| 435 | elapsed = start_timer() |
| 436 | gate_results: list[GateResult] = [] |
| 437 | overall_passed = True |
| 438 | |
| 439 | for gate in config["gates"]: |
| 440 | logger.info("🔷 ci: running gate %r: %s", gate["name"], " ".join(gate["command"])) |
| 441 | result = _run_gate(gate, env, cwd) |
| 442 | gate_results.append(result) |
| 443 | |
| 444 | if not result["passed"] and gate["required"]: |
| 445 | overall_passed = False |
| 446 | logger.error( |
| 447 | "❌ ci: required gate %r failed (exit %d)", |
| 448 | gate["name"], |
| 449 | result["exit_code"], |
| 450 | ) |
| 451 | if result["timed_out"]: |
| 452 | logger.error("❌ ci: aborting remaining gates due to timeout") |
| 453 | break |
| 454 | elif not result["passed"] and not gate["required"]: |
| 455 | logger.warning( |
| 456 | "⚠️ ci: non-required gate %r failed (exit %d) — continuing", |
| 457 | gate["name"], |
| 458 | result["exit_code"], |
| 459 | ) |
| 460 | else: |
| 461 | logger.info("✅ ci: gate %r passed", gate["name"]) |
| 462 | |
| 463 | return CiRunResult( |
| 464 | passed=overall_passed, |
| 465 | gates=gate_results, |
| 466 | total_duration_ms=elapsed(), |
| 467 | timestamp=timestamp, |
| 468 | ) |
File History
11 commits
sha256:37af4ab271a64f11d7bf14b4871ebaaf5dd68fb0247521007c6b034d35aa01be
Merge branch 'fix/hub-user-read-update-path' into dev
Human
9 days ago
sha256:b7be56ec091919a612cffe7f3c8b600209d5155517443fdac0e16954c8c7d81f
Merge 'task/git-export-ignored-file-deletion' into 'dev' — …
Human
12 days ago
sha256:8de4334a98c945aace420969d389ad678aa926d4ab4e886b2ac4c4241cb3bf2b
revert: keep pyproject.toml in canonical PEP 440 form
Sonnet 4.6
patch
15 days ago
sha256:50bb615c573aa4b928fca75b60f82ba2a659910066507cec6a95c412ae22ccb9
docs: add issue docs for push have-negotiation bug (#55) an…
Sonnet 4.6
18 days ago
sha256:d90d175cded68aae1d4ffcf4858917854195d0cd8ce1fe73cee4dbc02541cb74
chore: trigger push to surface null-OID paths
Sonnet 4.6
24 days ago
sha256:e452ad9a6ace6ccc6d875a35e06caf9da5576a970c1c36133b69a891ce5fefa8
chore: prebuild timing test
Sonnet 4.6
34 days ago
sha256:0008ab6695e3e064b3e236b24fd19e538fef6a588eb0d211622f4466d919c0b1
merge: pull staging/dev — advance to 0.2.0rc12
Sonnet 4.6
patch
36 days ago
sha256:9c33d61749fff814c5226d5386aa2af7064c2c02788594a25fdd709358132eea
fix: _PROPOSAL_PREFIX_RESOLVE_LIMIT 200 → 100 to match hub …
Sonnet 4.6
47 days ago
sha256:36c3cb3e76619d4c30a6d9bf81b5ec4ff148e30dcfed913e3114ca7b43b81c7e
fix: rename objects→blobs in push client and all stale test…
Sonnet 4.6
patch
50 days ago
sha256:c06a9b9b9fee26c68ea725b44d54b2c0a171301ce9de746d5b656617b4463a9a
fix: repair four test failures from post-migration audit
Sonnet 4.6
patch
56 days ago
sha256:1900655993c83c4107067375548a7be823e471d2515830842f1a12cba4bd3cdf
fix: unified object store migration — idempotent writes, JS…
Sonnet 4.6
minor
⚠
57 days ago