gabriel / muse public
fetch.py python
597 lines 20.5 KB
Raw
sha256:ac29b8ba9021514a03ab2782d92bf67671f0efa5b3b70d46f7598c5d4e923378 docs: record muse#74 Phase 4 live verification (SCR_13, SCR_14) Sonnet 5 15 days ago
1 """muse fetch — download commits, snapshots, and objects from a remote.
2
3 Fetches the latest state of a remote branch without touching the local branch
4 HEAD or working tree. After a successful fetch:
5
6 - All new commits, snapshots, and objects from the remote are stored locally.
7 - The remote tracking pointer ``.muse/remotes/<remote>/<branch>`` is updated.
8
9 Use ``muse pull`` to fetch *and* merge into the current branch, or run
10 ``muse merge`` after fetching to integrate on your own schedule.
11
12 Flags
13 -----
14 ``--all``
15 Fetch every configured remote instead of just one. When combined with
16 ``--branch``, that branch is fetched from every remote.
17
18 ``--prune / -p``
19 After fetching, delete local remote-tracking refs (pointers under
20 ``.muse/remotes/<remote>/``) for branches that no longer exist on the
21 remote. Mirrors ``git fetch --prune``.
22
23 ``--dry-run / -n``
24 Show what would be fetched without writing anything.
25
26 ``--tags``
27 Also fetch tags from the remote (default behaviour when tags exist).
28
29 ``--no-tags``
30 Do not fetch tags from the remote.
31
32 ``--format {text,json}`` / ``--json``
33 Emit a machine-readable JSON object on stdout instead of human text.
34 Human-readable diagnostics always go to stderr regardless of format.
35
36 JSON output schema
37 ------------------
38 Always emits a single JSON object on stdout::
39
40 {
41 "results": [
42 {
43 "remote": "<name>",
44 "branch": "<branch>",
45 "status": "fetched | up_to_date | dry_run | branch_missing",
46 "commits_received": <N>,
47 "blobs_written": <N>,
48 "head": "<commit-id> | null",
49 "pruned": ["<remote>/<branch>", ...],
50 "dry_run": false
51 }
52 ],
53 "dry_run": false
54 }
55
56 Exit codes::
57
58 0 — success (fetched, up_to_date, dry_run, or branch_missing + prune)
59 1 — remote not configured, network error, or no remotes when using --all
60 """
61
62 import argparse
63 import json
64 import logging
65 import pathlib
66 import sys
67 import time
68 from collections.abc import Callable
69 from typing import TypedDict
70
71 from muse.cli.config import (
72 get_signing_identity,
73 get_remote,
74 get_remote_head,
75 list_remotes,
76 set_remote_head,
77 )
78 from muse.core.envelope import EnvelopeJson, make_envelope
79 from muse.core.errors import ExitCode
80 from muse.core.mpack import apply_mpack
81 from muse.core.repo import require_repo
82 from muse.core.refs import read_current_branch
83 from muse.core.commits import get_all_commits
84 from muse.core.timing import start_timer
85 from muse.core.transport import TransportError, make_transport
86 from muse.core.validation import sanitize_display
87 from muse.core.types import BranchHeads
88 from muse.core.paths import remote_tracking_dir as _remote_tracking_dir
89
90 logger = logging.getLogger(__name__)
91
92 class _RemoteResultJson(TypedDict):
93 """Per-remote/branch fetch result nested inside :class:`_FetchJson`.
94
95 Fields
96 ------
97 remote Remote name (e.g. ``"origin"``, ``"local"``).
98 branch Branch name fetched from the remote.
99 status Outcome string — one of:
100 ``"fetched"`` (new data received),
101 ``"up_to_date"`` (remote matches local),
102 ``"dry_run"`` (no writes performed),
103 ``"branch_missing"`` (branch not found on remote).
104 commits_received Number of new commit records written to local storage.
105 blobs_written Number of new blobs written.
106 head Remote commit ID that the branch tip points to after the
107 fetch, or ``None`` when nothing was fetched.
108 pruned List of ``"<remote>/<branch>"`` strings for each ref that
109 existed locally but is no longer present on the remote.
110 dry_run True when the fetch was simulated — no blobs were written.
111 """
112
113 remote: str
114 branch: str
115 status: str
116 commits_received: int
117 blobs_written: int
118 head: str | None
119 pruned: list[str]
120 dry_run: bool
121
122 class _FetchJson(EnvelopeJson):
123 """JSON output for ``muse fetch --json``.
124
125 Inherits the 6 standard envelope fields from :class:`~muse.core.envelope.EnvelopeJson`.
126
127 Fields
128 ------
129 results One result entry per remote/branch combination fetched;
130 see :class:`_RemoteResultJson`.
131 dry_run True when no objects were written (``--dry-run`` was passed).
132 """
133
134 results: list[_RemoteResultJson]
135 dry_run: bool
136
137 class _FetchErrorJsonBase(EnvelopeJson):
138 """Required fields for all ``muse fetch`` error JSON outputs."""
139
140 error: str
141 message: str
142
143 class _FetchErrorJson(_FetchErrorJsonBase, total=False):
144 """JSON error output for ``muse fetch --json`` on failure.
145
146 Fields
147 ------
148 error Machine-readable error code (e.g. ``"remote_not_configured"``).
149 message Human-readable description of the failure.
150 remote Remote name involved, when applicable.
151 branch Branch name involved, when applicable.
152 available Sorted list of available branch names (for ``branch_not_found``).
153 hint Suggested remediation command.
154 retryable True when error is a 503 that exhausted the retry budget.
155 """
156
157 remote: str
158 branch: str
159 available: list[str]
160 hint: str
161 retryable: bool
162
163 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
164 """Register the ``muse fetch`` subcommand and all its flags."""
165 parser = subparsers.add_parser(
166 "fetch",
167 help="Download commits, snapshots, and objects from a remote.",
168 description=__doc__,
169 formatter_class=argparse.RawDescriptionHelpFormatter,
170 )
171 parser.add_argument(
172 "remote",
173 nargs="?",
174 default="origin",
175 help="Remote name to fetch from (default: origin). Ignored when --all is set.",
176 )
177 parser.add_argument(
178 "--branch", "-b",
179 default=None,
180 help=(
181 "Remote branch to fetch (default: current branch). "
182 "When combined with --all, this branch is fetched from every remote."
183 ),
184 )
185 parser.add_argument(
186 "--all",
187 action="store_true",
188 default=False,
189 help="Fetch all configured remotes.",
190 )
191 parser.add_argument(
192 "--prune", "-p",
193 action="store_true",
194 default=False,
195 help=(
196 "Remove local remote-tracking refs for branches that no longer exist "
197 "on the remote (mirrors git fetch --prune)."
198 ),
199 )
200 parser.add_argument(
201 "--dry-run", "-n",
202 action="store_true",
203 default=False,
204 dest="dry_run",
205 help="Show what would be fetched without writing any objects or tracking refs.",
206 )
207 # Tag handling flags — reserved for future use when tag storage is added.
208 tag_group = parser.add_mutually_exclusive_group()
209 tag_group.add_argument(
210 "--tags",
211 action="store_true",
212 default=None,
213 dest="tags",
214 help="Fetch tags from the remote (default).",
215 )
216 tag_group.add_argument(
217 "--no-tags",
218 action="store_false",
219 dest="tags",
220 help="Do not fetch tags from the remote.",
221 )
222 parser.add_argument(
223 "--json", "-j",
224 action="store_true",
225 dest="json_out",
226 help="Emit machine-readable JSON.",
227 )
228 parser.add_argument(
229 "--retry-timeout",
230 type=float,
231 default=None,
232 dest="retry_budget_s",
233 metavar="SECONDS",
234 help=(
235 "Maximum seconds to wait for the remote to finish preparing fetch data "
236 "(503 Retry-After loop). Default: 120 s or MUSE_FETCH_RETRY_BUDGET_S env."
237 ),
238 )
239 parser.add_argument(
240 "--no-retry",
241 action="store_const",
242 const=0.0,
243 dest="retry_budget_s",
244 help=(
245 "Disable the 503 retry loop — fail immediately on the first 503 response. "
246 "Useful for deterministic CI. Overrides --retry-timeout."
247 ),
248 )
249 parser.set_defaults(func=run)
250
251 def _stale_ref_names(
252 root: pathlib.Path,
253 remote: str,
254 live_branch_heads: BranchHeads,
255 ) -> list[str]:
256 """Return branch names whose local tracking refs are absent from *live_branch_heads*.
257
258 Branch names may contain slashes (e.g. ``feat/my-thing``), so the refs are
259 stored as nested files under ``.muse/remotes/<remote>/``. We walk the tree
260 recursively and compute the relative path from ``refs_dir`` to get the full
261 branch name to compare against *live_branch_heads*.
262
263 Symlinks inside the refs directory are skipped to prevent path-traversal
264 attacks via a malicious remote name or branch name.
265 """
266 refs_dir = _remote_tracking_dir(root, remote)
267 if not refs_dir.is_dir():
268 return []
269 stale: list[str] = []
270 for ref_file in refs_dir.rglob("*"):
271 if ref_file.is_symlink() or not ref_file.is_file():
272 continue
273 branch_name = str(ref_file.relative_to(refs_dir))
274 if branch_name not in live_branch_heads:
275 stale.append(branch_name)
276 return stale
277
278 def _prune_stale_refs(
279 root: pathlib.Path,
280 remote: str,
281 live_branch_heads: BranchHeads,
282 *,
283 dry_run: bool,
284 ) -> list[str]:
285 """Prune stale remote-tracking refs, returning the list of pruned branch names.
286
287 Walks ``.muse/remotes/<remote>/`` recursively (branch names may contain
288 slashes and are stored as nested paths) and, unless *dry_run* is True,
289 deletes any file whose relative path is not a key in *live_branch_heads*.
290 Prints a ``- [deleted]`` or ``Would prune`` line for each, mirroring
291 ``git fetch --prune`` output. All output goes to stderr so stdout stays
292 clean for structured JSON.
293 """
294 refs_dir = _remote_tracking_dir(root, remote)
295 pruned: list[str] = []
296 for branch_name in sorted(_stale_ref_names(root, remote, live_branch_heads)):
297 safe_ref = f"{sanitize_display(remote)}/{sanitize_display(branch_name)}"
298 if dry_run:
299 print(f" Would prune {safe_ref}", file=sys.stderr)
300 else:
301 ref_file = refs_dir / branch_name
302 ref_file.unlink()
303 # Remove empty parent directories left behind (e.g. feat/ after feat/my-thing).
304 for parent in ref_file.parents:
305 if parent == refs_dir:
306 break
307 try:
308 parent.rmdir()
309 except OSError:
310 break
311 logger.debug("🗑 Pruned stale tracking ref %s/%s", remote, branch_name)
312 print(f" - [deleted] {safe_ref}", file=sys.stderr)
313 pruned.append(f"{remote}/{branch_name}")
314 return pruned
315
316 def _fetch_one(
317 root: pathlib.Path,
318 remote: str,
319 branch: str,
320 *,
321 prune: bool,
322 dry_run: bool,
323 json_out: bool = False,
324 elapsed: Callable[[], float] = lambda: 0.0,
325 retry_budget_s: "float | None" = None,
326 ) -> _RemoteResultJson:
327 """Fetch a single remote/branch pair.
328
329 Returns a :class:`_RemoteResultJson` describing the outcome. Raises
330 ``SystemExit`` on unrecoverable errors (unknown remote, network failure).
331 Writes nothing when *dry_run* is True.
332
333 Full local history is sent as the ``have`` list so the server can compute
334 the minimal delta.
335 """
336 result: _RemoteResultJson = {
337 "remote": remote,
338 "branch": branch,
339 "status": "fetched",
340 "commits_received": 0,
341 "blobs_written": 0,
342 "head": None,
343 "pruned": [],
344 "dry_run": dry_run,
345 }
346
347 url = get_remote(remote, root)
348 if url is None:
349 if json_out:
350 print(json.dumps(_FetchErrorJson(
351 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
352 error="remote_not_configured",
353 remote=remote,
354 message=f"remote '{remote}' is not configured",
355 hint=f"muse remote add {remote} <url>",
356 )))
357 print(
358 f"❌ Remote '{sanitize_display(remote)}' is not configured.",
359 file=sys.stderr,
360 )
361 print(
362 f" Add it with: muse remote add {sanitize_display(remote)} <url>",
363 file=sys.stderr,
364 )
365 raise SystemExit(ExitCode.USER_ERROR)
366
367 token = get_signing_identity(root, remote_url=url)
368 transport = make_transport(url)
369
370 try:
371 info = transport.fetch_remote_info(url, token)
372 except TransportError as exc:
373 if json_out:
374 print(json.dumps(_FetchErrorJson(
375 **make_envelope(elapsed, exit_code=ExitCode.INTERNAL_ERROR),
376 error="remote_unreachable",
377 remote=remote,
378 message=str(exc),
379 )))
380 print(
381 f"❌ Cannot reach remote '{sanitize_display(remote)}': "
382 f"{sanitize_display(str(exc))}",
383 file=sys.stderr,
384 )
385 raise SystemExit(ExitCode.INTERNAL_ERROR)
386
387 remote_commit_id = info["branch_heads"].get(branch)
388 if remote_commit_id is None:
389 if prune:
390 # The branch we were tracking is gone from the remote.
391 # Prune it (and any other stale refs) then return cleanly —
392 # this mirrors `git fetch --prune` behaviour where a deleted
393 # upstream branch produces " - [deleted] remote/branch" rather
394 # than an error.
395 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=dry_run)
396 result["status"] = "branch_missing"
397 result["pruned"] = pruned
398 return result
399 available = sorted(info["branch_heads"])
400 if json_out:
401 print(json.dumps(_FetchErrorJson(
402 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
403 error="branch_not_found",
404 remote=remote,
405 branch=branch,
406 available=available,
407 message=f"branch '{branch}' does not exist on remote '{remote}'",
408 )))
409 print(
410 f"❌ Branch '{sanitize_display(branch)}' does not exist on "
411 f"remote '{sanitize_display(remote)}'.",
412 file=sys.stderr,
413 )
414 print(f" Available branches: {', '.join(sanitize_display(b) for b in available)}", file=sys.stderr)
415 raise SystemExit(ExitCode.USER_ERROR)
416
417 already_known = get_remote_head(remote, branch, root)
418 if already_known == remote_commit_id:
419 print(
420 f"✅ {sanitize_display(remote)}/{sanitize_display(branch)} "
421 f"is already up to date ({remote_commit_id})",
422 file=sys.stderr,
423 )
424 if prune and not dry_run:
425 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=False)
426 result["pruned"] = pruned
427 result["status"] = "up_to_date"
428 result["head"] = remote_commit_id
429 return result
430
431 if dry_run:
432 print(
433 f" Would fetch {sanitize_display(remote)}/{sanitize_display(branch)} "
434 f"→ {remote_commit_id}",
435 file=sys.stderr,
436 )
437 if prune:
438 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=True)
439 result["pruned"] = pruned
440 result["status"] = "dry_run"
441 result["head"] = remote_commit_id
442 return result
443
444 have_for_fetch = [c.commit_id for c in get_all_commits(root)]
445
446 print(f"Fetching {sanitize_display(remote)}/{sanitize_display(branch)} …", file=sys.stderr)
447
448 t0_fetch = time.perf_counter()
449 try:
450 fetch_result = transport.fetch_mpack(
451 url, token,
452 want=[remote_commit_id],
453 have=have_for_fetch,
454 retry_budget_s=retry_budget_s,
455 )
456 except TransportError as exc:
457 _is_503 = exc.status_code == 503
458 if _is_503:
459 from muse.core.transport import _resolve_retry_budget
460 _budget = _resolve_retry_budget(retry_budget_s)
461 _msg = f"Remote still preparing fetch data after {int(_budget)}s — try again shortly."
462 else:
463 _msg = str(exc)
464 if json_out:
465 print(json.dumps(_FetchErrorJson(
466 **make_envelope(elapsed, exit_code=ExitCode.INTERNAL_ERROR),
467 error="fetch_failed",
468 remote=remote,
469 branch=branch,
470 message=_msg,
471 retryable=_is_503,
472 )))
473 print(f"❌ {sanitize_display(_msg)}", file=sys.stderr)
474 raise SystemExit(ExitCode.INTERNAL_ERROR)
475 t_fetch = time.perf_counter() - t0_fetch
476
477 apply_result = apply_mpack(root, {
478 "commits": fetch_result["commits"],
479 "snapshots": fetch_result["snapshots"],
480 "blobs": fetch_result["blobs"],
481 })
482
483 if not apply_result["failed_blobs"]:
484 set_remote_head(remote, branch, remote_commit_id, root)
485
486 commits_received: int = apply_result["commits_written"]
487 blobs_written: int = apply_result["blobs_written"]
488 print(
489 f"[mpack] fetch/mpack: {t_fetch:.2f}s "
490 f"blobs: {fetch_result.get('blobs_received', 0)} "
491 f"commits: {commits_received}",
492 file=sys.stderr,
493 )
494 print(
495 f"✅ Fetched {commits_received} commit(s), "
496 f"{blobs_written} new blob(s) "
497 f"from {sanitize_display(remote)}/{sanitize_display(branch)} "
498 f"({remote_commit_id})",
499 file=sys.stderr,
500 )
501
502 if prune:
503 pruned = _prune_stale_refs(root, remote, info["branch_heads"], dry_run=False)
504 result["pruned"] = pruned
505
506 result["commits_received"] = commits_received
507 result["blobs_written"] = blobs_written
508 result["head"] = remote_commit_id
509 return result
510
511 def run(args: argparse.Namespace) -> None:
512 """Download commits, snapshots, and objects from a remote.
513
514 Updates remote tracking pointers but does NOT change local HEAD or the
515 working tree. Run ``muse pull`` to fetch and merge in one step.
516 ``--all`` fetches every configured remote; ``--prune`` deletes stale
517 remote-tracking refs after a successful fetch.
518
519 Agent quickstart
520 ----------------
521 ::
522
523 muse fetch local --json
524 muse fetch local main --json
525 muse fetch --all --json
526 muse fetch local --prune --json
527
528 JSON fields
529 -----------
530 results List of per-remote result objects: ``remote``, ``branch``,
531 ``new_commits``, ``objects_fetched``, ``ok``, ``error``.
532 dry_run ``true`` if ``--dry-run`` was passed (no writes occurred).
533
534 Exit codes
535 ----------
536 0 Fetch complete (all remotes succeeded).
537 1 One or more remotes failed, no remotes configured, or bad arguments.
538 2 Not inside a Muse repository.
539 """
540 elapsed = start_timer()
541 root = require_repo()
542 current_branch = read_current_branch(root)
543 dry_run: bool = args.dry_run
544 prune: bool = args.prune
545 retry_budget_s: float | None = getattr(args, "retry_budget_s", None)
546 json_out: bool = args.json_out
547
548 if dry_run:
549 print("(dry run — no objects or refs will be written)", file=sys.stderr)
550
551 results: list[_RemoteResultJson] = []
552
553 if args.all:
554 remotes = list_remotes(root)
555 if not remotes:
556 if json_out:
557 print(json.dumps(_FetchErrorJson(
558 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
559 error="no_remotes",
560 message="no remotes configured",
561 hint="muse remote add <name> <url>",
562 )))
563 print("❌ No remotes configured.", file=sys.stderr)
564 print(" Add one with: muse remote add <name> <url>", file=sys.stderr)
565 raise SystemExit(ExitCode.USER_ERROR)
566 # --branch with --all: fetch the specified branch from every remote.
567 branch: str = args.branch or current_branch
568 for remote_cfg in remotes:
569 result = _fetch_one(
570 root,
571 remote_cfg["name"],
572 branch,
573 prune=prune,
574 dry_run=dry_run,
575 json_out=json_out,
576 elapsed=elapsed,
577 retry_budget_s=retry_budget_s,
578 )
579 results.append(result)
580 else:
581 remote: str = args.remote
582 # Use the explicitly passed branch, or fall back to the current local branch.
583 # Do NOT use get_upstream() here — that returns the remote *name*, not branch.
584 branch_single: str = args.branch or current_branch
585 result = _fetch_one(
586 root, remote, branch_single,
587 prune=prune, dry_run=dry_run, json_out=json_out, elapsed=elapsed,
588 retry_budget_s=retry_budget_s,
589 )
590 results.append(result)
591
592 if json_out:
593 print(json.dumps(_FetchJson(
594 **make_envelope(elapsed),
595 results=results,
596 dry_run=dry_run,
597 )))
File History 2 commits
sha256:ac29b8ba9021514a03ab2782d92bf67671f0efa5b3b70d46f7598c5d4e923378 docs: record muse#74 Phase 4 live verification (SCR_13, SCR_14) Sonnet 5 15 days ago
sha256:2562dffa0a0822ac1bdea854f9b267843c6ce95b497a9dc5c55837c80a3ebd0a feat: domain_command_registry — Phase 1 of muse#74 (SCR_01-03) Sonnet 4.6 patch 15 days ago