gabriel / muse public
clone.py python
610 lines 21.9 KB
Raw
sha256:b5a46a9923166b1435c3d7549801a23fbf281f8f78c766142e923b35c23bdc13 feat(#192): Phase 4 — discoverability nudges at clone/status time Sonnet 5 patch 4 days ago
1 """muse clone — create a local copy of a remote Muse repository.
2
3 Downloads the complete commit history, snapshots, and objects from a remote
4 MuseHub repository into a new local directory. After cloning:
5
6 - A full ``.muse/`` directory is created with the remote's repo_id and domain.
7 - The ``origin`` remote is configured to point at the source URL.
8 - The default branch is checked out into the working tree.
9
10 Usage
11 -----
12
13 muse clone <url> Clone into a directory named after the last URL segment.
14 muse clone <url> <dir> Clone into a specific directory.
15 muse clone <url> --branch dev Clone and check out 'dev'.
16 muse clone <url> --dry-run Show what would happen without writing anything.
17 muse clone <url> --no-checkout Skip working-tree restore after cloning.
18 muse clone <url> --json Emit a machine-readable JSON result to stdout.
19
20 Auth
21 ----
22
23 Signing identities are read from ``~/.muse/identity.toml`` keyed by hostname.
24 No signing identity is required for public repositories.
25
26 JSON schema (``--json``)
27 ------------------------
28
29 ::
30
31 {
32 "status": "cloned | dry_run | already_exists",
33 "url": "<remote_url>",
34 "directory": "<local_path>",
35 "branch": "<branch_checked_out>",
36 "commits_received": <N>,
37 "blobs_written": <N>,
38 "head": "<sha256> | null",
39 "domain": "<domain>",
40 "dry_run": false
41 }
42
43 Exit codes
44 ----------
45
46 0 — success (including dry-run)
47 1 — user error (target already exists, empty repository, unknown branch)
48 2 — internal / transport error
49 """
50
51 import argparse
52 import json
53 import logging
54 import pathlib
55 import shutil
56 import sys
57 from typing import TYPE_CHECKING, TypedDict
58
59 import time
60
61 from muse._version import __version__ as _SCHEMA_VERSION
62 from muse.core.timing import start_timer
63 from muse.core.envelope import EnvelopeJson, make_envelope
64 from muse.cli.config import get_signing_identity, set_remote, set_remote_head, set_upstream
65 from muse.core.types import content_hash, now_utc_iso
66 from muse.core.paths import muse_dir as _muse_dir, ref_path as _ref_path
67 from muse.core.errors import ExitCode
68 from muse.core.mpack import apply_mpack
69 from muse.core.io import write_text_atomic
70 from muse.core.refs import (
71 write_branch_ref,
72 write_head_branch,
73 )
74 from muse.core.commits import read_commit
75 from muse.core.snapshots import read_snapshot
76 from muse.core.transport import TransportError, make_transport
77 from muse.core.validation import sanitize_display
78 from muse.core.workdir import apply_manifest
79 from muse.core.hooks import install_notice
80
81 type _RepoMeta = dict[str, str]
82 if TYPE_CHECKING:
83 from muse.core.mpack import ApplyResult
84
85 logger = logging.getLogger(__name__)
86
87 # Canonical set of subdirectories — must match muse init's _INIT_SUBDIRS.
88 _CLONE_SUBDIRS: tuple[str, ...] = (
89 "refs",
90 "refs/heads",
91 "objects",
92 "commits",
93 "snapshots",
94 "tags",
95 "cache",
96 )
97
98 _DEFAULT_CONFIG = """\
99 [user]
100 name = ""
101 email = ""
102
103 [remotes]
104
105 [domain]
106 # Domain-specific configuration keys depend on the active domain.
107 """
108
109 class _CloneJson(EnvelopeJson):
110 """Stable JSON schema emitted by ``muse clone --json``."""
111
112 status: str # "cloned" | "dry_run" | "already_exists" | "partial"
113 url: str
114 directory: str # resolved local path
115 branch: str # branch checked out
116 commits_received: int
117 blobs_written: int
118 skipped_blobs: int # blobs skipped due to integrity failure (0 on clean clone)
119 head: str | None # HEAD commit ID after clone, null on dry-run
120 domain: str
121 dry_run: bool
122 shallow_commits: list[str]
123
124 class _CloneErrorJson(EnvelopeJson):
125 """JSON output for clone transport/network error paths."""
126
127 error: str # "remote_unreachable" | "empty_repository" | "fetch_failed"
128 url: str
129 message: str
130 retryable: bool # True when error is a 503 that exhausted the retry budget
131
132 def _infer_dir_name(url: str) -> str:
133 """Derive a safe local directory name from the last non-empty segment of *url*.
134
135 Strips query strings, fragments, and path-traversal components so that a
136 crafted URL like ``http://attacker.example.com/../../../../etc`` cannot escape
137 the current working directory.
138 """
139 # Drop fragment and query before splitting on path separators.
140 stripped = url.split("#")[0].split("?")[0].rstrip("/")
141 last = stripped.rsplit("/", 1)[-1]
142 # pathlib.Path.name always strips leading dots and directory separators,
143 # eliminating traversal attempts like ".." or "../../secret".
144 safe = pathlib.PurePosixPath(last).name
145 return safe if safe and safe not in (".", "..") else "muse-repo"
146
147 def _init_muse_dir(
148 target: pathlib.Path,
149 repo_id: str,
150 domain: str,
151 default_branch: str,
152 ) -> None:
153 """Create the ``.muse/`` directory tree inside *target*.
154
155 Uses the same subdirectory set as ``muse init`` so that every command that
156 relies on the standard layout (tags, objects, etc.) works out of the box.
157 """
158 muse_dir = _muse_dir(target)
159 for subdir in _CLONE_SUBDIRS:
160 (muse_dir / subdir).mkdir(parents=True, exist_ok=True)
161
162 repo_meta: _RepoMeta = {
163 "repo_id": repo_id,
164 "schema_version": _SCHEMA_VERSION,
165 "created_at": now_utc_iso(),
166 "domain": domain,
167 }
168 write_text_atomic(muse_dir / "repo.json", f"{json.dumps(repo_meta)}\n")
169 write_head_branch(muse_dir.parent, default_branch)
170 write_text_atomic(_ref_path(target, default_branch), "")
171 write_text_atomic(muse_dir / "config.toml", _DEFAULT_CONFIG)
172
173 def _restore_working_tree(root: pathlib.Path, commit_id: str) -> None:
174 """Restore the working tree to the snapshot referenced by *commit_id*.
175
176 Logs a warning to stderr (rather than silently returning) if the commit or
177 snapshot cannot be read — this surfaces bugs where apply_mpack did not write
178 the expected objects.
179 """
180 commit = read_commit(root, commit_id)
181 if commit is None:
182 logger.warning(
183 "⚠️ clone: commit %s not found after apply_mpack — working tree not restored",
184 commit_id,
185 )
186 return
187 snap = read_snapshot(root, commit.snapshot_id)
188 if snap is None:
189 logger.warning(
190 "⚠️ clone: snapshot %s not found after apply_mpack — working tree not restored",
191 commit.snapshot_id,
192 )
193 return
194 try:
195 apply_manifest(root, {}, snap.manifest)
196 except RuntimeError as exc:
197 logger.warning(
198 "⚠️ clone: working tree partially restored — %s",
199 exc,
200 )
201
202 def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
203 """Register the ``muse clone`` subcommand and all its flags."""
204 parser = subparsers.add_parser(
205 "clone",
206 help="Create a local copy of a remote Muse repository.",
207 description=__doc__,
208 formatter_class=argparse.RawDescriptionHelpFormatter,
209 )
210 parser.add_argument(
211 "url",
212 help="URL of the remote Muse repository to clone.",
213 )
214 parser.add_argument(
215 "directory",
216 nargs="?",
217 default=None,
218 help=(
219 "Local directory to clone into. "
220 "Defaults to the last path segment of the URL."
221 ),
222 )
223 parser.add_argument(
224 "--branch", "-b",
225 default=None,
226 help="Branch to check out after cloning (default: remote default branch).",
227 )
228 parser.add_argument(
229 "--dry-run", "-n",
230 action="store_true",
231 default=False,
232 dest="dry_run",
233 help=(
234 "Contact the remote and show what would be cloned without writing "
235 "any files or creating any directories."
236 ),
237 )
238 parser.add_argument(
239 "--no-checkout",
240 action="store_true",
241 default=False,
242 dest="no_checkout",
243 help="Skip restoring the working tree after cloning.",
244 )
245 parser.add_argument(
246 "--depth",
247 type=int,
248 default=None,
249 metavar="N",
250 help=(
251 "Shallow clone: fetch only the N most recent commits from each branch tip. "
252 "Writes .muse/shallow with the boundary commit IDs. "
253 "Must be >= 1."
254 ),
255 )
256 parser.add_argument(
257 "--json", "-j",
258 action="store_true",
259 dest="json_out",
260 help="Emit JSON output to stdout.",
261 )
262 parser.add_argument(
263 "--retry-timeout",
264 type=float,
265 default=None,
266 dest="retry_budget_s",
267 metavar="SECONDS",
268 help=(
269 "Maximum seconds to wait for the remote to finish preparing fetch data "
270 "(503 Retry-After loop). Default: 120 s or MUSE_FETCH_RETRY_BUDGET_S env."
271 ),
272 )
273 parser.add_argument(
274 "--no-retry",
275 action="store_const",
276 const=0.0,
277 dest="retry_budget_s",
278 help=(
279 "Disable the 503 retry loop — fail immediately on the first 503 response. "
280 "Useful for deterministic CI. Overrides --retry-timeout."
281 ),
282 )
283 parser.set_defaults(func=run)
284
285 def run(args: argparse.Namespace) -> None:
286 """Clone a remote Muse repository into a new local directory.
287
288 Downloads the full commit history, snapshots, objects, and branch heads.
289 Configures ``origin`` remote and upstream tracking. Checks out the default
290 branch unless ``--no-checkout`` is given. On any error after the target
291 directory has been partially created the directory is removed to leave the
292 filesystem clean.
293
294 Agent quickstart
295 ----------------
296 ::
297
298 muse clone https://musehub.ai/gabriel/muse --json
299 muse clone https://musehub.ai/gabriel/muse --branch dev --json
300 muse clone https://musehub.ai/gabriel/muse mydir --dry-run --json
301
302 JSON fields
303 -----------
304 status ``"cloned"``, ``"already_exists"``, or ``"error"``.
305 url The remote URL cloned from.
306 directory Absolute path to the created local directory.
307 branch Branch checked out after cloning.
308 commits_received Number of commits received.
309 blobs_written Number of content blobs written.
310 head Full commit ID at the branch tip; ``null`` on error.
311 domain Repository domain (e.g. ``"code"``).
312 dry_run ``true`` when ``--dry-run`` was passed.
313
314 Exit codes
315 ----------
316 0 Clone completed successfully.
317 1 Target already exists or bad URL.
318 3 Network or internal error during fetch.
319 """
320 elapsed = start_timer()
321 url: str = args.url
322 directory: str | None = args.directory
323 branch: str | None = args.branch
324 dry_run: bool = args.dry_run
325 no_checkout: bool = args.no_checkout
326 json_out: bool = args.json_out
327 depth: int | None = getattr(args, "depth", None)
328 retry_budget_s: float | None = getattr(args, "retry_budget_s", None)
329
330 if depth is not None and depth < 1:
331 print("❌ --depth must be >= 1 (got 0)", file=sys.stderr)
332 raise SystemExit(ExitCode.USER_ERROR)
333
334 # clone does not need to be inside a Muse repo — it creates a new one.
335 # Resolve the target name and path before any network I/O.
336 target_name = directory or _infer_dir_name(url)
337 target = pathlib.Path.cwd() / target_name
338
339 if dry_run:
340 print("(dry run — no files will be created)", file=sys.stderr)
341
342 if _muse_dir(target).exists():
343 msg = f"❌ '{sanitize_display(str(target))}' is already a Muse repository."
344 print(msg, file=sys.stderr)
345 if json_out:
346 print(json.dumps(_CloneJson(
347 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
348 status="already_exists",
349 url=url,
350 directory=str(target),
351 branch=branch or "",
352 commits_received=0,
353 blobs_written=0,
354 head=None,
355 domain="",
356 dry_run=dry_run,
357 shallow_commits=[],
358 )))
359 raise SystemExit(ExitCode.USER_ERROR)
360
361 signing = get_signing_identity(remote_url=url)
362
363 transport = make_transport(url)
364
365 print(
366 f"Cloning from {sanitize_display(url)} …",
367 file=sys.stderr,
368 )
369 try:
370 info = transport.fetch_remote_info(url, signing=signing)
371 except TransportError as exc:
372 if json_out:
373 print(json.dumps(_CloneErrorJson(
374 **make_envelope(elapsed, exit_code=ExitCode.INTERNAL_ERROR),
375 error="remote_unreachable",
376 url=url,
377 message=str(exc),
378 retryable=False,
379 )))
380 print(f"❌ Cannot reach remote: {exc}", file=sys.stderr)
381 raise SystemExit(ExitCode.INTERNAL_ERROR)
382
383 # Use "code" as the domain fallback — "midi" was the first plugin but is
384 # not the canonical default domain for new repositories.
385 if info["repo_id"]:
386 remote_repo_id = info["repo_id"]
387 else:
388 _cloned_at = now_utc_iso()
389 remote_repo_id = content_hash({"cloned_at": _cloned_at, "url": url})
390 domain = info["domain"] or "code"
391 default_branch = branch or info["default_branch"] or "main"
392
393 if not info["branch_heads"]:
394 if json_out:
395 print(json.dumps(_CloneErrorJson(
396 **make_envelope(elapsed, exit_code=ExitCode.USER_ERROR),
397 error="empty_repository",
398 url=url,
399 message="remote repository has no branches",
400 retryable=False,
401 )))
402 print(
403 "❌ Remote repository has no branches (empty repository).",
404 file=sys.stderr,
405 )
406 raise SystemExit(ExitCode.USER_ERROR)
407
408 default_commit_id = info["branch_heads"].get(default_branch)
409 if default_commit_id is None:
410 # Fall back to the first available branch rather than failing hard —
411 # a user who requests a non-existent branch gets a clear warning.
412 first_branch, default_commit_id = next(iter(info["branch_heads"].items()))
413 print(
414 f" ⚠️ Branch '{sanitize_display(default_branch)}' not found on remote; "
415 f"checking out '{sanitize_display(first_branch)}' instead.",
416 file=sys.stderr,
417 )
418 default_branch = first_branch
419
420 available = sorted(info["branch_heads"])
421 logger.debug(
422 "Remote has %d branch(es): %s",
423 len(available),
424 ", ".join(sanitize_display(b) for b in available),
425 )
426
427 # ── dry-run exits here — no filesystem changes after this point ──────────
428 if dry_run:
429 want_count = len(info["branch_heads"])
430 if json_out:
431 print(json.dumps(_CloneJson(
432 **make_envelope(elapsed),
433 status="dry_run",
434 url=url,
435 directory=str(target),
436 branch=default_branch,
437 commits_received=0,
438 blobs_written=0,
439 head=default_commit_id,
440 domain=domain,
441 dry_run=True,
442 shallow_commits=[],
443 )))
444 else:
445 print(
446 f"Would clone {sanitize_display(url)} → {sanitize_display(str(target))}",
447 file=sys.stderr,
448 )
449 print(
450 f" branch={sanitize_display(default_branch)}, "
451 f"domain={sanitize_display(domain)}, "
452 f"{want_count} branch head(s) to fetch",
453 file=sys.stderr,
454 )
455 return
456
457 # ── real clone ────────────────────────────────────────────────────────────
458 target.mkdir(parents=True, exist_ok=True)
459 try:
460 _init_muse_dir(target, remote_repo_id, domain, default_branch)
461 except OSError as exc:
462 print(
463 f"❌ Failed to create repository at '{sanitize_display(str(target))}': {exc}",
464 file=sys.stderr,
465 )
466 shutil.rmtree(target, ignore_errors=True)
467 raise SystemExit(ExitCode.INTERNAL_ERROR)
468
469 # ── fetch/mpack ───────────────────────────────────────────────────────────
470 # Clone always starts with an empty have list — no local history yet.
471 want = list(info["branch_heads"].values())
472
473 t0 = time.perf_counter()
474 try:
475 fetch_result = transport.fetch_mpack(
476 url, signing,
477 want=want,
478 have=[],
479 retry_budget_s=retry_budget_s,
480 )
481 except TransportError as exc:
482 _is_503 = exc.status_code == 503
483 if _is_503:
484 from muse.core.transport import _resolve_retry_budget
485 _budget = _resolve_retry_budget(retry_budget_s)
486 _msg = (
487 f"Remote still preparing clone data after {int(_budget)}s — try again shortly."
488 )
489 else:
490 _msg = str(exc)
491 if json_out:
492 print(json.dumps(_CloneErrorJson(
493 **make_envelope(elapsed, exit_code=ExitCode.INTERNAL_ERROR),
494 error="fetch_failed",
495 url=url,
496 message=_msg,
497 retryable=_is_503,
498 )))
499 print(f"❌ {_msg}", file=sys.stderr)
500 shutil.rmtree(target, ignore_errors=True)
501 raise SystemExit(ExitCode.INTERNAL_ERROR)
502 t_fetch = time.perf_counter() - t0
503
504 fetched_shallow: list[str] = fetch_result.get("shallow_commits") or []
505 _raw_blobs = fetch_result.get("blobs") or []
506 _t_apply0 = time.perf_counter()
507 print(f"[clone] apply_mpack START blobs={len(_raw_blobs)} commits={len(fetch_result['commits'])} snaps={len(fetch_result['snapshots'])}", file=sys.stderr, flush=True)
508 apply_result: ApplyResult = apply_mpack(
509 target,
510 {
511 "commits": fetch_result["commits"],
512 "snapshots": fetch_result["snapshots"],
513 "blobs": _raw_blobs,
514 },
515 shallow_commits=set(fetched_shallow),
516 )
517 _t_apply1 = time.perf_counter()
518 blobs_written: int = apply_result["blobs_written"]
519 blobs_skipped: int = apply_result["blobs_skipped"]
520 print(
521 f"[clone] apply_mpack DONE t={int((_t_apply1-_t_apply0)*1000)}ms"
522 f" blobs_written={blobs_written} blobs_skipped={blobs_skipped}"
523 f" commits_written={apply_result['commits_written']}",
524 file=sys.stderr, flush=True,
525 )
526 print(
527 f"[mpack] fetch/mpack: {t_fetch:.2f}s "
528 f"blobs: {fetch_result['blobs_received']} "
529 f"commits: {apply_result['commits_written']}",
530 file=sys.stderr,
531 )
532
533 # Write branch head refs for every remote branch atomically and record
534 # the remote tracking pointer so future fetches can detect staleness.
535 # Only advance refs for branches whose tip commit landed cleanly.
536 failed = set(apply_result.get("failed_blobs") or [])
537 for b, cid in info["branch_heads"].items():
538 write_branch_ref(target, b, cid)
539 set_remote_head("origin", b, cid, target)
540
541 # Configure origin remote and upstream tracking.
542 set_remote("origin", url, target)
543 set_upstream(default_branch, "origin", target)
544
545 # Write .muse/shallow for shallow clones so future pull/fetch can recognise boundary.
546 if fetched_shallow:
547 shallow_path = _muse_dir(target) / "shallow"
548 write_text_atomic(shallow_path, "\n".join(fetched_shallow) + "\n")
549
550 # Restore working tree unless the caller opted out.
551 if not no_checkout:
552 _t_wt0 = time.perf_counter()
553 print(f"[clone] apply_manifest START (working tree restore)", file=sys.stderr, flush=True)
554 _restore_working_tree(target, default_commit_id)
555 _t_wt1 = time.perf_counter()
556 print(f"[clone] apply_manifest DONE t={int((_t_wt1-_t_wt0)*1000)}ms", file=sys.stderr, flush=True)
557
558 # musehub#192 Phase 4: a fresh clone can never have hooks installed
559 # yet (install state is deliberately local-only, never cloned) — so
560 # if the repo declares any, surface it immediately rather than
561 # leaving it to be discovered only when muse commit later warns.
562 notice = install_notice(target)
563 if notice:
564 print(f"ℹ️ {notice}", file=sys.stderr)
565
566 commits_received = apply_result["commits_written"]
567 blobs_failed = len(failed)
568 total_skipped = blobs_skipped + blobs_failed
569 exit_code = ExitCode.PARTIAL if total_skipped > 0 else ExitCode.SUCCESS
570
571 if json_out:
572 print(json.dumps(_CloneJson(
573 **make_envelope(elapsed, exit_code=exit_code),
574 status="partial" if total_skipped > 0 else "cloned",
575 url=url,
576 directory=str(target),
577 branch=default_branch,
578 commits_received=commits_received,
579 blobs_written=blobs_written,
580 skipped_blobs=total_skipped,
581 head=default_commit_id,
582 domain=domain,
583 dry_run=False,
584 shallow_commits=fetched_shallow,
585 )))
586 else:
587 if total_skipped > 0:
588 print(
589 f"⚠️ Cloned into '{sanitize_display(target_name)}' with {total_skipped} skipped blob(s) — "
590 f"{commits_received} commit(s), {blobs_written} blob(s) written, "
591 f"domain={sanitize_display(domain)}, "
592 f"branch={sanitize_display(default_branch)} ({default_commit_id})",
593 file=sys.stderr,
594 )
595 else:
596 print(
597 f"✅ Cloned into '{sanitize_display(target_name)}' — "
598 f"{commits_received} commit(s), {blobs_written} blob(s), "
599 f"domain={sanitize_display(domain)}, "
600 f"branch={sanitize_display(default_branch)} ({default_commit_id})",
601 file=sys.stderr,
602 )
603
604 logger.info(
605 "✅ clone: %s → %s commits=%d blobs=%d skipped=%d",
606 url, target, commits_received, blobs_written, total_skipped,
607 )
608
609 if exit_code != ExitCode.SUCCESS:
610 raise SystemExit(exit_code)
File History 1 commit
sha256:b5a46a9923166b1435c3d7549801a23fbf281f8f78c766142e923b35c23bdc13 feat(#192): Phase 4 — discoverability nudges at clone/status time Sonnet 5 patch 4 days ago