gabriel / muse public
migrate.py python
867 lines 31.8 KB
Raw
sha256:707b4461365efd52a6360948db35151dd01ccdfbb8e5c62d6ed14afca7a0dc35 migrate_blob_ids now correctly skips snapshot and commit ob… Human patch 43 days ago
1 """Full layout and commit-ID migration engine.
2
3 Migrates any Muse repository from legacy on-disk layouts to the canonical
4 current layout:
5
6 Old layout Canonical layout
7 -------------------------------------- ---------------------------------------
8 objects/<shard>/<rest> objects/sha256/<shard>/<rest>
9 commits/<hex>.msgpack commits/sha256/<hex>.msgpack
10 snapshots/<hex>.msgpack snapshots/sha256/<hex>.msgpack
11 branch ref: bare hex branch ref: sha256:<hex>
12 remote ref: bare hex or stale ID remote ref: sha256:<current-hex>
13 repo_id: plain string (pre-sha256) repo_id: sha256:<hex>
14 commit field: created_on_branch commit field: branch
15 commit field: format_version < 8 commit field: format_version = 8
16 commit signature: bare base64url commit signature: ed25519:<base64url>
17 commit_id: legacy formula commit_id: hash_commit(...)
18
19 Entry points
20 ------------
21 - :func:`migrate` — full execution or dry-run
22 - :class:`MigrateResult` — returned by :func:`migrate`
23 """
24
25 import collections
26 import datetime
27 import json
28 import logging
29 import pathlib
30 import shutil
31 from dataclasses import dataclass, field
32 from typing import TYPE_CHECKING
33
34 import msgpack
35
36 from muse.core.paths import (
37 commits_dir, heads_dir, logs_dir, muse_dir,
38 objects_dir, rebase_merge_dir, remotes_dir,
39 repo_json_path, snapshots_dir,
40 )
41 from muse.core.ids import hash_blob, hash_commit, hash_snapshot
42 from muse.core.types import b64url_decode, blob_id, encode_sig, long_id
43 from muse.core.refs import get_all_branch_heads, write_branch_ref
44
45 if TYPE_CHECKING:
46 from muse.core.transport import SigningIdentity
47
48 logger = logging.getLogger(__name__)
49
50 type _IdMap = dict[str, str]
51 type _RawCommit = dict[str, str | int | float | bytes | datetime.datetime | None]
52
53
54 # ---------------------------------------------------------------------------
55 # Phase 7 — blob ID migration result
56 # ---------------------------------------------------------------------------
57
58 @dataclass
59 class BlobMigrateResult:
60 """Result of :func:`migrate_blob_ids`."""
61 id_map: dict[str, str] = field(default_factory=dict)
62 blobs_written: int = 0
63 blobs_skipped: int = 0
64 dry_run: bool = False
65 type _CommitEntry = tuple[_RawCommit, pathlib.Path]
66 type _CommitIndex = dict[str, _CommitEntry]
67
68 # ---------------------------------------------------------------------------
69 # Result type
70 # ---------------------------------------------------------------------------
71
72 @dataclass
73 class MigrateResult:
74 id_map: dict[str, str] = field(default_factory=dict)
75 commits_rewritten: int = 0
76 blobs_migrated: int = 0
77 legacy_dirs_removed: int = 0
78 dry_run: bool = False
79 commits_relocated: int = 0
80 snapshots_relocated: int = 0
81 refs_updated: int = 0
82 remote_refs_updated: int = 0
83 repo_id_updated: bool = False
84 branch_fields_renamed: int = 0
85 signatures_normalised: int = 0
86 format_versions_bumped: int = 0
87 reflogs_updated: int = 0
88 commits_signed: int = 0
89 unsigned_commits_skipped: int = 0
90 # Phase 7 — unified object store backfill
91 blobs_rewritten: int = 0
92 snapshots_rewritten: int = 0
93 commits_unified: int = 0
94
95 # ---------------------------------------------------------------------------
96 # Internal helpers
97 # ---------------------------------------------------------------------------
98
99 def _normalise_sig(sig: str) -> str:
100 if not sig or sig.startswith("ed25519:"):
101 return sig
102 raw = b64url_decode(sig)
103 return encode_sig("ed25519", raw)
104
105 def _committed_at_iso(raw: _RawCommit) -> str:
106 val = raw.get("committed_at", "")
107 if isinstance(val, datetime.datetime):
108 return val.isoformat()
109 return str(val)
110
111 def _collect_all_commits(root: pathlib.Path) -> _CommitIndex:
112 """Read every commit file in the store.
113
114 Returns a mapping of commit_id → (raw_dict, file_path). Covers:
115 - Legacy msgpack files in ``.muse/commits/sha256/``
116 - Unified object store entries in ``.muse/objects/sha256/`` with a
117 ``commit <size>\\0<json>`` header
118 """
119 result: _CommitIndex = {}
120
121 # Legacy msgpack commits
122 _dir = commits_dir(root)
123 if _dir.exists():
124 for path in _dir.rglob("*.msgpack"):
125 try:
126 raw = msgpack.unpackb(path.read_bytes(), raw=False)
127 cid = raw.get("commit_id", "")
128 if cid:
129 result[cid] = (raw, path)
130 except Exception as exc:
131 logger.warning("Skipping unreadable commit %s: %s", path, exc)
132
133 # Unified object store commits (commit <size>\0<json>)
134 obj_dir = objects_dir(root) / "sha256"
135 if obj_dir.exists():
136 for path in obj_dir.glob("*/*"):
137 if not path.is_file():
138 continue
139 try:
140 data = path.read_bytes()
141 if not data.startswith(b"commit "):
142 continue
143 nl = data.index(b"\x00")
144 raw = json.loads(data[nl + 1:])
145 cid = raw.get("commit_id", "")
146 if cid and cid not in result:
147 result[cid] = (raw, path)
148 except Exception as exc:
149 logger.warning("Skipping unreadable object %s: %s", path, exc)
150
151 return result
152
153 def _topological_sort(commits: _CommitIndex) -> list[str]:
154 """Return commit IDs in topological order (parents before children)."""
155 in_degree: dict[str, int] = {cid: 0 for cid in commits}
156 children: dict[str, list[str]] = {cid: [] for cid in commits}
157
158 for cid, (raw, _) in commits.items():
159 for key in ("parent_commit_id", "parent2_commit_id"):
160 pid = raw.get(key)
161 if pid and pid in commits:
162 in_degree[cid] = in_degree.get(cid, 0) + 1
163 children[pid].append(cid)
164
165 queue: collections.deque[str] = collections.deque(
166 sorted(cid for cid, deg in in_degree.items() if deg == 0)
167 )
168 order: list[str] = []
169 while queue:
170 cid = queue.popleft()
171 order.append(cid)
172 for child in sorted(children.get(cid, [])):
173 in_degree[child] -= 1
174 if in_degree[child] == 0:
175 queue.append(child)
176 return order
177
178 def _make_canonical_dict(
179 raw: _RawCommit,
180 id_map: _IdMap,
181 signing_identity: SigningIdentity | None = None,
182 force_resign: bool = False,
183 ) -> tuple[_RawCommit, str, bool]:
184 """Build the canonical version of a raw commit dict.
185
186 Returns ``(canonical_dict, new_commit_id, was_signed)``.
187 ``was_signed`` is True when a new signature was created (unsigned commit,
188 parent cascade, or ``force_resign``).
189 """
190 p1 = raw.get("parent_commit_id")
191 p2 = raw.get("parent2_commit_id")
192 new_p1 = id_map.get(p1, p1) if p1 else None
193 new_p2 = id_map.get(p2, p2) if p2 else None
194
195 parent_ids: list[str] = []
196 if new_p1:
197 parent_ids.append(new_p1)
198 if new_p2:
199 parent_ids.append(new_p2)
200
201 snapshot_id: str = raw.get("snapshot_id", "")
202 message: str = raw.get("message", "")
203 committed_at_iso = _committed_at_iso(raw)
204 author: str = raw.get("author", "")
205
206 parents_changed = (new_p1 != p1) or (new_p2 != p2)
207 is_unsigned = not raw.get("signature") and not raw.get("signer_public_key")
208 # Re-sign when: (a) commit was unsigned, (b) parent IDs changed so the
209 # old signature covers the wrong commit_id, or (c) force_resign is set.
210 should_sign = signing_identity is not None and (is_unsigned or parents_changed or force_resign)
211
212 # Normalise empty author to the signing identity's handle so every
213 # resigned commit carries proper attribution baked into its content hash.
214 if should_sign and signing_identity is not None and not author:
215 author = signing_identity.handle
216
217 if should_sign:
218 from muse.core.provenance import encode_public_key, provenance_payload, sign_commit_ed25519
219 _, pub_str = encode_public_key(signing_identity.private_key) # type: ignore[arg-type]
220 new_id = hash_commit(
221 parent_ids=parent_ids,
222 snapshot_id=snapshot_id,
223 message=message,
224 committed_at_iso=committed_at_iso,
225 author=author,
226 signer_public_key=pub_str,
227 )
228 payload = provenance_payload(
229 commit_id=new_id,
230 author=author,
231 agent_id=raw.get("agent_id", ""),
232 model_id=raw.get("model_id", ""),
233 toolchain_id=raw.get("toolchain_id", ""),
234 prompt_hash=raw.get("prompt_hash", ""),
235 committed_at=committed_at_iso,
236 )
237 sig = sign_commit_ed25519(payload, signing_identity.private_key) # type: ignore[arg-type]
238 else:
239 pub_str = raw.get("signer_public_key", "")
240 new_id = hash_commit(
241 parent_ids=parent_ids,
242 snapshot_id=snapshot_id,
243 message=message,
244 committed_at_iso=committed_at_iso,
245 author=author,
246 signer_public_key=pub_str,
247 )
248 sig = _normalise_sig(raw.get("signature", ""))
249
250 branch_val = raw.get("branch") or raw.get("created_on_branch") or "main"
251
252 out = dict(raw)
253 out["commit_id"] = new_id
254 out["parent_commit_id"] = new_p1
255 out["parent2_commit_id"] = new_p2
256 out["author"] = author
257 out["signature"] = sig
258 out["branch"] = branch_val
259 out["format_version"] = 8
260 out["committed_at"] = committed_at_iso
261 if should_sign:
262 out["signer_public_key"] = pub_str
263 out.pop("created_on_branch", None)
264 out.pop("repo_id", None)
265
266 return out, new_id, should_sign
267
268 def _canonical_commit_path(root: pathlib.Path, hex_id: str) -> pathlib.Path:
269 return commits_dir(root) / "sha256" / f"{hex_id}.msgpack"
270
271 def _write_raw_commit(root: pathlib.Path, canonical: _RawCommit) -> None:
272 hex_id = long_id(canonical["commit_id"], strip=True)
273 path = _canonical_commit_path(root, hex_id)
274 path.parent.mkdir(parents=True, exist_ok=True)
275 path.write_bytes(msgpack.packb(canonical, use_bin_type=True))
276
277 def _is_flat_commit_path(root: pathlib.Path, path: pathlib.Path) -> bool:
278 _dir = commits_dir(root)
279 try:
280 rel = path.relative_to(_dir)
281 except ValueError:
282 return False
283 return rel.parts[0] != "sha256"
284
285 def _migrate_objects(root: pathlib.Path, dry_run: bool) -> tuple[int, int]:
286 _dir = objects_dir(root)
287 if not _dir.exists():
288 return 0, 0
289
290 blobs_moved = 0
291 dirs_removed = 0
292
293 for entry in sorted(_dir.iterdir()):
294 if not entry.is_dir() or entry.name == "sha256":
295 continue
296 prefix = entry.name
297 for blob_path in sorted(entry.iterdir()):
298 if not blob_path.is_file():
299 continue
300 rest = blob_path.name
301 canonical = _dir / "sha256" / prefix / rest
302 if canonical.exists():
303 if not dry_run:
304 blob_path.unlink()
305 else:
306 if not dry_run:
307 canonical.parent.mkdir(parents=True, exist_ok=True)
308 blob_path.rename(canonical)
309 blobs_moved += 1
310 if not dry_run and entry.is_dir() and not any(entry.iterdir()):
311 entry.rmdir()
312 dirs_removed += 1
313
314 return blobs_moved, dirs_removed
315
316 def _migrate_snapshots(root: pathlib.Path, dry_run: bool) -> int:
317 snaps_dir = snapshots_dir(root)
318 if not snaps_dir.exists():
319 return 0
320 count = 0
321 for path in sorted(snaps_dir.iterdir()):
322 if not path.is_file() or not path.name.endswith(".msgpack"):
323 continue
324 hex_id = path.stem
325 canonical = snaps_dir / "sha256" / f"{hex_id}.msgpack"
326 if canonical.exists():
327 if not dry_run:
328 path.unlink()
329 else:
330 if not dry_run:
331 canonical.parent.mkdir(parents=True, exist_ok=True)
332 path.rename(canonical)
333 count += 1
334 return count
335
336 def _normalise_ref_value(val: str, id_map: _IdMap) -> str:
337 val = val.strip()
338 val = long_id(val)
339 return id_map.get(val, val)
340
341 def _migrate_refs(root: pathlib.Path, id_map: _IdMap, dry_run: bool) -> int:
342 refs_dir = heads_dir(root)
343 if not refs_dir.exists():
344 return 0
345 count = 0
346 for path in sorted(refs_dir.rglob("*")):
347 if not path.is_file():
348 continue
349 val = path.read_text(encoding="utf-8").strip()
350 new_val = _normalise_ref_value(val, id_map)
351 if new_val != val:
352 count += 1
353 if not dry_run:
354 path.write_text(f"{new_val}\n", encoding="utf-8")
355 return count
356
357 def _migrate_remote_refs(
358 root: pathlib.Path, id_map: dict[str, str], dry_run: bool
359 ) -> int:
360 _dir = remotes_dir(root)
361 if not _dir.exists():
362 return 0
363 count = 0
364 for path in sorted(_dir.rglob("*")):
365 if not path.is_file():
366 continue
367 val = path.read_text(encoding="utf-8").strip()
368 new_val = _normalise_ref_value(val, id_map)
369 if new_val != val:
370 count += 1
371 if not dry_run:
372 path.write_text(f"{new_val}\n", encoding="utf-8")
373 return count
374
375 def _migrate_repo_id(root: pathlib.Path, dry_run: bool) -> bool:
376 repo_json = repo_json_path(root)
377 if not repo_json.exists():
378 return False
379 data = json.loads(repo_json.read_text(encoding="utf-8"))
380 rid = data.get("repo_id", "")
381 if rid.startswith("sha256:"):
382 return False
383 new_id = blob_id(rid.encode())
384 if not dry_run:
385 data["repo_id"] = new_id
386 repo_json.write_text(json.dumps(data), encoding="utf-8")
387 return True
388
389 def _migrate_reflogs(
390 root: pathlib.Path, id_map: dict[str, str], dry_run: bool
391 ) -> int:
392 _dir = logs_dir(root)
393 if not _dir.exists():
394 return 0
395 count = 0
396 for path in sorted(_dir.rglob("*")):
397 if not path.is_file():
398 continue
399 content = path.read_text(encoding="utf-8")
400 new_content = content
401 for old_id, new_id in id_map.items():
402 new_content = new_content.replace(old_id, new_id)
403 if new_content != content:
404 count += 1
405 if not dry_run:
406 path.write_text(new_content, encoding="utf-8")
407 return count
408
409 # ---------------------------------------------------------------------------
410 # Phase 7 — blob ID migration
411 # ---------------------------------------------------------------------------
412
413 def migrate_blob_ids(
414 repo_root: pathlib.Path,
415 dry_run: bool = False,
416 ) -> BlobMigrateResult:
417 """Migrate raw (old-formula) blobs to muse-format at new hash_blob paths.
418
419 For each blob stored with the pre-Phase-2 formula ``sha256(content)``,
420 this function:
421
422 1. Reads the raw bytes from the old object path.
423 2. Detects whether the file is already muse-format (has a valid
424 ``"blob <size>\\0"`` header whose declared size matches the payload).
425 3. If old-format: writes ``"blob <size>\\0<content>"`` to the new object
426 path derived from ``hash_blob(content)``.
427 4. Builds and returns an old_id → new_id map for use by the snapshot
428 migration pass.
429
430 Non-destructive: old files are never deleted.
431
432 Args:
433 repo_root: Repository root (contains ``.muse/``).
434 dry_run: When True, compute the mapping but write nothing.
435
436 Returns:
437 :class:`BlobMigrateResult` with the ID map and write counts.
438 """
439 from muse.core.object_store import iter_stored_objects, object_path as _object_path
440
441 result = BlobMigrateResult(dry_run=dry_run)
442
443 for old_id, path in iter_stored_objects(repo_root):
444 data = path.read_bytes()
445
446 # Detect typed objects: "blob|snapshot|commit <size>\0<payload>".
447 # Snapshot and commit objects must never be treated as blobs.
448 null_idx = data.find(b"\0")
449 if null_idx > 0:
450 header = data[:null_idx].decode(errors="replace")
451 parts = header.split(" ", 1)
452 if len(parts) == 2 and parts[1].isdigit() and len(data) - null_idx - 1 == int(parts[1]):
453 if parts[0] in ("snapshot", "commit"):
454 # Not a blob — skip entirely, never rewrite.
455 result.id_map[old_id] = old_id
456 result.blobs_skipped += 1
457 continue
458 if parts[0] == "blob":
459 # Already muse-format blob — map to itself and skip.
460 result.id_map[old_id] = old_id
461 result.blobs_skipped += 1
462 continue
463
464 # Old-format raw blob: compute the new muse-idiomatic ID.
465 new_id = hash_blob(data)
466 result.id_map[old_id] = new_id
467
468 if new_id == old_id:
469 result.blobs_skipped += 1
470 continue
471
472 new_path = _object_path(repo_root, new_id)
473 if new_path.exists():
474 # New blob already written — delete the stale old-format file.
475 if not dry_run:
476 path.unlink(missing_ok=True)
477 result.blobs_skipped += 1
478 continue
479
480 if not dry_run:
481 header_bytes = f"blob {len(data)}\0".encode()
482 new_path.parent.mkdir(parents=True, exist_ok=True)
483 new_path.write_bytes(header_bytes + data)
484 # Remove the old-format file now that the new one is safely written.
485 path.unlink(missing_ok=True)
486
487 result.blobs_written += 1
488
489 return result
490
491
492 # ---------------------------------------------------------------------------
493 # Phase 7 — snapshot ID migration
494 # ---------------------------------------------------------------------------
495
496 @dataclass
497 class SnapshotMigrateResult:
498 """Result of :func:`migrate_snapshot_ids`."""
499 id_map: dict[str, str] = field(default_factory=dict)
500 snapshots_written: int = 0
501 snapshots_skipped: int = 0
502 dry_run: bool = False
503
504
505 def migrate_snapshot_ids(
506 repo_root: pathlib.Path,
507 blob_id_map: dict[str, str],
508 dry_run: bool = False,
509 ) -> SnapshotMigrateResult:
510 """Migrate old-formula snapshots to muse-format in the unified object store.
511
512 For each snapshot msgpack in ``.muse/snapshots/sha256/``:
513
514 1. Reads the record and updates every manifest entry using *blob_id_map*
515 (old blob ID → new blob ID).
516 2. Recomputes the snapshot ID with the current ``hash_snapshot`` formula.
517 3. Writes the updated record as JSON to ``.muse/objects/sha256/<2>/<62>``
518 in ``"snapshot <size>\\0<json>"`` format.
519
520 Non-destructive: old msgpack files are never deleted.
521
522 Args:
523 repo_root: Repository root (contains ``.muse/``).
524 blob_id_map: Old blob ID → new blob ID map from :func:`migrate_blob_ids`.
525 dry_run: When True, compute the mapping but write nothing.
526
527 Returns:
528 :class:`SnapshotMigrateResult` with the ID map and write counts.
529 """
530 from muse.core.object_store import object_path as _object_path
531
532 result = SnapshotMigrateResult(dry_run=dry_run)
533 snaps = snapshots_dir(repo_root) / "sha256"
534 if not snaps.exists():
535 return result
536
537 from muse.core.io import zstd_decompress_if_needed
538
539 for path in sorted(snaps.glob("*.msgpack")):
540 try:
541 raw = msgpack.unpackb(zstd_decompress_if_needed(path.read_bytes()), raw=False)
542 except Exception as exc:
543 logger.warning("Skipping unreadable snapshot %s: %s", path, exc)
544 continue
545
546 old_id: str = raw.get("snapshot_id", "")
547 if not old_id:
548 continue
549
550 old_manifest: dict[str, str] = dict(raw.get("manifest") or {})
551 new_manifest = {
552 file_path: blob_id_map.get(oid, oid)
553 for file_path, oid in old_manifest.items()
554 }
555 directories = raw.get("directories") or []
556 new_snap_id = hash_snapshot(new_manifest, directories or None)
557 result.id_map[old_id] = new_snap_id
558
559 new_path = _object_path(repo_root, new_snap_id)
560 if new_path.exists():
561 result.snapshots_skipped += 1
562 continue
563
564 if not dry_run:
565 updated = dict(raw)
566 updated["snapshot_id"] = new_snap_id
567 updated["manifest"] = new_manifest
568 # created_at may be a datetime — normalise to ISO string for JSON.
569 if hasattr(updated.get("created_at"), "isoformat"):
570 updated["created_at"] = updated["created_at"].isoformat()
571 payload = json.dumps(updated, separators=(",", ":")).encode()
572 header = f"snapshot {len(payload)}\0".encode()
573 new_path.parent.mkdir(parents=True, exist_ok=True)
574 new_path.write_bytes(header + payload)
575
576 result.snapshots_written += 1
577
578 return result
579
580
581 # ---------------------------------------------------------------------------
582 # Phase 7 — commit ID migration
583 # ---------------------------------------------------------------------------
584
585 @dataclass
586 class CommitMigrateResult:
587 """Result of :func:`migrate_commit_ids`."""
588 id_map: dict[str, str] = field(default_factory=dict)
589 commits_written: int = 0
590 commits_skipped: int = 0
591 dry_run: bool = False
592
593
594 def migrate_commit_ids(
595 repo_root: pathlib.Path,
596 snapshot_id_map: dict[str, str],
597 dry_run: bool = False,
598 ) -> CommitMigrateResult:
599 """Migrate old-formula commits to muse-format in the unified object store.
600
601 For each commit msgpack in ``.muse/commits/sha256/``:
602
603 1. Reads the record and updates ``snapshot_id`` using *snapshot_id_map*.
604 2. Updates parent commit IDs from the result's own ``id_map`` (two-pass:
605 all new IDs are computed first, then files are written).
606 3. Recomputes the commit ID with the current ``hash_commit`` formula.
607 4. Writes the updated record as JSON to ``.muse/objects/sha256/<2>/<62>``
608 in ``"commit <size>\\0<json>"`` format.
609
610 Non-destructive: old msgpack files are never deleted.
611
612 Args:
613 repo_root: Repository root (contains ``.muse/``).
614 snapshot_id_map: Old snapshot ID → new snapshot ID map from
615 :func:`migrate_snapshot_ids`.
616 dry_run: When True, compute the mapping but write nothing.
617
618 Returns:
619 :class:`CommitMigrateResult` with the ID map and write counts.
620 """
621 from muse.core.object_store import object_path as _object_path
622
623 result = CommitMigrateResult(dry_run=dry_run)
624 all_commits = _collect_all_commits(repo_root)
625 order = _topological_sort(all_commits)
626
627 # Pass 1: build the commit id_map (parents-before-children order ensures
628 # parent IDs are already in id_map when processing each child).
629 commit_id_map: dict[str, str] = {}
630 for old_id in order:
631 raw, _ = all_commits[old_id]
632 p1 = raw.get("parent_commit_id")
633 p2 = raw.get("parent2_commit_id")
634 new_p1 = commit_id_map.get(p1, p1) if p1 else None
635 new_p2 = commit_id_map.get(p2, p2) if p2 else None
636
637 parent_ids: list[str] = [x for x in (new_p1, new_p2) if x]
638 old_snap = raw.get("snapshot_id", "")
639 new_snap = snapshot_id_map.get(old_snap, old_snap)
640 committed_at_iso = _committed_at_iso(raw)
641
642 new_id = hash_commit(
643 parent_ids=parent_ids,
644 snapshot_id=new_snap,
645 message=raw.get("message", ""),
646 committed_at_iso=committed_at_iso,
647 author=raw.get("author", ""),
648 signer_public_key=raw.get("signer_public_key", "") or "",
649 )
650 if new_id != old_id:
651 commit_id_map[old_id] = new_id
652
653 result.id_map = {**commit_id_map}
654
655 # Pass 2: write updated records.
656 for old_id in order:
657 raw, _ = all_commits[old_id]
658 p1 = raw.get("parent_commit_id")
659 p2 = raw.get("parent2_commit_id")
660 new_p1 = commit_id_map.get(p1, p1) if p1 else None
661 new_p2 = commit_id_map.get(p2, p2) if p2 else None
662
663 parent_ids = [x for x in (new_p1, new_p2) if x]
664 old_snap = raw.get("snapshot_id", "")
665 new_snap = snapshot_id_map.get(old_snap, old_snap)
666 new_id = commit_id_map.get(old_id, old_id)
667 committed_at_iso = _committed_at_iso(raw)
668
669 obj_path = _object_path(repo_root, new_id)
670 if obj_path.exists():
671 result.commits_skipped += 1
672 continue
673
674 if not dry_run:
675 updated = dict(raw)
676 updated["commit_id"] = new_id
677 updated["snapshot_id"] = new_snap
678 updated["parent_commit_id"] = new_p1
679 updated["parent2_commit_id"] = new_p2
680 updated["committed_at"] = committed_at_iso
681 updated.pop("created_on_branch", None)
682
683 payload = json.dumps(updated, separators=(",", ":"), default=str).encode()
684 header = f"commit {len(payload)}\0".encode()
685 obj_path.parent.mkdir(parents=True, exist_ok=True)
686 obj_path.write_bytes(header + payload)
687
688
689 result.commits_written += 1
690
691 return result
692
693
694 # ---------------------------------------------------------------------------
695 # Public API
696 # ---------------------------------------------------------------------------
697
698 def migrate(
699 root: pathlib.Path,
700 dry_run: bool = False,
701 signing_identity: SigningIdentity | None = None,
702 force_resign: bool = False,
703 ) -> MigrateResult:
704 """Migrate a Muse repository to the canonical current layout.
705
706 Args:
707 root: Repo root (contains ``.muse/``).
708 dry_run: When True, compute all changes and report them but
709 write nothing to disk.
710 signing_identity: When provided, unsigned commits are signed with this
711 identity during migration. Already-signed commits
712 are never re-signed.
713
714 Returns:
715 :class:`MigrateResult` with full migration summary.
716
717 Raises:
718 RuntimeError: If a merge or rebase is in progress.
719 """
720 if force_resign and signing_identity is None:
721 raise ValueError(
722 "force_resign=True requires a signing identity. "
723 "Run `muse auth keygen && muse auth register`, or connect a hub "
724 "with `muse hub connect` so credentials can be resolved."
725 )
726 if (muse_dir(root) / "MERGE_STATE").exists():
727 raise RuntimeError(
728 "A merge is in progress — finish or abort it before running migrate."
729 )
730 if rebase_merge_dir(root).exists():
731 raise RuntimeError(
732 "A rebase is in progress — finish or abort it before running migrate."
733 )
734
735 # --- repo.json ---
736 repo_id_updated = _migrate_repo_id(root, dry_run=dry_run)
737
738 # --- snapshots ---
739 snapshots_relocated = _migrate_snapshots(root, dry_run=dry_run)
740
741 # --- collect all commits (flat + canonical) ---
742 all_commits = _collect_all_commits(root)
743 order = _topological_sort(all_commits)
744
745 id_map: dict[str, str] = {}
746 commits_rewritten = 0
747 commits_relocated = 0
748 branch_fields_renamed = 0
749 signatures_normalised = 0
750 format_versions_bumped = 0
751 commits_signed = 0
752 unsigned_commits_skipped = 0
753
754 # First pass: build id_map (must be complete before second pass).
755 # Signing-aware: if signing_identity is provided, unsigned commits will
756 # get a new signer_public_key baked into their commit_id.
757 for old_id in order:
758 raw, _ = all_commits[old_id]
759 _, new_id, _ = _make_canonical_dict(raw, id_map, signing_identity, force_resign)
760 if new_id != old_id:
761 id_map[old_id] = new_id
762
763 # Second pass: write canonical files, collect stats
764 for old_id in order:
765 raw, old_path = all_commits[old_id]
766 canonical, new_id, was_signed = _make_canonical_dict(raw, id_map, signing_identity, force_resign)
767
768 is_flat = _is_flat_commit_path(root, old_path)
769 id_changed = new_id != old_id
770 branch_renamed = "created_on_branch" in raw and "branch" not in raw
771 sig_normalised = bool(raw.get("signature")) and not raw["signature"].startswith("ed25519:")
772 fv_bumped = raw.get("format_version", 1) != 8
773 is_unsigned = not raw.get("signature") and not raw.get("signer_public_key")
774 has_repo_id = "repo_id" in raw
775
776 if id_changed:
777 commits_rewritten += 1
778 if is_flat:
779 commits_relocated += 1
780 if branch_renamed:
781 branch_fields_renamed += 1
782 if sig_normalised:
783 signatures_normalised += 1
784 if fv_bumped:
785 format_versions_bumped += 1
786 if was_signed:
787 commits_signed += 1
788 elif is_unsigned and signing_identity is None:
789 unsigned_commits_skipped += 1
790
791 needs_write = id_changed or is_flat or branch_renamed or sig_normalised or fv_bumped or was_signed or has_repo_id
792 if not dry_run and needs_write:
793 _write_raw_commit(root, canonical)
794 # Delete old file if it's at a different location than the new canonical path
795 new_hex = long_id(new_id, strip=True)
796 new_path = _canonical_commit_path(root, new_hex)
797 if old_path != new_path and old_path.exists():
798 old_path.unlink()
799
800 # --- objects (legacy layout: objects/<shard> → objects/sha256/<shard>) ---
801 blobs_migrated, legacy_dirs_removed = _migrate_objects(root, dry_run=dry_run)
802
803 # --- Phase 7: ID chain migration (blobs → snapshots → commits) ---
804 # Pass 1: rewrite raw blobs to muse-format at new hash_blob paths.
805 blob_result = migrate_blob_ids(root, dry_run=dry_run)
806 # Pass 2: update snapshot manifests with new blob IDs, write to object store.
807 snap_result = migrate_snapshot_ids(root, blob_result.id_map, dry_run=dry_run)
808 # Pass 3: update commit snapshot_id fields, recompute commit IDs, write to
809 # object store and new msgpack paths.
810 commit_result = migrate_commit_ids(root, snap_result.id_map, dry_run=dry_run)
811
812 # Build the combined old→new ID map for ref and reflog updates.
813 #
814 # The existing loop maps old_id → legacy_id (parent cascade, signing).
815 # Phase 7 maps legacy_id → phase7_id (full ID chain).
816 #
817 # A naïve merge ({**id_map, **commit_result.id_map}) only resolves one hop:
818 # refs containing old_id would land at legacy_id, not phase7_id. We need
819 # the transitive closure: old_id → phase7_id in one step.
820 combined_id_map: dict[str, str] = {}
821 for old_id, legacy_id in id_map.items():
822 final_id = commit_result.id_map.get(legacy_id, legacy_id)
823 combined_id_map[old_id] = final_id
824 # Also carry through any Phase 7 entries not covered by the legacy map
825 # (commits that were already at their legacy form before this run).
826 for legacy_id, phase7_id in commit_result.id_map.items():
827 if legacy_id not in combined_id_map:
828 combined_id_map[legacy_id] = phase7_id
829
830 # --- branch refs ---
831 refs_updated = _migrate_refs(root, combined_id_map, dry_run=dry_run)
832
833 # --- remote refs ---
834 remote_refs_updated = _migrate_remote_refs(root, combined_id_map, dry_run=dry_run)
835
836 # --- reflogs ---
837 reflogs_updated = _migrate_reflogs(root, combined_id_map, dry_run=dry_run)
838
839 # --- remove legacy directories ---
840 # All content has been written to the unified object store by this point.
841 # Orphaned objects unreferenced by any branch will be collected by gc.
842 if not dry_run:
843 for legacy_dir in (commits_dir(root), snapshots_dir(root)):
844 if legacy_dir.exists():
845 shutil.rmtree(legacy_dir)
846
847 return MigrateResult(
848 id_map=combined_id_map,
849 commits_rewritten=commits_rewritten,
850 blobs_migrated=blobs_migrated,
851 legacy_dirs_removed=legacy_dirs_removed,
852 dry_run=dry_run,
853 commits_relocated=commits_relocated,
854 snapshots_relocated=snapshots_relocated,
855 refs_updated=refs_updated,
856 remote_refs_updated=remote_refs_updated,
857 repo_id_updated=repo_id_updated,
858 branch_fields_renamed=branch_fields_renamed,
859 signatures_normalised=signatures_normalised,
860 format_versions_bumped=format_versions_bumped,
861 reflogs_updated=reflogs_updated,
862 commits_signed=commits_signed,
863 unsigned_commits_skipped=unsigned_commits_skipped,
864 blobs_rewritten=blob_result.blobs_written,
865 snapshots_rewritten=snap_result.snapshots_written,
866 commits_unified=commit_result.commits_written,
867 )
File History 1 commit
sha256:707b4461365efd52a6360948db35151dd01ccdfbb8e5c62d6ed14afca7a0dc35 migrate_blob_ids now correctly skips snapshot and commit ob… Human patch 43 days ago