"""TDD — push → clone round-trip correctness, built from first principles. Each tier proves one layer works before the next builds on it. Tier 1 — Storage layer T1a: put_mpack/get_mpack round-trip: sha256(get(key)) == key T1b: presigned PUT / get_mpack round-trip: sha256(get(key)) == key T1c: put_mpack with real MUSE-format bytes round-trips correctly Tier 2 — Wire push: one commit, one file T2: push a repo with 1 commit, 1 file → server accepts → mpack_index populated Tier 3 — Wire fetch after push T3: fetch after push returns the blob that was pushed Tier 4 — Full clone T4: clone after push produces a working tree with correct file content Run with: python3 -m pytest tests/test_push_clone_roundtrip.py -v --tb=short """ from __future__ import annotations import hashlib import json import secrets import subprocess import urllib.request from pathlib import Path import pytest # ── MinIO / hub constants ────────────────────────────────────────────────────── MINIO_ENDPOINT = "http://localhost:9000" MINIO_BUCKET = "muse-objects" MINIO_ACCESS_KEY = "minioadmin" MINIO_SECRET_KEY = "minioadmin" HUB = "https://localhost:1337" REPO_ROOT = Path(__file__).parent.parent # ── reachability guards ──────────────────────────────────────────────────────── def _minio_up() -> bool: try: urllib.request.urlopen(f"{MINIO_ENDPOINT}/minio/health/live", timeout=2) return True except Exception: return False def _hub_up() -> bool: try: import ssl ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE urllib.request.urlopen(f"{HUB}/healthz", context=ctx, timeout=3) return True except Exception: return False requires_minio = pytest.mark.skipif(not _minio_up(), reason="MinIO not reachable at localhost:9000") requires_hub = pytest.mark.skipif(not _hub_up(), reason="Hub not reachable at localhost:1337") # ── storage backend helper ───────────────────────────────────────────────────── def _backend(): from musehub.storage.backends import BlobBackend return BlobBackend( bucket=MINIO_BUCKET, endpoint_url=MINIO_ENDPOINT, public_endpoint_url=MINIO_ENDPOINT, access_key_id=MINIO_ACCESS_KEY, secret_access_key=MINIO_SECRET_KEY, region="us-east-1", ) def _sha256_key(data: bytes) -> str: return "sha256:" + hashlib.sha256(data).hexdigest() # ── muse subprocess helper ───────────────────────────────────────────────────── def muse(*args: str, cwd: Path, timeout: int = 120, check: bool = True) -> subprocess.CompletedProcess: r = subprocess.run( ["muse"] + list(args), cwd=str(cwd), capture_output=True, text=True, timeout=timeout, ) if check and r.returncode != 0: raise AssertionError( f"muse {' '.join(args)} failed (exit {r.returncode}):\n" f"stdout: {r.stdout[:600]}\nstderr: {r.stderr[:600]}" ) return r # ── Tier 1: Storage layer ────────────────────────────────────────────────────── class TestStorageRoundTrip: """Tier 1: prove the mpack storage layer is content-addressed.""" @requires_minio async def test_t1a_put_mpack_get_mpack(self) -> None: """put_mpack(key, data) → get_mpack(key) must return data where sha256==key.""" backend = _backend() data = secrets.token_bytes(256) key = _sha256_key(data) await backend.put_mpack(key, data) result = await backend.get_mpack(key) assert result is not None, "get_mpack returned None" assert _sha256_key(result) == key, ( f"sha256(result)={_sha256_key(result)[:30]} ≠ key={key[:30]}" ) assert result == data @requires_minio async def test_t1b_presign_put_get_mpack(self) -> None: """Presigned PUT → get_mpack must return data where sha256==key. This is the exact path used by muse push: client PUTs to presigned URL, server later reads via get_mpack. Any key-encoding mismatch surfaces here. """ backend = _backend() data = secrets.token_bytes(256) key = _sha256_key(data) upload_url = await backend.presign_mpack_put(key, ttl_seconds=300) req = urllib.request.Request( upload_url, data=data, method="PUT", headers={"Content-Type": "application/x-muse-pack"}, ) with urllib.request.urlopen(req) as resp: assert resp.status == 200, f"presigned PUT → HTTP {resp.status}" result = await backend.get_mpack(key) assert result is not None, ( f"get_mpack returned None after presigned PUT\n" f"upload_url={upload_url}" ) assert _sha256_key(result) == key, ( f"sha256(result)={_sha256_key(result)[:30]} ≠ key={key[:30]}\n" f"upload_url={upload_url}" ) @requires_minio @pytest.mark.parametrize("size_mb", [0.001, 0.1, 1, 10, 50]) async def test_t1b_presign_put_large_payload(self, size_mb: float) -> None: """Presigned PUT → get_mpack round-trip at increasing sizes. Production mpacks range from ~100 KB to ~400 MB. A bug that only manifests above a certain size will show up here. """ backend = _backend() data = secrets.token_bytes(int(size_mb * 1024 * 1024)) key = _sha256_key(data) upload_url = await backend.presign_mpack_put(key, ttl_seconds=300) req = urllib.request.Request( upload_url, data=data, method="PUT", headers={"Content-Type": "application/x-muse-pack"}, ) with urllib.request.urlopen(req) as resp: assert resp.status == 200, f"presigned PUT ({size_mb} MB) → HTTP {resp.status}" result = await backend.get_mpack(key) assert result is not None, f"get_mpack returned None for {size_mb} MB payload" assert _sha256_key(result) == key, ( f"{size_mb} MB: sha256(result)={_sha256_key(result)[:30]} ≠ key={key[:30]}" ) @requires_minio async def test_t1c_real_muse_mpack_format(self) -> None: """A real MUSE-format mpack round-trips correctly through put/get.""" from muse.core.mpack import build_wire_mpack # Minimal valid mpack payload mpack = { "commits": [], "snapshots": [], "blobs": [{"object_id": _sha256_key(b"hello"), "content": b"hello"}], "tags": [], } data = build_wire_mpack(mpack) key = _sha256_key(data) backend = _backend() await backend.put_mpack(key, data) result = await backend.get_mpack(key) assert result is not None assert _sha256_key(result) == key assert result == data # ── Tier 2: Wire push ────────────────────────────────────────────────────────── @pytest.fixture def tmp_repo(tmp_path: Path) -> Path: """Fresh muse repo with 3 commits and varied file content.""" repo = tmp_path / "src" repo.mkdir() muse("init", cwd=repo) # commit 1 (repo / "hello.txt").write_text("hello world\n") (repo / "data.bin").write_bytes(secrets.token_bytes(1024)) (repo / "notes.txt").write_text("line one\nline two\nline three\n") muse("code", "add", ".", cwd=repo) muse("commit", "-m", "initial commit", "--agent-id", "test", "--model-id", "test", cwd=repo) # commit 2 — modify one file, add one (repo / "hello.txt").write_text("hello world v2\n") (repo / "extra.bin").write_bytes(secrets.token_bytes(512)) muse("code", "add", ".", cwd=repo) muse("commit", "-m", "second commit", "--agent-id", "test", "--model-id", "test", cwd=repo) # commit 3 — delete one file, modify another (repo / "notes.txt").write_text("updated notes\n") muse("code", "add", ".", cwd=repo) muse("commit", "-m", "third commit", "--agent-id", "test", "--model-id", "test", cwd=repo) return repo @pytest.fixture def hub_repo(tmp_path: Path): """Create a fresh hub repo, yield its slug, delete after test.""" name = f"roundtrip-probe-{tmp_path.name[-8:]}" out = muse( "-C", str(REPO_ROOT), "hub", "repo", "create", "--name", name, "--visibility", "public", "--no-init", "--hub", HUB, "--json", cwd=REPO_ROOT, ) slug = json.loads(out.stdout)["slug"] full = f"gabriel/{slug}" yield full muse( "-C", str(REPO_ROOT), "hub", "repo", "delete", full, "--yes", "--hub", HUB, "--json", cwd=REPO_ROOT, check=False, ) @requires_hub class TestPushCloneRoundTrip: """Tier 2–4: push a repo, clone it, verify every file matches exactly.""" def test_t2_push_succeeds(self, tmp_repo: Path, hub_repo: str) -> None: """muse push must complete without error.""" hub_url = f"{HUB}/{hub_repo}" muse("remote", "add", "origin", hub_url, cwd=tmp_repo) muse("push", "origin", "main", cwd=tmp_repo, timeout=180) def test_t3_clone_succeeds(self, tmp_repo: Path, hub_repo: str, tmp_path: Path) -> None: """muse clone after push must complete without error.""" hub_url = f"{HUB}/{hub_repo}" muse("remote", "add", "origin", hub_url, cwd=tmp_repo) muse("push", "origin", "main", cwd=tmp_repo, timeout=180) clone_dir = tmp_path / "clone" muse("-C", str(REPO_ROOT), "clone", hub_url, str(clone_dir), cwd=REPO_ROOT, timeout=180) assert clone_dir.exists(), "clone directory was not created" def test_t4_clone_files_match_source(self, tmp_repo: Path, hub_repo: str, tmp_path: Path) -> None: """Every file in the cloned working tree must match the source exactly.""" hub_url = f"{HUB}/{hub_repo}" muse("remote", "add", "origin", hub_url, cwd=tmp_repo) muse("push", "origin", "main", cwd=tmp_repo, timeout=180) clone_dir = tmp_path / "clone" muse("-C", str(REPO_ROOT), "clone", hub_url, str(clone_dir), cwd=REPO_ROOT, timeout=180) # Every file tracked in the source must exist in the clone with identical content source_files = { p.relative_to(tmp_repo): p.read_bytes() for p in tmp_repo.rglob("*") if p.is_file() and ".muse" not in p.parts } assert source_files, "no source files found — fixture broken" mismatches = [] for rel, expected in source_files.items(): cloned = clone_dir / rel if not cloned.exists(): mismatches.append(f"MISSING: {rel}") elif cloned.read_bytes() != expected: mismatches.append(f"WRONG CONTENT: {rel}") assert not mismatches, "Clone → source mismatches:\n" + "\n".join(mismatches)