gabriel / musehub public
test_push_xs_unit.py python
430 lines 15.8 KB
Raw
sha256:4992098130166d191cefed0a2821d19cd3cdd3cf50867a4e715c2b30636826c7 fix: repair syntax errors from typing annotation cleanup Sonnet 4.6 20 days ago
1 """Push XS unit tests — issue #64.
2
3 One verb. One size. Proven correct at each layer before moving to the next.
4
5 P1 mpack integrity — object_id == sha256(content), mpack_key == sha256(mpack)
6 P2 muse push XS — real muse CLI pushes XS repo to localhost:1337, exit 0
7 P3 unpack stores correctly — sha256(stored_bytes) == object_id for every object
8 P4 mpack index rows — every object has an mpack index row with correct mpack_id
9 P5 fetch round-trip — fetch/mpack presigned URL unpacks to correct objects
10
11 Tests hit real infrastructure (musehub at localhost:1337, MinIO at localhost:9000).
12 No conftest. No ASGI. No mocks.
13 """
14 from __future__ import annotations
15
16 import asyncio
17 import json
18 import os
19 import shutil
20 import subprocess
21 import tempfile
22 import time as _time
23 from pathlib import Path
24
25 import msgpack
26 import pytest
27 from sqlalchemy import select
28 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
29 from sqlalchemy.orm import sessionmaker
30
31 from muse.core.types import blob_id
32 from musehub.db.musehub_repo_models import MusehubMPackIndex
33
34 _PROD_DB_URL = "postgresql+asyncpg://musehub:musehub@localhost:5434/musehub"
35
36
37 async def _wait_indexed(oid: str, timeout: float = 15.0) -> bool:
38 engine = create_async_engine(_PROD_DB_URL)
39 async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
40 try:
41 deadline = _time.monotonic() + timeout
42 while _time.monotonic() < deadline:
43 async with async_session() as session:
44 row = await session.scalar(
45 select(MusehubMPackIndex).where(MusehubMPackIndex.entity_id == oid)
46 )
47 if row is not None:
48 return True
49 await asyncio.sleep(0.5)
50 return False
51 finally:
52 await engine.dispose()
53
54
55 async def _fetch_index_rows(oid: str, timeout: float = 10.0) -> list[MusehubMPackIndex]:
56 engine = create_async_engine(_PROD_DB_URL)
57 async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
58 try:
59 deadline = _time.monotonic() + timeout
60 while _time.monotonic() < deadline:
61 async with async_session() as session:
62 result = await session.scalars(
63 select(MusehubMPackIndex).where(MusehubMPackIndex.entity_id == oid)
64 )
65 rows = result.all()
66 if rows:
67 return list(rows)
68 await asyncio.sleep(0.5)
69 return []
70 finally:
71 await engine.dispose()
72
73 LOCALHOST = "https://localhost:1337"
74 REPO_ROOT = Path(__file__).parent.parent
75
76
77 # ---------------------------------------------------------------------------
78 # Shared constants — XS is exactly 1 object, 4 KB, 1 commit
79 # ---------------------------------------------------------------------------
80
81 OBJ_CONTENT = b"a" * 4096
82 OBJ_ID = blob_id(OBJ_CONTENT)
83
84 COMMIT_ID = blob_id(b"xs-commit")
85 SNAPSHOT_ID = blob_id(b"xs-snapshot")
86
87
88 def _build_mpack() -> tuple[bytes, str]:
89 """Build the XS mpack and return (wire_bytes, mpack_key)."""
90 mpack = {
91 "commits": [{
92 "commit_id": COMMIT_ID,
93 "branch": "main",
94 "message": "xs unit test commit",
95 "author": "gabriel",
96 "committed_at": "2026-01-01T00:00:00+00:00",
97 "parent_commit_id": None,
98 "parent2_commit_id": None,
99 "snapshot_id": SNAPSHOT_ID,
100 "agent_id": "",
101 "model_id": "",
102 "toolchain_id": "",
103 "sem_ver_bump": "none",
104 "breaking_changes": [],
105 "signature": "",
106 "signer_key_id": "",
107 "signer_public_key": "",
108 "prompt_hash": "",
109 }],
110 "snapshots": [{
111 "snapshot_id": SNAPSHOT_ID,
112 "parent_snapshot_id": None,
113 "delta_upsert": {"file.txt": OBJ_ID},
114 "delta_remove": [],
115 }],
116 "blobs": [{"object_id": OBJ_ID, "content": OBJ_CONTENT}],
117 "branch_heads": {"main": COMMIT_ID},
118 }
119 wire_bytes = msgpack.packb(mpack, use_bin_type=True)
120 mpack_key = blob_id(wire_bytes)
121 return wire_bytes, mpack_key
122
123
124 def _muse(*args: str, cwd: Path) -> subprocess.CompletedProcess:
125 return subprocess.run(
126 ["muse"] + list(args),
127 cwd=str(cwd), capture_output=True, text=True, timeout=60,
128 )
129
130
131 def _muse_check(*args: str, cwd: Path) -> str:
132 r = _muse(*args, cwd=cwd)
133 if r.returncode != 0:
134 raise AssertionError(f"muse {' '.join(args)} failed:\n{r.stderr[:600]}")
135 return r.stdout
136
137
138 # ---------------------------------------------------------------------------
139 # P1 — mpack integrity (no network, no DB, no fixtures)
140 # ---------------------------------------------------------------------------
141
142 def test_p1_object_id_matches_content() -> None:
143 """object_id must equal sha256 of the raw content bytes."""
144 expected = blob_id(OBJ_CONTENT)
145 assert OBJ_ID == expected, (
146 f"object_id mismatch\n got: {OBJ_ID}\n expected: {expected}"
147 )
148
149
150 def test_p1_mpack_key_matches_wire_bytes() -> None:
151 """mpack_key must equal sha256 of the msgpack-encoded mpack."""
152 wire_bytes, mpack_key = _build_mpack()
153 expected = blob_id(wire_bytes)
154 assert mpack_key == expected, (
155 f"mpack_key mismatch\n got: {mpack_key}\n expected: {expected}"
156 )
157
158
159 def test_p1_mpack_objects_round_trip() -> None:
160 """Every object unpacked from the mpack must hash to its declared object_id."""
161 wire_bytes, _ = _build_mpack()
162 mpack = msgpack.unpackb(wire_bytes, raw=False)
163
164 for obj in mpack["blobs"]:
165 oid = obj["object_id"]
166 content = obj["content"]
167 computed = blob_id(content)
168 assert oid == computed, (
169 f"object content integrity failure\n"
170 f" declared object_id: {oid}\n"
171 f" sha256(content): {computed}"
172 )
173
174
175 # ---------------------------------------------------------------------------
176 # P2 — muse push XS to real localhost:1337 server, assert exit 0
177 # ---------------------------------------------------------------------------
178
179 def test_p2_muse_push_xs_exits_zero() -> None:
180 """muse push of a 1-commit, 1-file XS repo to localhost must exit 0."""
181 tmpdir = Path(tempfile.mkdtemp(prefix="muse_p2_"))
182 try:
183 # Init repo
184 _muse_check("init", cwd=tmpdir)
185
186 # Write one 4 KB file — same content as our mpack constants
187 (tmpdir / "file.txt").write_bytes(OBJ_CONTENT)
188 _muse_check("code", "add", "file.txt", cwd=tmpdir)
189 _muse_check(
190 "commit", "-m", "xs unit test commit",
191 "--agent-id", "bench", "--model-id", "bench",
192 cwd=tmpdir,
193 )
194
195 # Create a hub repo and push
196 name = f"bench-push-xs-p2-{os.urandom(3).hex()}"
197 out = _muse_check(
198 "hub", "repo", "create", "--name", name,
199 "--visibility", "public", "--no-init", "--hub", LOCALHOST, "--json",
200 cwd=REPO_ROOT,
201 )
202 slug = json.loads(out)["slug"]
203
204 _muse_check("remote", "add", "origin", f"{LOCALHOST}/gabriel/{slug}", cwd=tmpdir)
205 r = _muse("push", "origin", "main", cwd=tmpdir)
206
207 assert r.returncode == 0, (
208 f"muse push XS failed (exit {r.returncode})\n"
209 f"stdout: {r.stdout[:400]}\n"
210 f"stderr: {r.stderr[:400]}"
211 )
212 finally:
213 shutil.rmtree(tmpdir, ignore_errors=True)
214
215
216 # ---------------------------------------------------------------------------
217 # P3 — after muse push, mpack in MinIO has correct object bytes
218 # ---------------------------------------------------------------------------
219
220 def test_p3_pushed_mpack_in_minio_is_muse_format() -> None:
221 """After muse push XS, the mpack in MinIO must be in MUSE wire format and
222 must contain the pushed object with correct content.
223
224 Objects are no longer stored individually under objects/{oid} — they live
225 inside the covering mpack. This test verifies the mpack is parseable and
226 its content is intact.
227 """
228 import boto3
229 from muse.core.mpack import parse_wire_mpack
230
231 unique_content = os.urandom(4096)
232 expected_oid = blob_id(unique_content)
233
234 tmpdir = Path(tempfile.mkdtemp(prefix="muse_p3_"))
235 try:
236 _muse_check("init", cwd=tmpdir)
237 (tmpdir / "file.txt").write_bytes(unique_content)
238 _muse_check("code", "add", "file.txt", cwd=tmpdir)
239 _muse_check(
240 "commit", "-m", "xs p3 commit",
241 "--agent-id", "bench", "--model-id", "bench",
242 cwd=tmpdir,
243 )
244 name = f"bench-push-xs-p3-{os.urandom(3).hex()}"
245 out = _muse_check(
246 "hub", "repo", "create", "--name", name,
247 "--visibility", "public", "--no-init", "--hub", LOCALHOST, "--json",
248 cwd=REPO_ROOT,
249 )
250 slug = json.loads(out)["slug"]
251 _muse_check("remote", "add", "origin", f"{LOCALHOST}/gabriel/{slug}", cwd=tmpdir)
252 r = _muse("push", "origin", "main", cwd=tmpdir)
253 assert r.returncode == 0, f"push failed:\n{r.stderr[:400]}"
254 finally:
255 shutil.rmtree(tmpdir, ignore_errors=True)
256
257 # Wait for the mpack.index job to write the index row.
258 indexed = asyncio.run(_wait_indexed(expected_oid, timeout=15))
259 assert indexed, (
260 f"mpack.index job did not complete within 15s\n object_id: {expected_oid}"
261 )
262
263 # Retrieve the mpack_id from the index.
264 rows = asyncio.run(_fetch_index_rows(expected_oid, timeout=5))
265 assert rows, f"No mpack index row found for object_id: {expected_oid}"
266 mpack_id = rows[0].mpack_id
267
268 # Fetch the mpack from MinIO and verify it is MUSE wire format.
269 s3 = boto3.client(
270 "s3",
271 endpoint_url="http://localhost:9000",
272 aws_access_key_id="minioadmin",
273 aws_secret_access_key="minioadmin",
274 region_name="us-east-1",
275 )
276 wire_bytes = s3.get_object(Bucket="muse-objects", Key=f"mpacks/{mpack_id}")["Body"].read()
277
278 assert wire_bytes[:4] == b"MUSE", (
279 f"Mpack in MinIO is not MUSE format — got magic {wire_bytes[:4]!r}"
280 )
281 mpack = parse_wire_mpack(wire_bytes)
282
283 oids_in_pack = {o["object_id"] for o in mpack.get("blobs", [])}
284 assert expected_oid in oids_in_pack, (
285 f"Pushed object not found in mpack\n"
286 f" expected: {expected_oid}\n"
287 f" objects in mpack: {len(oids_in_pack)}"
288 )
289
290
291 # ---------------------------------------------------------------------------
292 # P4 — mpack index rows exist and mpack_id points to the correct mpack
293 # ---------------------------------------------------------------------------
294
295 def test_p4_mpack_index_has_row_for_pushed_object() -> None:
296 """After muse push XS, musehub_mpack_index must have a row for the pushed
297 object_id, and mpack_id must point to an mpack that contains that object.
298
299 Queries the real production DB directly (same DB the server uses).
300 """
301 import asyncio
302 # Known content — same derivation as P3
303 unique_content = os.urandom(4096)
304 expected_oid = blob_id(unique_content)
305
306 # Push
307 tmpdir = Path(tempfile.mkdtemp(prefix="muse_p4_"))
308 try:
309 _muse_check("init", cwd=tmpdir)
310 (tmpdir / "file.txt").write_bytes(unique_content)
311 _muse_check("code", "add", "file.txt", cwd=tmpdir)
312 _muse_check(
313 "commit", "-m", "xs p4 commit",
314 "--agent-id", "bench", "--model-id", "bench",
315 cwd=tmpdir,
316 )
317 name = f"bench-push-xs-p4-{os.urandom(3).hex()}"
318 out = _muse_check(
319 "hub", "repo", "create", "--name", name,
320 "--visibility", "public", "--no-init", "--hub", LOCALHOST, "--json",
321 cwd=REPO_ROOT,
322 )
323 slug = json.loads(out)["slug"]
324 _muse_check("remote", "add", "origin", f"{LOCALHOST}/gabriel/{slug}", cwd=tmpdir)
325 r = _muse("push", "origin", "main", cwd=tmpdir)
326 assert r.returncode == 0, f"push failed:\n{r.stderr[:400]}"
327 finally:
328 shutil.rmtree(tmpdir, ignore_errors=True)
329
330 # Query the real DB — poll up to 10s for the async mpack.index job to complete
331 rows = asyncio.run(_fetch_index_rows(expected_oid, timeout=10))
332
333 assert rows, (
334 f"No mpack index row for object after push\n"
335 f" object_id: {expected_oid}"
336 )
337
338 # Verify the mpack_id points to an mpack in MinIO that contains our object
339 import boto3
340 s3 = boto3.client(
341 "s3",
342 endpoint_url="http://localhost:9000",
343 aws_access_key_id="minioadmin",
344 aws_secret_access_key="minioadmin",
345 region_name="us-east-1",
346 )
347 bucket = "muse-objects"
348
349 for row in rows:
350 mpack_id = row.mpack_id
351 s3_key = f"mpacks/{mpack_id}"
352 try:
353 wire_bytes = s3.get_object(Bucket=bucket, Key=s3_key)["Body"].read()
354 except Exception as exc:
355 raise AssertionError(
356 f"mpack_id {mpack_id} not found in MinIO\n"
357 f" key tried: {s3_key}\n"
358 f" error: {exc}"
359 )
360
361 from muse.core.mpack import parse_wire_mpack
362 mpack = parse_wire_mpack(wire_bytes)
363 oids_in_pack = {obj["object_id"] for obj in mpack.get("blobs", [])}
364 assert expected_oid in oids_in_pack, (
365 f"mpack index row exists but mpack does not contain the object\n"
366 f" object_id: {expected_oid}\n"
367 f" mpack_id: {mpack_id}\n"
368 f" objects in mpack: {len(oids_in_pack)}"
369 )
370
371
372 # ---------------------------------------------------------------------------
373 # P5 — muse clone round-trip: push then clone, no integrity errors
374 # ---------------------------------------------------------------------------
375
376 def test_p5_muse_clone_xs_no_integrity_errors() -> None:
377 """Push an XS repo then clone it. The clone must exit 0 with no content
378 integrity errors. This is the exact failure mode from bench_cli.py.
379
380 If this passes, the full push → clone round-trip is correct for XS.
381 """
382 unique_content = os.urandom(4096)
383
384 # Push
385 push_dir = Path(tempfile.mkdtemp(prefix="muse_p5_push_"))
386 slug = None
387 try:
388 _muse_check("init", cwd=push_dir)
389 (push_dir / "file.txt").write_bytes(unique_content)
390 _muse_check("code", "add", "file.txt", cwd=push_dir)
391 _muse_check(
392 "commit", "-m", "xs p5 commit",
393 "--agent-id", "bench", "--model-id", "bench",
394 cwd=push_dir,
395 )
396 name = f"bench-push-xs-p5-{os.urandom(3).hex()}"
397 out = _muse_check(
398 "hub", "repo", "create", "--name", name,
399 "--visibility", "public", "--no-init", "--hub", LOCALHOST, "--json",
400 cwd=REPO_ROOT,
401 )
402 slug = json.loads(out)["slug"]
403 _muse_check("remote", "add", "origin", f"{LOCALHOST}/gabriel/{slug}", cwd=push_dir)
404 r = _muse("push", "origin", "main", cwd=push_dir)
405 assert r.returncode == 0, f"push failed:\n{r.stderr[:400]}"
406 finally:
407 shutil.rmtree(push_dir, ignore_errors=True)
408
409 # Wait for mpack.index job to complete (same as P4)
410 expected_oid = blob_id(unique_content)
411 indexed = asyncio.run(_wait_indexed(expected_oid, timeout=10))
412 assert indexed, "mpack index row never appeared — mpack.index job did not complete"
413
414 # Clone
415 clone_parent = Path(tempfile.mkdtemp(prefix="muse_p5_clone_"))
416 try:
417 r = _muse("clone", f"{LOCALHOST}/gabriel/{slug}", cwd=clone_parent)
418 assert r.returncode == 0, (
419 f"muse clone failed (exit {r.returncode})\n"
420 f"stdout: {r.stdout[:600]}\n"
421 f"stderr: {r.stderr[:600]}"
422 )
423 assert "integrity failure" not in r.stderr.lower(), (
424 f"clone exited 0 but reported integrity failures:\n{r.stderr[:600]}"
425 )
426 assert "corrupted object" not in r.stderr.lower(), (
427 f"clone exited 0 but reported corrupted objects:\n{r.stderr[:600]}"
428 )
429 finally:
430 shutil.rmtree(clone_parent, ignore_errors=True)
File History 2 commits
sha256:4992098130166d191cefed0a2821d19cd3cdd3cf50867a4e715c2b30636826c7 fix: repair syntax errors from typing annotation cleanup Sonnet 4.6 20 days ago
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ 20 days ago