gabriel / musehub public
test_wire_fetch_step1.py python
448 lines 19.7 KB
Raw
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ breaking 58 days ago
1 """TDD — POST /{owner}/{slug}/fetch — Step 1 of the fetch protocol (issue #68).
2
3 The fetch endpoint is the server side of the three-step fetch flow:
4
5 1. POST /{owner}/{slug}/fetch
6 body: {"want": ["sha256:..."], "have": ["sha256:..."]}
7 → {"mpack_url": "...", "mpack_id": "sha256:...", "commit_count": N, "object_count": N}
8
9 2. Client GETs mpack bytes directly from MinIO via presigned URL.
10 3. Client calls apply_mpack() locally.
11
12 Tests:
13 F01 Happy path — want=[tip], have=[] → 200, mpack_url set, sha256 mpack_id.
14 F02 Up-to-date — want=[tip], have=[tip] → 200, commit_count=0, object_count=0, mpack_url null.
15 F03 Delta — want=[c2], have=[c1] → only new commit + new object in delta.
16 F04 Missing repo → 404.
17 F05 Empty want → 422.
18 F06 Malformed want entry (not sha256:*) → 422.
19 F07 Unknown want commit_id → 404.
20 F08 Objects not in pack_index → 503 with Retry-After header.
21 F09 Private repo without auth → 404.
22 F10 mpack_id == sha256(mpack_bytes) — content-addressing proof survives end-to-end.
23 """
24 from __future__ import annotations
25
26 import hashlib
27 from collections.abc import Mapping
28 from datetime import datetime, timezone
29 from unittest.mock import AsyncMock
30
31 import msgpack
32 import pytest
33 from httpx import AsyncClient
34 from sqlalchemy.dialects.postgresql import insert as pg_insert
35 from sqlalchemy.ext.asyncio import AsyncSession
36
37 from muse.core.mpack import parse_wire_mpack
38 from muse.core.types import blob_id, fake_id
39 from musehub.db.musehub_repo_models import (
40 MusehubBranch,
41 MusehubCommit,
42 MusehubCommitGraph,
43 MusehubCommitRef,
44 MusehubMPackIndex,
45 MusehubObject,
46 MusehubObjectRef,
47 MusehubSnapshot,
48 MusehubSnapshotRef,
49 )
50 from tests.factories import create_repo
51 from musehub.types.json_types import StrDict
52
53 type _ByteStore = dict[str, bytes]
54
55
56 # ── helpers ───────────────────────────────────────────────────────────────────
57
58 def _now() -> datetime:
59 return datetime.now(tz=timezone.utc)
60
61
62 def _stub_backend(monkeypatch: pytest.MonkeyPatch) -> _ByteStore:
63 """Replace MinIO backend with an in-memory dict. Returns the store."""
64 store: dict[str, bytes] = {}
65
66 async def _put(oid: str, data: bytes, **_) -> str:
67 store[oid] = data
68 return f"mem://{oid}"
69
70 async def _get(oid: str) -> bytes | None:
71 return store.get(oid)
72
73 async def _exists(oid: str, **_) -> bool:
74 return oid in store
75
76 async def _presign_get(oid: str, ttl_seconds: int = 3600) -> str:
77 return f"https://minio.test/mpacks/{oid}?ttl={ttl_seconds}"
78
79 async def _get_mpack(mpack_id: str) -> bytes | None:
80 return store.get(mpack_id)
81
82 async def _put_mpack(mpack_id: str, data: bytes) -> str:
83 store[mpack_id] = data
84 return f"mem://mpacks/{mpack_id}"
85
86 async def _presign_mpack_get(mpack_id: str, ttl: int) -> str:
87 return f"https://minio.test/mpacks/{mpack_id}?ttl={ttl}"
88
89 backend = AsyncMock()
90 backend.put = _put
91 backend.get = _get
92 backend.exists = _exists
93 backend.presign_get = _presign_get
94 backend.get_mpack = _get_mpack
95 backend.put_mpack = _put_mpack
96 backend.presign_mpack_get = _presign_mpack_get
97 backend.supports_presign = True
98 monkeypatch.setattr("musehub.services.musehub_wire.get_backend", lambda: backend)
99 monkeypatch.setattr("musehub.services.musehub_wire_fetch.get_backend", lambda: backend)
100 monkeypatch.setattr("musehub.services.musehub_wire_shared.get_backend", lambda: backend)
101 return store
102
103
104 async def _store_object(
105 session: AsyncSession,
106 repo_id: str,
107 oid: str,
108 content: bytes,
109 *,
110 indexed: bool = True,
111 mpack_key: str = "sha256:" + "a" * 64,
112 ) -> None:
113 """Write a musehub_objects row + ref + optional pack_index entry."""
114 await session.execute(
115 pg_insert(MusehubObject)
116 .values(object_id=oid, path="file.dat", size_bytes=len(content),
117 storage_uri=f"mem://{oid}", content_cache=content)
118 .on_conflict_do_nothing(index_elements=["object_id"])
119 )
120 await session.execute(
121 pg_insert(MusehubObjectRef)
122 .values(repo_id=repo_id, object_id=oid)
123 .on_conflict_do_nothing()
124 )
125 if indexed:
126 await session.execute(
127 pg_insert(MusehubMPackIndex)
128 .values(entity_id=oid, mpack_id=mpack_key, entity_type="object")
129 .on_conflict_do_nothing()
130 )
131 await session.commit()
132
133
134 async def _make_commit(
135 session: AsyncSession,
136 repo_id: str,
137 *,
138 manifest: dict[str, str],
139 seed: str,
140 parent_ids: list[str] | None = None,
141 generation: int = 0,
142 ) -> tuple[MusehubCommit, MusehubSnapshot]:
143 snap_id = fake_id(f"snap-f01-{seed}")
144 snap = MusehubSnapshot(
145 snapshot_id=snap_id,
146 directories=[],
147 manifest_blob=msgpack.packb(manifest, use_bin_type=True),
148 entry_count=len(manifest),
149 created_at=_now(),
150 )
151 session.add(snap)
152 await session.execute(
153 pg_insert(MusehubSnapshotRef)
154 .values(repo_id=repo_id, snapshot_id=snap_id)
155 .on_conflict_do_nothing()
156 )
157
158 commit_id = fake_id(f"commit-f01-{seed}")
159 pids = parent_ids or []
160 commit = MusehubCommit(
161 commit_id=commit_id,
162 branch="main",
163 parent_ids=pids,
164 message=f"commit {seed}",
165 author="gabriel",
166 timestamp=_now(),
167 snapshot_id=snap_id,
168 )
169 session.add(commit)
170 await session.execute(
171 pg_insert(MusehubCommitRef)
172 .values(repo_id=repo_id, commit_id=commit_id)
173 .on_conflict_do_nothing()
174 )
175 graph = MusehubCommitGraph(
176 commit_id=commit_id,
177 parent_ids=pids,
178 generation=generation,
179 snapshot_id=snap_id,
180 )
181 session.add(graph)
182
183 from sqlalchemy import select as _select
184 branch_q = await session.execute(
185 _select(MusehubBranch).where(
186 MusehubBranch.repo_id == repo_id,
187 MusehubBranch.name == "main",
188 )
189 )
190 branch = branch_q.scalar_one_or_none()
191 if branch:
192 branch.head_commit_id = commit_id
193 await session.commit()
194 return commit, snap
195
196
197 def _fetch(
198 client: AsyncClient,
199 owner: str,
200 slug: str,
201 want: list[str],
202 have: list[str],
203 headers: dict[str, str] | None = None,
204 ):
205 merged = {"Content-Type": "application/x-msgpack", **(headers or {})}
206 return client.post(
207 f"/{owner}/{slug}/fetch",
208 content=msgpack.packb({"want": want, "have": have}, use_bin_type=True),
209 headers=merged,
210 )
211
212
213 # ══════════════════════════════════════════════════════════════════════════════
214 # F01 — happy path: want=[tip], have=[] → 200, mpack_url, sha256 mpack_id
215 # ══════════════════════════════════════════════════════════════════════════════
216
217 @pytest.mark.asyncio
218 async def test_f01_happy_path_returns_presigned_url(
219 client: AsyncClient, db_session: AsyncSession,
220 monkeypatch: pytest.MonkeyPatch, wire_headers: StrDict,
221 ) -> None:
222 """want=[tip], have=[] → 200, mpack_url non-empty, mpack_id sha256-prefixed."""
223 _stub_backend(monkeypatch)
224 repo = await create_repo(db_session, owner="gabriel", visibility="public")
225 raw = b"hello fetch world"
226 oid = blob_id(raw)
227 await _store_object(db_session, repo.repo_id, oid, raw)
228 commit, _ = await _make_commit(db_session, repo.repo_id, manifest={"f.txt": oid}, seed="f01")
229
230 resp = await _fetch(client, "gabriel", repo.slug, [commit.commit_id], [], headers=wire_headers)
231
232 assert resp.status_code == 200
233 data = msgpack.unpackb(resp.content, raw=False)
234 assert data["mpack_id"].startswith("sha256:"), f"bad mpack_id: {data['mpack_id']!r}"
235 assert data["mpack_url"], "mpack_url must be non-empty"
236 assert data["commit_count"] == 1
237 assert data["object_count"] == 1
238
239
240 # ══════════════════════════════════════════════════════════════════════════════
241 # F02 — up-to-date: want=[tip], have=[tip] → 200, all counts zero, mpack_url null
242 # ══════════════════════════════════════════════════════════════════════════════
243
244 @pytest.mark.asyncio
245 async def test_f02_up_to_date_returns_zero_counts(
246 client: AsyncClient, db_session: AsyncSession,
247 monkeypatch: pytest.MonkeyPatch, wire_headers: StrDict,
248 ) -> None:
249 """Client already has everything — server returns 200 with commit_count=0."""
250 _stub_backend(monkeypatch)
251 repo = await create_repo(db_session, owner="gabriel", visibility="public")
252 raw = b"already have this"
253 oid = blob_id(raw)
254 await _store_object(db_session, repo.repo_id, oid, raw)
255 commit, _ = await _make_commit(db_session, repo.repo_id, manifest={"g.txt": oid}, seed="f02")
256
257 resp = await _fetch(client, "gabriel", repo.slug, [commit.commit_id], [commit.commit_id],
258 headers=wire_headers)
259
260 assert resp.status_code == 200
261 data = msgpack.unpackb(resp.content, raw=False)
262 assert data["commit_count"] == 0
263 assert data["object_count"] == 0
264 assert not data.get("mpack_url"), "mpack_url must be null/empty when nothing to send"
265
266
267 # ══════════════════════════════════════════════════════════════════════════════
268 # F03 — delta: want=[c2], have=[c1] → only new commit + new object
269 # ══════════════════════════════════════════════════════════════════════════════
270
271 @pytest.mark.asyncio
272 async def test_f03_delta_excludes_have_side_objects(
273 client: AsyncClient, db_session: AsyncSession,
274 monkeypatch: pytest.MonkeyPatch, wire_headers: StrDict,
275 ) -> None:
276 """have=[c1] cuts the delta — c1 and its object must not appear in the mpack."""
277 store = _stub_backend(monkeypatch)
278 repo = await create_repo(db_session, owner="gabriel", visibility="public")
279
280 raw1 = b"base object"
281 oid1 = blob_id(raw1)
282 await _store_object(db_session, repo.repo_id, oid1, raw1)
283 c1, _ = await _make_commit(db_session, repo.repo_id,
284 manifest={"base.txt": oid1}, seed="f03-c1", generation=0)
285
286 raw2 = b"delta object"
287 oid2 = blob_id(raw2)
288 await _store_object(db_session, repo.repo_id, oid2, raw2)
289 c2, _ = await _make_commit(db_session, repo.repo_id,
290 manifest={"base.txt": oid1, "delta.txt": oid2},
291 seed="f03-c2", parent_ids=[c1.commit_id], generation=1)
292
293 resp = await _fetch(client, "gabriel", repo.slug, [c2.commit_id], [c1.commit_id],
294 headers=wire_headers)
295
296 assert resp.status_code == 200
297 data = msgpack.unpackb(resp.content, raw=False)
298 assert data["commit_count"] == 1
299 assert data["object_count"] == 1
300
301 mpack = parse_wire_mpack(store[data["mpack_id"]])
302 commit_ids = {c["commit_id"] for c in mpack["commits"]}
303 assert c2.commit_id in commit_ids
304 assert c1.commit_id not in commit_ids, "have-side commit must be excluded"
305
306 obj_ids = {o["object_id"] for o in mpack["objects"]}
307 assert oid2 in obj_ids
308 assert oid1 not in obj_ids, "have-side object must be excluded"
309
310
311 # ══════════════════════════════════════════════════════════════════════════════
312 # F04 — missing repo → 404
313 # ══════════════════════════════════════════════════════════════════════════════
314
315 @pytest.mark.asyncio
316 async def test_f04_missing_repo_returns_404(
317 client: AsyncClient, wire_headers: StrDict,
318 ) -> None:
319 """POST to a repo that doesn't exist → 404."""
320 resp = await _fetch(client, "nobody", "no-such-repo",
321 ["sha256:" + "a" * 64], [], headers=wire_headers)
322 assert resp.status_code == 404
323
324
325 # ══════════════════════════════════════════════════════════════════════════════
326 # F05 — empty want → 422
327 # ══════════════════════════════════════════════════════════════════════════════
328
329 @pytest.mark.asyncio
330 async def test_f05_empty_want_returns_422(
331 client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict,
332 ) -> None:
333 """Empty want list → 422 (not a valid fetch request)."""
334 repo = await create_repo(db_session, owner="gabriel", visibility="public")
335 resp = await _fetch(client, "gabriel", repo.slug, [], [], headers=wire_headers)
336 assert resp.status_code == 422
337
338
339 # ══════════════════════════════════════════════════════════════════════════════
340 # F06 — malformed want entry → 422
341 # ══════════════════════════════════════════════════════════════════════════════
342
343 @pytest.mark.asyncio
344 async def test_f06_malformed_want_entry_returns_422(
345 client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict,
346 ) -> None:
347 """want entries that don't match sha256:* → 422."""
348 repo = await create_repo(db_session, owner="gabriel", visibility="public")
349 resp = await _fetch(client, "gabriel", repo.slug, ["not-a-hash"], [], headers=wire_headers)
350 assert resp.status_code == 422
351
352
353 # ══════════════════════════════════════════════════════════════════════════════
354 # F07 — unknown want commit_id → 404
355 # ══════════════════════════════════════════════════════════════════════════════
356
357 @pytest.mark.asyncio
358 async def test_f07_unknown_want_commit_returns_404(
359 client: AsyncClient, db_session: AsyncSession, wire_headers: StrDict,
360 ) -> None:
361 """want contains a sha256 ID that doesn't exist in musehub_commits → 404."""
362 repo = await create_repo(db_session, owner="gabriel", visibility="public")
363 ghost_id = "sha256:" + "b" * 64
364 resp = await _fetch(client, "gabriel", repo.slug, [ghost_id], [], headers=wire_headers)
365 assert resp.status_code == 404
366
367
368 # ══════════════════════════════════════════════════════════════════════════════
369 # F08 — objects not in pack_index → 503 with Retry-After
370 # ══════════════════════════════════════════════════════════════════════════════
371
372 @pytest.mark.asyncio
373 async def test_f08_unindexed_objects_returns_503(
374 client: AsyncClient, db_session: AsyncSession,
375 monkeypatch: pytest.MonkeyPatch, wire_headers: StrDict,
376 ) -> None:
377 """Objects that haven't been indexed in musehub_mpack_index → 503 Retry-After.
378
379 This happens when unpack-mpack returned 200 but the mpack.index background
380 job hasn't completed yet. The client must retry rather than receiving a
381 partial or corrupt fetch mpack.
382 """
383 _stub_backend(monkeypatch)
384 repo = await create_repo(db_session, owner="gabriel", visibility="public")
385 raw = b"not yet indexed"
386 oid = blob_id(raw)
387 # Write object WITHOUT a pack_index entry (indexed=False)
388 await _store_object(db_session, repo.repo_id, oid, raw, indexed=False)
389 commit, _ = await _make_commit(db_session, repo.repo_id,
390 manifest={"pending.txt": oid}, seed="f08")
391
392 resp = await _fetch(client, "gabriel", repo.slug, [commit.commit_id], [],
393 headers=wire_headers)
394
395 assert resp.status_code == 503
396 assert "retry-after" in {k.lower() for k in resp.headers}, (
397 "503 response must include Retry-After header"
398 )
399
400
401 # ══════════════════════════════════════════════════════════════════════════════
402 # F09 — private repo without auth → 404
403 # ══════════════════════════════════════════════════════════════════════════════
404
405 @pytest.mark.asyncio
406 async def test_f09_private_repo_without_auth_returns_404(
407 client: AsyncClient, db_session: AsyncSession,
408 ) -> None:
409 """Private repo must 404 for unauthenticated requests (don't leak existence)."""
410 repo = await create_repo(db_session, owner="gabriel", visibility="private")
411 resp = await _fetch(client, "gabriel", repo.slug,
412 ["sha256:" + "c" * 64], [])
413 assert resp.status_code == 404
414
415
416 # ══════════════════════════════════════════════════════════════════════════════
417 # F10 — mpack_id == sha256(mpack_bytes) end-to-end
418 # ══════════════════════════════════════════════════════════════════════════════
419
420 @pytest.mark.asyncio
421 async def test_f10_mpack_id_equals_sha256_of_stored_bytes(
422 client: AsyncClient, db_session: AsyncSession,
423 monkeypatch: pytest.MonkeyPatch, wire_headers: StrDict,
424 ) -> None:
425 """mpack_id in the HTTP response == sha256(bytes written to MinIO).
426
427 Content-addressing is the integrity proof — no secondary verification needed.
428 """
429 store = _stub_backend(monkeypatch)
430 repo = await create_repo(db_session, owner="gabriel", visibility="public")
431 raw = b"content-addressing proof"
432 oid = blob_id(raw)
433 await _store_object(db_session, repo.repo_id, oid, raw)
434 commit, _ = await _make_commit(db_session, repo.repo_id,
435 manifest={"proof.bin": oid}, seed="f10")
436
437 resp = await _fetch(client, "gabriel", repo.slug, [commit.commit_id], [],
438 headers=wire_headers)
439
440 assert resp.status_code == 200
441 data = msgpack.unpackb(resp.content, raw=False)
442 mpack_id = data["mpack_id"]
443
444 stored_bytes = store[mpack_id]
445 expected_id = blob_id(stored_bytes)
446 assert mpack_id == expected_id, (
447 f"mpack_id {mpack_id!r} != blob_id(stored_bytes) {expected_id!r}"
448 )
File History 1 commit
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ 58 days ago