gabriel / musehub public
test_wire_mpack_presign_step1_e2e.py python
372 lines 14.1 KB
Raw
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ breaking 20 days ago
1 """E2E — Push Protocol Step 1: POST /{owner}/{slug}/push/mpack-presign.
2
3 Exercises the entire Step 1 pseudocode flow end-to-end against the real ASGI
4 app with a real DB session and a mocked MinIO backend. Every pseudocode step
5 is logged so failures are easy to locate.
6
7 Pseudocode under test
8 ---------------------
9 Client computes mpack_id = "sha256:" + sha256(mpack_bytes).hexdigest()
10 Client sends: { mpack_key: mpack_id, size_bytes: len(mpack_bytes) }
11
12 Server:
13 authenticate(request) # MSign → claims.identity_id, claims.handle
14 validate:
15 mpack_key present → 422 if missing
16 size_bytes ≤ mpack_max_bytes → 413 if exceeded
17 if mpack_daily_upload_limit_bytes > 0:
18 today_bytes = SUM(musehub_daily_push_bytes WHERE identity_id=me AND date=today)
19 if today_bytes >= daily_limit → 429
20 INSERT INTO musehub_daily_push_bytes (...)
21 COMMIT
22 upload_url = MinIO.presign_put("mpacks/" + mpack_key, ttl=3600s)
23 respond: { upload_url, mpack_key }
24
25 Tests
26 -----
27 S1E2E-1 Happy path: valid payload → 200, upload_url and mpack_key in response.
28 S1E2E-2 Missing mpack_key → 422.
29 S1E2E-3 size_bytes exceeds mpack_max_bytes → 413.
30 S1E2E-4 Daily quota exhausted → 429.
31 S1E2E-5 Quota row written to musehub_daily_push_bytes after successful presign.
32 S1E2E-6 Quota is cumulative: second presign adds to existing daily row.
33 S1E2E-7 Unauthenticated request → 401/403.
34 """
35 from __future__ import annotations
36
37 import datetime
38 import logging
39
40 import msgpack
41 import pytest
42 import pytest_asyncio
43 from httpx import AsyncClient, ASGITransport
44 from sqlalchemy import select, func
45 from sqlalchemy.ext.asyncio import AsyncSession
46
47 from muse.core.mpack import build_presign_payload
48 from muse.core.types import blob_id
49 from musehub.auth.dependencies import require_valid_token
50 from musehub.auth.request_signing import MSignContext
51 from musehub.config import get_settings
52 from musehub.db.database import get_db
53 from musehub.db.musehub_abuse_models import MusehubDailyPushBytes
54 from musehub.main import app
55 from musehub.services.musehub_repository import create_repo
56 from musehub.core.genesis import compute_identity_id
57
58 logger = logging.getLogger(__name__)
59
60 _FAKE_UPLOAD_URL = "https://minio.example.com/mpacks/sha256:fake?sig=presigned"
61 _OWNER = "gabriel"
62 _IDENTITY_ID = compute_identity_id(b"gabriel")
63 _REPO_NAME = "step1-e2e-test"
64
65 _AUTH_CTX = MSignContext(
66 handle=_OWNER,
67 identity_id=_IDENTITY_ID,
68 is_agent=False,
69 is_admin=False,
70 )
71
72
73 # ---------------------------------------------------------------------------
74 # Fixtures
75 # ---------------------------------------------------------------------------
76
77 @pytest_asyncio.fixture()
78 async def client(db_session: AsyncSession) -> None:
79 async def _override_db() -> None:
80 yield db_session
81
82 app.dependency_overrides[get_db] = _override_db
83 app.dependency_overrides[require_valid_token] = lambda: _AUTH_CTX
84
85 async with AsyncClient(
86 transport=ASGITransport(app=app),
87 base_url="https://localhost:1337",
88 ) as c:
89 yield c
90
91 app.dependency_overrides.clear()
92
93
94 @pytest_asyncio.fixture()
95 async def repo(db_session: AsyncSession) -> MusehubRepo:
96 r = await create_repo(
97 db_session,
98 name=_REPO_NAME,
99 owner=_OWNER,
100 owner_user_id=_IDENTITY_ID,
101 visibility="public",
102 initialize=False,
103 )
104 await db_session.commit()
105 return r
106
107
108 @pytest_asyncio.fixture(autouse=True)
109 async def mock_presign_put() -> None:
110 """Stub MinIO presign_put so tests don't need a live object store."""
111 from unittest.mock import AsyncMock, MagicMock, patch
112
113 mock_backend = MagicMock()
114 mock_backend.presign_mpack_put = AsyncMock(return_value=_FAKE_UPLOAD_URL)
115 with patch("musehub.services.musehub_wire.get_backend", return_value=mock_backend), \
116 patch("musehub.services.musehub_wire_push.get_backend", return_value=mock_backend):
117 yield mock_backend
118
119
120 def _presign_body(mpack_bytes: bytes) -> bytes:
121 payload = build_presign_payload(mpack_bytes)
122 logger.info(
123 "[step1] CLIENT: computed mpack_key=%s size_bytes=%d",
124 payload["mpack_key"][:27], payload["size_bytes"],
125 )
126 return msgpack.packb(payload, use_bin_type=True)
127
128
129 # ---------------------------------------------------------------------------
130 # S1E2E-1 — Happy path
131 # ---------------------------------------------------------------------------
132
133 @pytest.mark.asyncio
134 async def test_s1e2e1_happy_path(client: AsyncClient, repo: MusehubRepo) -> None:
135 """Full Step 1 happy path: valid payload → 200, upload_url + mpack_key returned."""
136 mpack_bytes = b"commits+snapshots+objects" * 100
137 expected_key = blob_id(mpack_bytes)
138
139 logger.info("[step1] CLIENT: building presign payload")
140 body = _presign_body(mpack_bytes)
141
142 logger.info("[step1] CLIENT: POST /%s/%s/push/mpack-presign", _OWNER, _REPO_NAME)
143 resp = await client.post(
144 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
145 content=body,
146 headers={"Content-Type": "application/x-msgpack"},
147 )
148 logger.info("[step1] SERVER: responded HTTP %d", resp.status_code)
149 assert resp.status_code == 200, f"expected 200, got {resp.status_code}: {resp.text}"
150
151 data = resp.json()
152 logger.info("[step1] SERVER: upload_url=%s mpack_key=%s", str(data.get("upload_url", ""))[:40], str(data.get("mpack_key", ""))[:27])
153
154 logger.info("[step1] ASSERT: upload_url is present and non-empty")
155 assert data.get("upload_url"), "upload_url missing from response"
156
157 logger.info("[step1] ASSERT: mpack_key echoed back matches client-computed key")
158 assert data["mpack_key"] == expected_key
159
160
161 # ---------------------------------------------------------------------------
162 # S1E2E-2 — Missing mpack_key → 422
163 # ---------------------------------------------------------------------------
164
165 @pytest.mark.asyncio
166 async def test_s1e2e2_missing_mpack_key_returns_422(client: AsyncClient, repo: MusehubRepo) -> None:
167 """Server validates mpack_key present → 422 when absent."""
168 body = msgpack.packb({"size_bytes": 1024}, use_bin_type=True)
169
170 logger.info("[step1] CLIENT: sending body WITHOUT mpack_key")
171 resp = await client.post(
172 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
173 content=body,
174 headers={"Content-Type": "application/x-msgpack"},
175 )
176 logger.info("[step1] SERVER: responded HTTP %d (expected 422)", resp.status_code)
177 assert resp.status_code == 422, f"expected 422, got {resp.status_code}"
178
179
180 # ---------------------------------------------------------------------------
181 # S1E2E-3 — size_bytes exceeds limit → 413
182 # ---------------------------------------------------------------------------
183
184 @pytest.mark.asyncio
185 async def test_s1e2e3_oversized_payload_returns_413(client: AsyncClient, repo: MusehubRepo) -> None:
186 """Server rejects size_bytes > mpack_max_bytes with 413."""
187 settings = get_settings()
188 over_limit = settings.mpack_max_bytes + 1
189
190 mpack_bytes = b"x"
191 payload = {"mpack_key": blob_id(mpack_bytes), "size_bytes": over_limit}
192 body = msgpack.packb(payload, use_bin_type=True)
193
194 logger.info(
195 "[step1] CLIENT: sending size_bytes=%d (limit=%d)",
196 over_limit, settings.mpack_max_bytes,
197 )
198 resp = await client.post(
199 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
200 content=body,
201 headers={"Content-Type": "application/x-msgpack"},
202 )
203 logger.info("[step1] SERVER: responded HTTP %d (expected 413)", resp.status_code)
204 assert resp.status_code == 413, f"expected 413, got {resp.status_code}"
205
206
207 # ---------------------------------------------------------------------------
208 # S1E2E-4 — Daily quota exhausted → 429
209 # ---------------------------------------------------------------------------
210
211 @pytest.mark.asyncio
212 async def test_s1e2e4_daily_quota_exhausted_returns_429(
213 client: AsyncClient, repo: MusehubRepo, db_session: AsyncSession
214 ) -> None:
215 """When today_bytes >= daily_limit, server returns 429 before presigning."""
216 settings = get_settings()
217 if settings.mpack_daily_upload_limit_bytes <= 0:
218 pytest.skip("daily quota disabled in this environment")
219
220 today = datetime.date.today()
221 logger.info(
222 "[step1] SETUP: seeding daily_push_bytes row at limit (%d bytes)",
223 settings.mpack_daily_upload_limit_bytes,
224 )
225 from sqlalchemy.dialects.postgresql import insert as pg_insert
226 await db_session.execute(
227 pg_insert(MusehubDailyPushBytes).values(
228 identity_id=_IDENTITY_ID,
229 date=today,
230 bytes_uploaded=settings.mpack_daily_upload_limit_bytes,
231 updated_at=datetime.datetime.now(datetime.timezone.utc),
232 ).on_conflict_do_update(
233 index_elements=["identity_id", "date"],
234 set_={"bytes_uploaded": settings.mpack_daily_upload_limit_bytes},
235 )
236 )
237 await db_session.commit()
238
239 mpack_bytes = b"small mpack"
240 body = _presign_body(mpack_bytes)
241
242 logger.info("[step1] CLIENT: POST presign after quota exhausted")
243 resp = await client.post(
244 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
245 content=body,
246 headers={"Content-Type": "application/x-msgpack"},
247 )
248 logger.info("[step1] SERVER: responded HTTP %d (expected 429)", resp.status_code)
249 assert resp.status_code == 429, f"expected 429, got {resp.status_code}"
250
251
252 # ---------------------------------------------------------------------------
253 # S1E2E-5 — Quota row written after successful presign
254 # ---------------------------------------------------------------------------
255
256 @pytest.mark.asyncio
257 async def test_s1e2e5_quota_row_written_after_presign(
258 client: AsyncClient, repo: MusehubRepo, db_session: AsyncSession
259 ) -> None:
260 """After a successful presign, musehub_daily_push_bytes has a row for today."""
261 settings = get_settings()
262 if settings.mpack_daily_upload_limit_bytes <= 0:
263 pytest.skip("daily quota disabled in this environment")
264
265 mpack_bytes = b"x" * 2048
266 body = _presign_body(mpack_bytes)
267
268 logger.info("[step1] CLIENT: POST presign (quota tracking enabled)")
269 resp = await client.post(
270 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
271 content=body,
272 headers={"Content-Type": "application/x-msgpack"},
273 )
274 logger.info("[step1] SERVER: responded HTTP %d", resp.status_code)
275 assert resp.status_code == 200
276
277 logger.info("[step1] DB: querying musehub_daily_push_bytes for identity_id=%s", _IDENTITY_ID[:20])
278 today = datetime.date.today()
279 result = await db_session.execute(
280 select(func.coalesce(func.sum(MusehubDailyPushBytes.bytes_uploaded), 0)).where(
281 MusehubDailyPushBytes.identity_id == _IDENTITY_ID,
282 MusehubDailyPushBytes.date == today,
283 )
284 )
285 recorded = int(result.scalar())
286 logger.info("[step1] DB: bytes_uploaded=%d (expected=%d)", recorded, len(mpack_bytes))
287 assert recorded == len(mpack_bytes), (
288 f"quota row not written correctly: got {recorded}, expected {len(mpack_bytes)}"
289 )
290
291
292 # ---------------------------------------------------------------------------
293 # S1E2E-6 — Quota is cumulative across calls
294 # ---------------------------------------------------------------------------
295
296 @pytest.mark.asyncio
297 async def test_s1e2e6_quota_is_cumulative(
298 client: AsyncClient, repo: MusehubRepo, db_session: AsyncSession
299 ) -> None:
300 """Second presign call adds to the existing daily row — does not reset it."""
301 settings = get_settings()
302 if settings.mpack_daily_upload_limit_bytes <= 0:
303 pytest.skip("daily quota disabled in this environment")
304
305 mpack_a = b"a" * 1000
306 mpack_b = b"b" * 500
307
308 logger.info("[step1] CLIENT: first presign (%d bytes)", len(mpack_a))
309 resp_a = await client.post(
310 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
311 content=_presign_body(mpack_a),
312 headers={"Content-Type": "application/x-msgpack"},
313 )
314 logger.info("[step1] SERVER: first presign → HTTP %d", resp_a.status_code)
315 assert resp_a.status_code == 200
316
317 logger.info("[step1] CLIENT: second presign (%d bytes)", len(mpack_b))
318 resp_b = await client.post(
319 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
320 content=_presign_body(mpack_b),
321 headers={"Content-Type": "application/x-msgpack"},
322 )
323 logger.info("[step1] SERVER: second presign → HTTP %d", resp_b.status_code)
324 assert resp_b.status_code == 200
325
326 today = datetime.date.today()
327 result = await db_session.execute(
328 select(func.coalesce(func.sum(MusehubDailyPushBytes.bytes_uploaded), 0)).where(
329 MusehubDailyPushBytes.identity_id == _IDENTITY_ID,
330 MusehubDailyPushBytes.date == today,
331 )
332 )
333 recorded = int(result.scalar())
334 expected = len(mpack_a) + len(mpack_b)
335 logger.info("[step1] DB: cumulative bytes_uploaded=%d (expected=%d)", recorded, expected)
336 assert recorded == expected, f"cumulative quota wrong: got {recorded}, expected {expected}"
337
338
339 # ---------------------------------------------------------------------------
340 # S1E2E-7 — Unauthenticated request → 401/403
341 # ---------------------------------------------------------------------------
342
343 @pytest.mark.asyncio
344 async def test_s1e2e7_unauthenticated_returns_401_or_403(
345 db_session: AsyncSession, repo: MusehubRepo
346 ) -> None:
347 """Without auth override, missing credentials → 401 or 403."""
348 async def _override_db() -> None:
349 yield db_session
350
351 app.dependency_overrides[get_db] = _override_db
352 # do NOT override require_valid_token — real auth enforcement
353
354 mpack_bytes = b"should be rejected"
355 body = _presign_body(mpack_bytes)
356
357 logger.info("[step1] CLIENT: POST presign with NO auth header")
358 async with AsyncClient(
359 transport=ASGITransport(app=app),
360 base_url="https://localhost:1337",
361 ) as c:
362 resp = await c.post(
363 f"/{_OWNER}/{_REPO_NAME}/push/mpack-presign",
364 content=body,
365 headers={"Content-Type": "application/x-msgpack"},
366 )
367
368 logger.info("[step1] SERVER: responded HTTP %d (expected 401 or 403)", resp.status_code)
369 assert resp.status_code in (401, 403), (
370 f"expected 401 or 403 for unauthenticated request, got {resp.status_code}"
371 )
372 app.dependency_overrides.clear()
File History 1 commit
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ 20 days ago