gabriel / musehub public
test_mpack_rate_limiting_phase4.py python
406 lines 14.5 KB
Raw
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ breaking 20 days ago
1 """TDD — Phase 4: per-user daily byte limits, anomaly detection, /api/caps (issue #49).
2
3 Phase 4 invariants:
4 4a. Per-user daily byte limit — tracked in musehub_daily_push_bytes; presign
5 endpoint rejects with 429 when the caller's daily total would exceed
6 settings.mpack_daily_upload_limit_bytes.
7 4b. Anomaly detection — after a successful mpack.index, the per-user 30-day
8 rolling average is computed; if today's upload is >10× the average a
9 musehub_push_anomaly row is inserted and a structured WARNING is logged.
10 The push is NOT rejected.
11 4c. GET /api/caps — public endpoint returning server limits as JSON.
12 """
13 from __future__ import annotations
14
15 import datetime
16 from collections.abc import AsyncGenerator
17 from typing import TypedDict
18
19 import pytest
20 import pytest_asyncio
21 from httpx import AsyncClient, ASGITransport
22 from sqlalchemy import select
23 from sqlalchemy.ext.asyncio import AsyncSession
24
25 from musehub.auth.request_signing import MSignContext, require_signed_request, optional_signed_request
26 from musehub.db.musehub_abuse_models import MusehubDailyPushBytes, MusehubPushAnomaly
27 from musehub.db.database import get_db
28 from musehub.main import app
29
30
31 class _RepoData(TypedDict):
32 owner: str
33 name: str
34 repoId: str
35
36
37 _AUTH_CTX = MSignContext(
38 handle="gabriel",
39 identity_id="sha256:" + "a" * 64,
40 is_agent=False,
41 is_admin=False,
42 )
43
44 _AUTH_CTX2 = MSignContext(
45 handle="carol",
46 identity_id="sha256:" + "b" * 64,
47 is_agent=False,
48 is_admin=False,
49 )
50
51
52 # ── fixtures ─────────────────────────────────────────────────────────────────
53
54 @pytest_asyncio.fixture()
55 async def client(db_session: AsyncSession) -> None:
56 async def _override_get_db() -> None:
57 yield db_session
58
59 app.dependency_overrides[get_db] = _override_get_db
60 app.dependency_overrides[require_signed_request] = lambda: _AUTH_CTX
61 app.dependency_overrides[optional_signed_request] = lambda: _AUTH_CTX
62
63 async with AsyncClient(
64 transport=ASGITransport(app=app),
65 base_url="https://localhost:1337",
66 ) as c:
67 yield c
68
69 app.dependency_overrides.clear()
70
71
72 @pytest_asyncio.fixture()
73 async def client2(db_session: AsyncSession) -> None:
74 """Second user client — verifies limits are per-user, not global."""
75 async def _override_get_db() -> None:
76 yield db_session
77
78 app.dependency_overrides[get_db] = _override_get_db
79 app.dependency_overrides[require_signed_request] = lambda: _AUTH_CTX2
80 app.dependency_overrides[optional_signed_request] = lambda: _AUTH_CTX2
81
82 async with AsyncClient(
83 transport=ASGITransport(app=app),
84 base_url="https://localhost:1337",
85 ) as c:
86 yield c
87
88 app.dependency_overrides.clear()
89
90
91 @pytest_asyncio.fixture()
92 async def repo(client: AsyncClient) -> AsyncGenerator[_RepoData, None]:
93 resp = await client.post(
94 "/api/repos",
95 json={"owner": "gabriel", "name": "phase4-limits-test", "visibility": "public", "initialize": False},
96 )
97 assert resp.status_code in (200, 201), resp.text
98 data = resp.json()
99 yield data
100 await client.delete(f"/api/repos/{data['repoId']}")
101
102
103 # ── helper ────────────────────────────────────────────────────────────────────
104
105 def _today() -> datetime.date:
106 return datetime.date.today()
107
108
109 # ══════════════════════════════════════════════════════════════════════════════
110 # 4a — per-user daily byte limit
111 # ══════════════════════════════════════════════════════════════════════════════
112
113 @pytest.mark.asyncio
114 async def test_daily_push_bytes_table_exists(db_session: AsyncSession) -> None:
115 """musehub_daily_push_bytes table is present and queryable."""
116 rows = (
117 await db_session.execute(select(MusehubDailyPushBytes))
118 ).scalars().all()
119 assert isinstance(rows, list)
120
121
122 @pytest.mark.asyncio
123 async def test_record_mpack_bytes_increments_daily_total(db_session: AsyncSession) -> None:
124 """record_mpack_bytes_uploaded upserts the per-user daily row."""
125 from musehub.services.musehub_wire import record_mpack_bytes_uploaded
126
127 identity_id = _AUTH_CTX.identity_id
128 today = _today()
129
130 await record_mpack_bytes_uploaded(db_session, identity_id, 1024)
131 await db_session.commit()
132
133 row = (
134 await db_session.execute(
135 select(MusehubDailyPushBytes).where(
136 MusehubDailyPushBytes.identity_id == identity_id,
137 MusehubDailyPushBytes.date == today,
138 )
139 )
140 ).scalar_one_or_none()
141
142 assert row is not None
143 assert row.bytes_uploaded == 1024
144
145
146 @pytest.mark.asyncio
147 async def test_record_mpack_bytes_accumulates_across_calls(db_session: AsyncSession) -> None:
148 """Two calls on the same day accumulate, not overwrite."""
149 from musehub.services.musehub_wire import record_mpack_bytes_uploaded
150
151 identity_id = _AUTH_CTX.identity_id
152
153 await record_mpack_bytes_uploaded(db_session, identity_id, 500)
154 await db_session.commit()
155 await record_mpack_bytes_uploaded(db_session, identity_id, 300)
156 await db_session.commit()
157
158 row = (
159 await db_session.execute(
160 select(MusehubDailyPushBytes).where(
161 MusehubDailyPushBytes.identity_id == identity_id,
162 MusehubDailyPushBytes.date == _today(),
163 )
164 )
165 ).scalar_one_or_none()
166 assert row is not None
167 assert row.bytes_uploaded == 800
168
169
170 @pytest.mark.skip(reason="muse wire protocol in flux")
171 @pytest.mark.asyncio
172 async def test_daily_limit_allows_push_under_quota(
173 client: AsyncClient,
174 repo: _RepoData,
175 ) -> None:
176 """mpack-presign succeeds when user is under the daily limit."""
177 import msgpack
178 body = msgpack.packb({"mpack_key": "sha256:" + "c" * 64, "size_bytes": 1024})
179 resp = await client.post(
180 f"/{repo['owner']}/{repo['name']}/push/mpack-presign",
181 content=body,
182 headers={"Content-Type": "application/x-msgpack"},
183 )
184 # 200 or 422 (no backend presign in test) — definitely NOT 429
185 assert resp.status_code != 429
186
187
188 @pytest.mark.asyncio
189 async def test_daily_limit_blocks_push_over_quota(
190 client: AsyncClient,
191 repo: _RepoData,
192 db_session: AsyncSession,
193 ) -> None:
194 """mpack-presign returns 429 when daily limit already exceeded."""
195 from musehub.config import get_settings
196 from musehub.services.musehub_wire import record_mpack_bytes_uploaded
197
198 limit = get_settings().mpack_daily_upload_limit_bytes
199
200 # Pre-fill the user's daily counter just over the limit
201 await record_mpack_bytes_uploaded(db_session, _AUTH_CTX.identity_id, limit + 1)
202 await db_session.commit()
203
204 import msgpack
205 body = msgpack.packb({"mpack_key": "sha256:" + "d" * 64, "size_bytes": 1024})
206 resp = await client.post(
207 f"/{repo['owner']}/{repo['name']}/push/mpack-presign",
208 content=body,
209 headers={"Content-Type": "application/x-msgpack"},
210 )
211 assert resp.status_code == 429
212 assert "daily" in resp.text.lower()
213
214
215 @pytest.mark.skip(reason="muse wire protocol in flux")
216 @pytest.mark.asyncio
217 async def test_daily_limit_is_per_user_not_global(
218 client: AsyncClient,
219 repo: _RepoData,
220 db_session: AsyncSession,
221 ) -> None:
222 """User carol's quota being exhausted does NOT block gabriel."""
223 from musehub.config import get_settings
224 from musehub.services.musehub_wire import record_mpack_bytes_uploaded
225
226 limit = get_settings().mpack_daily_upload_limit_bytes
227
228 # Exhaust carol's quota
229 await record_mpack_bytes_uploaded(db_session, _AUTH_CTX2.identity_id, limit + 1)
230 await db_session.commit()
231
232 # Gabriel's presign should still work (not 429)
233 import msgpack
234 body = msgpack.packb({"mpack_key": "sha256:" + "e" * 64, "size_bytes": 512})
235 resp = await client.post(
236 f"/{repo['owner']}/{repo['name']}/push/mpack-presign",
237 content=body,
238 headers={"Content-Type": "application/x-msgpack"},
239 )
240 assert resp.status_code != 429
241
242
243 # ══════════════════════════════════════════════════════════════════════════════
244 # 4b — anomaly detection
245 # ══════════════════════════════════════════════════════════════════════════════
246
247 @pytest.mark.asyncio
248 async def test_push_anomaly_table_exists(db_session: AsyncSession) -> None:
249 """musehub_push_anomaly table is present and queryable."""
250 rows = (
251 await db_session.execute(select(MusehubPushAnomaly))
252 ).scalars().all()
253 assert isinstance(rows, list)
254
255
256 @pytest.mark.asyncio
257 async def test_check_push_anomaly_does_not_flag_normal_volume(db_session: AsyncSession) -> None:
258 """No anomaly row created when today's upload ≤ 10× 30-day average."""
259 from musehub.services.musehub_wire import check_push_anomaly
260
261 identity_id = _AUTH_CTX.identity_id
262 today = _today()
263
264 # Seed 7 days of 1 MB pushes for a 1 MB average
265 for i in range(7):
266 day = today - datetime.timedelta(days=i + 1)
267 db_session.add(MusehubDailyPushBytes(
268 identity_id=identity_id,
269 date=day,
270 bytes_uploaded=1024 * 1024,
271 ))
272 await db_session.commit()
273
274 # Today's upload is 5 MB — 5× the average, under the 10× threshold
275 flagged = await check_push_anomaly(db_session, identity_id, 5 * 1024 * 1024)
276 assert flagged is False
277
278 rows = (
279 await db_session.execute(
280 select(MusehubPushAnomaly).where(
281 MusehubPushAnomaly.identity_id == identity_id
282 )
283 )
284 ).scalars().all()
285 assert len(rows) == 0
286
287
288 @pytest.mark.asyncio
289 async def test_check_push_anomaly_flags_spike(db_session: AsyncSession) -> None:
290 """Anomaly row inserted when today's upload is >10× 30-day average."""
291 from musehub.services.musehub_wire import check_push_anomaly
292
293 identity_id = _AUTH_CTX.identity_id
294 today = _today()
295
296 # 30-day average of 1 MB/day
297 for i in range(30):
298 day = today - datetime.timedelta(days=i + 1)
299 db_session.add(MusehubDailyPushBytes(
300 identity_id=identity_id,
301 date=day,
302 bytes_uploaded=1024 * 1024,
303 ))
304 await db_session.commit()
305
306 # Today's upload is 11 MB — 11× the average → should flag
307 flagged = await check_push_anomaly(db_session, identity_id, 11 * 1024 * 1024)
308 assert flagged is True
309 await db_session.commit()
310
311 row = (
312 await db_session.execute(
313 select(MusehubPushAnomaly).where(
314 MusehubPushAnomaly.identity_id == identity_id
315 )
316 )
317 ).scalar_one_or_none()
318 assert row is not None
319 assert row.bytes_today >= 11 * 1024 * 1024
320 assert row.rolling_avg_bytes > 0
321 assert row.ratio >= 10.0
322
323
324 @pytest.mark.asyncio
325 async def test_check_push_anomaly_no_history_does_not_flag(db_session: AsyncSession) -> None:
326 """First-ever push for a user (no history) is never flagged as anomalous."""
327 from musehub.services.musehub_wire import check_push_anomaly
328
329 identity_id = "sha256:" + "f" * 64
330 flagged = await check_push_anomaly(db_session, identity_id, 100 * 1024 * 1024)
331 assert flagged is False
332
333
334 @pytest.mark.asyncio
335 async def test_anomaly_does_not_block_push(db_session: AsyncSession) -> None:
336 """check_push_anomaly returns True but raises no exception."""
337 from musehub.services.musehub_wire import check_push_anomaly
338
339 identity_id = _AUTH_CTX.identity_id
340 today = _today()
341
342 for i in range(10):
343 day = today - datetime.timedelta(days=i + 1)
344 db_session.add(MusehubDailyPushBytes(
345 identity_id=identity_id,
346 date=day,
347 bytes_uploaded=100,
348 ))
349 await db_session.commit()
350
351 # 100 MB vs 100 bytes average — far above 10× threshold
352 # Should return True but NOT raise
353 try:
354 result = await check_push_anomaly(db_session, identity_id, 100 * 1024 * 1024)
355 assert result is True
356 except Exception as exc:
357 pytest.fail(f"check_push_anomaly raised unexpectedly: {exc}")
358
359
360 # ══════════════════════════════════════════════════════════════════════════════
361 # 4c — GET /api/caps
362 # ══════════════════════════════════════════════════════════════════════════════
363
364 @pytest.mark.asyncio
365 async def test_caps_endpoint_exists(client: AsyncClient) -> None:
366 """GET /api/caps returns 200."""
367 resp = await client.get("/api/caps")
368 assert resp.status_code == 200
369
370
371 @pytest.mark.asyncio
372 async def test_caps_returns_required_fields(client: AsyncClient) -> None:
373 """GET /api/caps body contains all required server limit fields."""
374 resp = await client.get("/api/caps")
375 assert resp.status_code == 200
376 data = resp.json()
377 assert "max_mpack_bytes" in data
378 assert "daily_upload_limit_bytes" in data
379 assert "max_commits_per_push" in data
380 assert "max_objects_per_push" in data
381
382
383 @pytest.mark.asyncio
384 async def test_caps_values_match_settings(client: AsyncClient) -> None:
385 """GET /api/caps values match the active settings object."""
386 from musehub.config import get_settings
387 s = get_settings()
388
389 resp = await client.get("/api/caps")
390 assert resp.status_code == 200
391 data = resp.json()
392 assert data["max_mpack_bytes"] == s.mpack_max_bytes
393 assert data["daily_upload_limit_bytes"] == s.mpack_daily_upload_limit_bytes
394 assert data["max_commits_per_push"] == s.mpack_max_commits
395 assert data["max_objects_per_push"] == s.mpack_max_objects
396
397
398 @pytest.mark.asyncio
399 async def test_caps_is_public_no_auth_required() -> None:
400 """GET /api/caps works without any auth header."""
401 async with AsyncClient(
402 transport=ASGITransport(app=app),
403 base_url="https://localhost:1337",
404 ) as c:
405 resp = await c.get("/api/caps")
406 assert resp.status_code == 200
File History 1 commit
sha256:ef10830ce231e0a20efcb0e2586cb879471247e916616e6fdd0d51df459e2595 fix: typing audit — 0 violations, 0 untyped defs across all… Sonnet 4.6 minor ⚠ 20 days ago