test_enqueue_batch.py
python
sha256:34035d72cef530c1ab9d6a6f53be18d803dde705fc3157617d70352a96d0747b
Merge branch 'fix/wire-push-external-parent-manifest' into dev
Human
9 days ago
| 1 | """TDD — enqueue_push_intel must use O(1) DB queries, not O(N). |
| 2 | |
| 3 | Root cause of the phase-10 slowness observed 2026-05-10: |
| 4 | enqueue_job called once per job type, each call doing a sequential SELECT |
| 5 | (idempotency check) + INSERT. With 17 job types + profile.snapshot = 18 |
| 6 | round-trips × ~80ms each = ~1.4s added to every push finalization. |
| 7 | |
| 8 | Fix: batch the idempotency check into a single |
| 9 | SELECT job_type FROM musehub_background_jobs |
| 10 | WHERE repo_id=X AND job_type IN (...) AND status='pending' |
| 11 | then bulk INSERT the missing ones in one statement. |
| 12 | |
| 13 | Tests: |
| 14 | B1 structural: enqueue_push_intel source must not call enqueue_job in a loop |
| 15 | B2 idempotency: second call inserts 0 new rows |
| 16 | B3 wall-clock for enqueue_push_intel + enqueue_profile_snapshot < 50ms |
| 17 | B4 all job types from job_types_for_push actually land in the DB |
| 18 | """ |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import inspect |
| 22 | import time |
| 23 | |
| 24 | import pytest |
| 25 | from sqlalchemy import func, select |
| 26 | from sqlalchemy.ext.asyncio import AsyncSession |
| 27 | |
| 28 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 29 | from musehub.services.musehub_jobs import enqueue_profile_snapshot, enqueue_push_intel |
| 30 | from musehub.services.musehub_intel_providers import job_types_for_push |
| 31 | from tests.factories import create_repo |
| 32 | |
| 33 | |
| 34 | # --------------------------------------------------------------------------- |
| 35 | # B1 — structural: enqueue_push_intel must not call enqueue_job in a loop |
| 36 | # --------------------------------------------------------------------------- |
| 37 | |
| 38 | def test_B1_no_enqueue_job_loop() -> None: |
| 39 | """enqueue_push_intel source must not contain a for-loop that calls enqueue_job.""" |
| 40 | import musehub.services.musehub_jobs as _jobs |
| 41 | |
| 42 | src = inspect.getsource(_jobs.enqueue_push_intel) |
| 43 | lines = src.splitlines() |
| 44 | |
| 45 | for_indent: int | None = None |
| 46 | for line in lines: |
| 47 | stripped = line.strip() |
| 48 | if not stripped or stripped.startswith("#"): |
| 49 | continue |
| 50 | indent = len(line) - len(line.lstrip()) |
| 51 | if stripped.startswith("for ") and "job_type" in stripped: |
| 52 | for_indent = indent |
| 53 | continue |
| 54 | if for_indent is not None: |
| 55 | if indent <= for_indent: |
| 56 | for_indent = None |
| 57 | elif "enqueue_job" in stripped and "await" in stripped: |
| 58 | raise AssertionError( |
| 59 | "enqueue_push_intel still calls enqueue_job inside a for-loop — " |
| 60 | "O(N) DB round-trips. Replace with a single batch " |
| 61 | "SELECT … IN + bulk INSERT … ON CONFLICT DO NOTHING." |
| 62 | ) |
| 63 | |
| 64 | |
| 65 | # --------------------------------------------------------------------------- |
| 66 | # B2 — idempotency: second call inserts 0 rows |
| 67 | # --------------------------------------------------------------------------- |
| 68 | |
| 69 | @pytest.mark.asyncio |
| 70 | async def test_B2_idempotent(db_session: AsyncSession) -> None: |
| 71 | """Second call for the same repo must insert 0 new rows.""" |
| 72 | repo = await create_repo(db_session, owner="gabriel") |
| 73 | repo_id = str(repo.repo_id) |
| 74 | |
| 75 | await enqueue_push_intel(db_session, repo_id, head="sha256:abc", domain_id=None) |
| 76 | await db_session.commit() |
| 77 | |
| 78 | count_before = (await db_session.execute( |
| 79 | select(func.count()).select_from(MusehubBackgroundJob).where( |
| 80 | MusehubBackgroundJob.repo_id == repo_id, |
| 81 | MusehubBackgroundJob.status == "pending", |
| 82 | ) |
| 83 | )).scalar() |
| 84 | |
| 85 | await enqueue_push_intel(db_session, repo_id, head="sha256:def", domain_id=None) |
| 86 | await db_session.commit() |
| 87 | |
| 88 | count_after = (await db_session.execute( |
| 89 | select(func.count()).select_from(MusehubBackgroundJob).where( |
| 90 | MusehubBackgroundJob.repo_id == repo_id, |
| 91 | MusehubBackgroundJob.status == "pending", |
| 92 | ) |
| 93 | )).scalar() |
| 94 | |
| 95 | assert count_after == count_before, ( |
| 96 | f"Second call inserted {count_after - count_before} extra rows — " |
| 97 | "idempotency broken." |
| 98 | ) |
| 99 | |
| 100 | |
| 101 | # --------------------------------------------------------------------------- |
| 102 | # B3 — wall-clock < 50ms with real DB |
| 103 | # --------------------------------------------------------------------------- |
| 104 | |
| 105 | @pytest.mark.asyncio |
| 106 | async def test_B3_fast(db_session: AsyncSession) -> None: |
| 107 | """enqueue_push_intel + enqueue_profile_snapshot must complete in <50ms.""" |
| 108 | repo = await create_repo(db_session, owner="gabriel") |
| 109 | repo_id = str(repo.repo_id) |
| 110 | |
| 111 | t0 = time.monotonic() |
| 112 | await enqueue_push_intel(db_session, repo_id, head="sha256:abc", domain_id=None) |
| 113 | await enqueue_profile_snapshot(db_session, repo_id, handle="gabriel") |
| 114 | elapsed = time.monotonic() - t0 |
| 115 | |
| 116 | assert elapsed < 0.05, ( |
| 117 | f"enqueue_push_intel + enqueue_profile_snapshot took {elapsed*1000:.1f}ms " |
| 118 | f"— expected <50ms. Sequential per-job DB round-trips are the cause." |
| 119 | ) |
| 120 | |
| 121 | |
| 122 | # --------------------------------------------------------------------------- |
| 123 | # B4 — all job types actually get inserted |
| 124 | # --------------------------------------------------------------------------- |
| 125 | |
| 126 | @pytest.mark.asyncio |
| 127 | async def test_B4_all_job_types_inserted(db_session: AsyncSession) -> None: |
| 128 | """Every type from job_types_for_push must appear in the DB after enqueueing.""" |
| 129 | repo = await create_repo(db_session, owner="gabriel") |
| 130 | repo_id = str(repo.repo_id) |
| 131 | expected = set(job_types_for_push(None)) |
| 132 | |
| 133 | await enqueue_push_intel(db_session, repo_id, head="sha256:abc", domain_id=None) |
| 134 | await db_session.commit() |
| 135 | |
| 136 | rows = (await db_session.execute( |
| 137 | select(MusehubBackgroundJob.job_type).where( |
| 138 | MusehubBackgroundJob.repo_id == repo_id, |
| 139 | MusehubBackgroundJob.status == "pending", |
| 140 | ) |
| 141 | )).scalars().all() |
| 142 | actual = set(rows) |
| 143 | |
| 144 | missing = expected - actual |
| 145 | assert not missing, f"These job types were not inserted: {missing}" |
File History
14 commits
sha256:34035d72cef530c1ab9d6a6f53be18d803dde705fc3157617d70352a96d0747b
Merge branch 'fix/wire-push-external-parent-manifest' into dev
Human
9 days ago
sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226
Merge branch 'fix/two-column-scroll-layout' into dev
Human
9 days ago
sha256:408916fc5973ba59c6e4eebaa80ebdcc801c0a63205651e25009d11548f79454
chore: bump version to 0.2.0.dev2 — nightly.2, matching muse
Sonnet 4.6
patch
12 days ago
sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2
chore: bump version to 0.2.0rc15 for musehub#113 fix release
Sonnet 4.6
patch
15 days ago
sha256:0032d6cfa33bc3c8367436ad768e7dd0e339b4332153160247da8266cb5fa352
Merge branch 'task/version-tags-phase3-server' into dev
Human
17 days ago
sha256:4669620efda9ff41c55bdefd1f7bfe1c239d468428744c84ead9957e5a003a53
merge: rescue snapshot-recovery hardening (c00aa21d) into d…
Opus 4.8
minor
⚠
30 days ago
sha256:a59da49c4611b970fc4b6ae48678ce4943261c213a07ddbd73ce9201df869b4a
fix: remove false-positive proposal_comments index drop fro…
Sonnet 4.6
patch
34 days ago
sha256:0a240d6dbff234f07d98a28a4a9a68db702f3f9ff9260196f24219bdb1c0b6f3
feat: render markdown mists as HTML with heading anchor links
Sonnet 4.6
patch
35 days ago
sha256:24a7d47486ebc4ebd1832830580e177ec6f877b48dced8c000e198cdec4ce9d6
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
36 days ago
sha256:b9ff931d147e0114a1f17060f415b89ed551c170a91ff226c70437aa5c85f9ee
Merge 'task/bump-version-rc12' into 'dev' — proposal: Bump …
Human
36 days ago
sha256:d1122d21e73471879b460037b22c0b50fded7c423444a176f248428f75dac39c
Merge 'task/fix-issue-pagination-cursor' into 'dev' — propo…
Human
36 days ago
sha256:01e18975e73d2b3cd5b6db7929c895bef9aa6e0d4391dc5b2adfc548b41318dd
Merge 'feat/adding-debug-logs-to-staging' into 'dev' — prop…
Human
36 days ago
sha256:6b1949fc2797ca4c1936a637a4cbfec828ef56cf52398a2e74ca3c4f494e728f
fix: use wire_bytes not mpack_bytes_raw in compute_object_b…
Sonnet 4.6
patch
48 days ago
sha256:b99f2455dc346966d040133f5203297e6e3ef5803a93728a2c30568d0a0f7583
rename: delta_add → delta_upsert across wire format, models…
Sonnet 4.6
patch
50 days ago