test_mpack_index_phase2.py
file-level
1
files
1
commits
0
hotspots
0
🧊 dead
0
💥 blast risk
| 1 | """TDD — Phase 2: MPack index covers objects; commits land in commit graph. |
| 2 | |
| 3 | PI-5 After process_mpack_index_job, every commit_id in the mpack has a row |
| 4 | in MusehubCommitGraph (the commit index, not MusehubMPackIndex). |
| 5 | PI-6 Objects in a mpack that also contains snapshot dicts are indexed |
| 6 | correctly in MusehubMPackIndex (snapshots are tracked by the sync push |
| 7 | path and are not indexed by the background job). |
| 8 | PI-7 One process_mpack_index_job call indexes objects in MusehubMPackIndex |
| 9 | AND commits in MusehubCommitGraph — both in one pass. |
| 10 | PI-8 A second push with different objects and commits adds new rows to both |
| 11 | tables without disturbing the first push's rows (idempotency). |
| 12 | """ |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import datetime |
| 16 | import hashlib |
| 17 | from collections.abc import Mapping |
| 18 | |
| 19 | import msgpack |
| 20 | import pytest |
| 21 | from sqlalchemy import select |
| 22 | |
| 23 | from sqlalchemy.ext.asyncio import AsyncSession |
| 24 | |
| 25 | from muse.core.snapshot import compute_commit_id, compute_snapshot_id |
| 26 | from muse.core.types import blob_id |
| 27 | from musehub.core.genesis import compute_identity_id |
| 28 | from musehub.db import musehub_repo_models as db |
| 29 | from musehub.services.musehub_repository import create_repo |
| 30 | |
| 31 | |
| 32 | # --------------------------------------------------------------------------- |
| 33 | # Helpers |
| 34 | # --------------------------------------------------------------------------- |
| 35 | |
| 36 | _DT = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) |
| 37 | |
| 38 | |
| 39 | def _make_commit_dict( |
| 40 | repo_id: str, |
| 41 | message: str, |
| 42 | snapshot_id: str, |
| 43 | parent_ids: list[str] | None = None, |
| 44 | ) -> tuple[dict, str]: |
| 45 | """Build a minimal commit dict and return (dict, commit_id).""" |
| 46 | cid = compute_commit_id( |
| 47 | parent_ids=parent_ids or [], |
| 48 | snapshot_id=snapshot_id, |
| 49 | message=message, |
| 50 | committed_at_iso=_DT.isoformat(), |
| 51 | author="gabriel", |
| 52 | ) |
| 53 | return { |
| 54 | "commit_id": cid, |
| 55 | "repo_id": repo_id, |
| 56 | "branch": "main", |
| 57 | "snapshot_id": snapshot_id, |
| 58 | "parent_commit_id": None, |
| 59 | "parent2_commit_id": None, |
| 60 | "message": message, |
| 61 | "committed_at": _DT.isoformat(), |
| 62 | "author": "gabriel", |
| 63 | "metadata": {}, |
| 64 | "structured_delta": None, |
| 65 | "sem_ver_bump": "none", |
| 66 | "breaking_changes": [], |
| 67 | "agent_id": "", |
| 68 | "model_id": "", |
| 69 | "toolchain_id": "", |
| 70 | "prompt_hash": "", |
| 71 | "signature": "", |
| 72 | "signer_key_id": "", |
| 73 | }, cid |
| 74 | |
| 75 | |
| 76 | def _make_snapshot_dict(manifest: Mapping[str, str]) -> tuple[dict, str]: |
| 77 | """Build a minimal snapshot dict and return (dict, snapshot_id).""" |
| 78 | sid = compute_snapshot_id(manifest) |
| 79 | return { |
| 80 | "snapshot_id": sid, |
| 81 | "parent_snapshot_id": None, |
| 82 | "delta_upsert": manifest, |
| 83 | "delta_remove": [], |
| 84 | }, sid |
| 85 | |
| 86 | |
| 87 | def _make_full_mpack( |
| 88 | repo_id: str, |
| 89 | objects: dict[str, bytes], |
| 90 | extra_tag: str = "", |
| 91 | ) -> tuple[bytes, str, list[str], list[str]]: |
| 92 | """Build an mpack with objects, one snapshot, and one commit. |
| 93 | |
| 94 | Returns (wire_bytes, mpack_key, [commit_id], [snapshot_id]). |
| 95 | """ |
| 96 | manifest = {f"src/file_{extra_tag}_{k[-8:]}.py": k for k in objects} |
| 97 | snap_dict, sid = _make_snapshot_dict(manifest) |
| 98 | commit_dict, cid = _make_commit_dict(repo_id, f"commit {extra_tag}", sid) |
| 99 | |
| 100 | mpack = { |
| 101 | "commits": [commit_dict], |
| 102 | "snapshots": [snap_dict], |
| 103 | "objects": [ |
| 104 | {"object_id": oid, "content": content} |
| 105 | for oid, content in objects.items() |
| 106 | ], |
| 107 | } |
| 108 | wire_bytes = msgpack.packb(mpack, use_bin_type=True) |
| 109 | mpack_key = "sha256:" + hashlib.sha256(wire_bytes).hexdigest() |
| 110 | return wire_bytes, mpack_key, [cid], [sid] |
| 111 | |
| 112 | |
| 113 | async def _store_mpack(wire_bytes: bytes, mpack_key: str) -> None: |
| 114 | import musehub.storage.backends as _backends_mod |
| 115 | backend = _backends_mod.get_backend() |
| 116 | await backend.put_mpack(mpack_key, wire_bytes) |
| 117 | |
| 118 | |
| 119 | async def _enqueue_and_process( |
| 120 | session: AsyncSession, |
| 121 | repo_id: str, |
| 122 | mpack_key: str, |
| 123 | n_objects: int, |
| 124 | n_commits: int = 1, |
| 125 | ) -> None: |
| 126 | from musehub.core.genesis import compute_job_id |
| 127 | from musehub.db.musehub_jobs_models import MusehubBackgroundJob |
| 128 | from musehub.services.musehub_wire import process_mpack_index_job |
| 129 | |
| 130 | now = datetime.datetime.now(datetime.timezone.utc) |
| 131 | job_id = compute_job_id(repo_id, "mpack.index", now.isoformat()) |
| 132 | session.add(MusehubBackgroundJob( |
| 133 | job_id=job_id, |
| 134 | repo_id=repo_id, |
| 135 | job_type="mpack.index", |
| 136 | payload={ |
| 137 | "mpack_key": mpack_key, |
| 138 | "branch": "main", |
| 139 | "head": "", |
| 140 | "pusher_id": "", |
| 141 | "declared_objects_count": n_objects, |
| 142 | "declared_commits_count": n_commits, |
| 143 | }, |
| 144 | status="pending", |
| 145 | created_at=now, |
| 146 | attempt=0, |
| 147 | )) |
| 148 | await session.commit() |
| 149 | await process_mpack_index_job(session, job_id) |
| 150 | await session.commit() |
| 151 | |
| 152 | |
| 153 | # --------------------------------------------------------------------------- |
| 154 | # PI-5 commit_ids land in MusehubCommitGraph |
| 155 | # --------------------------------------------------------------------------- |
| 156 | |
| 157 | @pytest.mark.asyncio |
| 158 | async def test_pi5_commit_ids_indexed_after_push(db_session: AsyncSession) -> None: |
| 159 | """Every commit_id in the mpack must appear in MusehubCommitGraph after the job runs. |
| 160 | |
| 161 | Commits are indexed in MusehubCommitGraph (not MusehubMPackIndex), enabling |
| 162 | O(1) BFS-frontier delta walks for the fetch path. |
| 163 | """ |
| 164 | repo = await create_repo( |
| 165 | db_session, |
| 166 | name="pi5-test", |
| 167 | owner="gabriel", |
| 168 | owner_user_id=compute_identity_id(b"gabriel"), |
| 169 | visibility="public", |
| 170 | initialize=False, |
| 171 | ) |
| 172 | objects = {blob_id(b"pi5-obj"): b"pi5-obj"} |
| 173 | wire_bytes, mpack_key, commit_ids, _ = _make_full_mpack( |
| 174 | repo.repo_id, objects, extra_tag="pi5" |
| 175 | ) |
| 176 | await _store_mpack(wire_bytes, mpack_key) |
| 177 | await _enqueue_and_process(db_session, repo.repo_id, mpack_key, len(objects)) |
| 178 | |
| 179 | rows = (await db_session.execute( |
| 180 | select(db.MusehubCommitGraph).where( |
| 181 | db.MusehubCommitGraph.commit_id.in_(commit_ids) |
| 182 | ) |
| 183 | )).scalars().all() |
| 184 | |
| 185 | assert {r.commit_id for r in rows} == set(commit_ids), ( |
| 186 | f"commit_ids not in MusehubCommitGraph — missing: " |
| 187 | f"{set(commit_ids) - {r.commit_id for r in rows}}" |
| 188 | ) |
| 189 | |
| 190 | |
| 191 | # --------------------------------------------------------------------------- |
| 192 | # PI-6 objects indexed correctly when mpack also contains snapshots |
| 193 | # --------------------------------------------------------------------------- |
| 194 | |
| 195 | @pytest.mark.asyncio |
| 196 | async def test_pi6_objects_indexed_when_mpack_has_snapshots(db_session: AsyncSession) -> None: |
| 197 | """Objects in a mpack that also contains snapshot dicts are still indexed correctly. |
| 198 | |
| 199 | Snapshots are tracked by the sync push path (wire_push_unpack_mpack) and |
| 200 | are not separately indexed by process_mpack_index_job. This test confirms |
| 201 | the job correctly indexes the object blobs even when snapshot dicts are |
| 202 | present in the mpack payload. |
| 203 | """ |
| 204 | repo = await create_repo( |
| 205 | db_session, |
| 206 | name="pi6-test", |
| 207 | owner="gabriel", |
| 208 | owner_user_id=compute_identity_id(b"gabriel"), |
| 209 | visibility="public", |
| 210 | initialize=False, |
| 211 | ) |
| 212 | objects = {blob_id(b"pi6-obj"): b"pi6-obj"} |
| 213 | wire_bytes, mpack_key, _, _ = _make_full_mpack( |
| 214 | repo.repo_id, objects, extra_tag="pi6" |
| 215 | ) |
| 216 | await _store_mpack(wire_bytes, mpack_key) |
| 217 | await _enqueue_and_process(db_session, repo.repo_id, mpack_key, len(objects)) |
| 218 | |
| 219 | # Objects must be in MusehubMPackIndex. |
| 220 | obj_rows = (await db_session.execute( |
| 221 | select(db.MusehubMPackIndex).where( |
| 222 | db.MusehubMPackIndex.entity_id.in_(list(objects.keys())), |
| 223 | db.MusehubMPackIndex.entity_type == "object", |
| 224 | ) |
| 225 | )).scalars().all() |
| 226 | |
| 227 | assert {r.entity_id for r in obj_rows} == set(objects.keys()), ( |
| 228 | f"object_ids not indexed — missing: " |
| 229 | f"{set(objects.keys()) - {r.entity_id for r in obj_rows}}" |
| 230 | ) |
| 231 | assert all(r.mpack_id == mpack_key for r in obj_rows), "object rows have wrong mpack_id" |
| 232 | |
| 233 | |
| 234 | # --------------------------------------------------------------------------- |
| 235 | # PI-7 Objects and commits indexed in a single job call |
| 236 | # --------------------------------------------------------------------------- |
| 237 | |
| 238 | @pytest.mark.asyncio |
| 239 | async def test_pi7_objects_and_commits_indexed_atomically(db_session: AsyncSession) -> None: |
| 240 | """One process_mpack_index_job call indexes objects in MusehubMPackIndex |
| 241 | and commits in MusehubCommitGraph — both in one pass.""" |
| 242 | repo = await create_repo( |
| 243 | db_session, |
| 244 | name="pi7-test", |
| 245 | owner="gabriel", |
| 246 | owner_user_id=compute_identity_id(b"gabriel"), |
| 247 | visibility="public", |
| 248 | initialize=False, |
| 249 | ) |
| 250 | objects = { |
| 251 | blob_id(f"pi7-obj-{i}".encode()): f"pi7-obj-{i}".encode() |
| 252 | for i in range(3) |
| 253 | } |
| 254 | wire_bytes, mpack_key, commit_ids, _ = _make_full_mpack( |
| 255 | repo.repo_id, objects, extra_tag="pi7" |
| 256 | ) |
| 257 | await _store_mpack(wire_bytes, mpack_key) |
| 258 | await _enqueue_and_process(db_session, repo.repo_id, mpack_key, len(objects)) |
| 259 | |
| 260 | # Objects in MusehubMPackIndex. |
| 261 | obj_rows = (await db_session.execute( |
| 262 | select(db.MusehubMPackIndex).where( |
| 263 | db.MusehubMPackIndex.entity_id.in_(list(objects.keys())) |
| 264 | ) |
| 265 | )).scalars().all() |
| 266 | assert {r.entity_id for r in obj_rows} == set(objects.keys()), ( |
| 267 | f"object_ids not fully indexed: {set(objects.keys()) - {r.entity_id for r in obj_rows}}" |
| 268 | ) |
| 269 | |
| 270 | # Commits in MusehubCommitGraph. |
| 271 | cg_rows = (await db_session.execute( |
| 272 | select(db.MusehubCommitGraph).where( |
| 273 | db.MusehubCommitGraph.commit_id.in_(commit_ids) |
| 274 | ) |
| 275 | )).scalars().all() |
| 276 | assert {r.commit_id for r in cg_rows} == set(commit_ids), ( |
| 277 | f"commit_ids not in MusehubCommitGraph: {set(commit_ids) - {r.commit_id for r in cg_rows}}" |
| 278 | ) |
| 279 | |
| 280 | |
| 281 | # --------------------------------------------------------------------------- |
| 282 | # PI-8 Second push idempotency for both object index and commit graph |
| 283 | # --------------------------------------------------------------------------- |
| 284 | |
| 285 | @pytest.mark.asyncio |
| 286 | async def test_pi8_second_push_indexes_without_overwriting(db_session: AsyncSession) -> None: |
| 287 | """A second push adds new object/commit rows; first push rows survive (idempotency).""" |
| 288 | repo = await create_repo( |
| 289 | db_session, |
| 290 | name="pi8-test", |
| 291 | owner="gabriel", |
| 292 | owner_user_id=compute_identity_id(b"gabriel"), |
| 293 | visibility="public", |
| 294 | initialize=False, |
| 295 | ) |
| 296 | |
| 297 | objs_a = {blob_id(f"pi8-a-{i}".encode()): f"pi8-a-{i}".encode() for i in range(2)} |
| 298 | wire_a, key_a, cids_a, _ = _make_full_mpack(repo.repo_id, objs_a, extra_tag="pi8a") |
| 299 | await _store_mpack(wire_a, key_a) |
| 300 | await _enqueue_and_process(db_session, repo.repo_id, key_a, len(objs_a)) |
| 301 | |
| 302 | objs_b = {blob_id(f"pi8-b-{i}".encode()): f"pi8-b-{i}".encode() for i in range(2)} |
| 303 | wire_b, key_b, cids_b, _ = _make_full_mpack(repo.repo_id, objs_b, extra_tag="pi8b") |
| 304 | await _store_mpack(wire_b, key_b) |
| 305 | await _enqueue_and_process(db_session, repo.repo_id, key_b, len(objs_b)) |
| 306 | |
| 307 | # All objects indexed. |
| 308 | all_oids = set(objs_a) | set(objs_b) |
| 309 | obj_rows = (await db_session.execute( |
| 310 | select(db.MusehubMPackIndex).where( |
| 311 | db.MusehubMPackIndex.entity_id.in_(list(all_oids)) |
| 312 | ) |
| 313 | )).scalars().all() |
| 314 | indexed_oids = {r.entity_id for r in obj_rows} |
| 315 | mpack_ids_found = {r.mpack_id for r in obj_rows} |
| 316 | |
| 317 | assert set(objs_a) <= indexed_oids, "first push objects missing after second push" |
| 318 | assert set(objs_b) <= indexed_oids, "second push objects missing" |
| 319 | assert key_a in mpack_ids_found, "first mpack_key lost from index" |
| 320 | assert key_b in mpack_ids_found, "second mpack_key missing from index" |
| 321 | |
| 322 | # All commits in commit graph. |
| 323 | all_cids = set(cids_a) | set(cids_b) |
| 324 | cg_rows = (await db_session.execute( |
| 325 | select(db.MusehubCommitGraph).where( |
| 326 | db.MusehubCommitGraph.commit_id.in_(list(all_cids)) |
| 327 | ) |
| 328 | )).scalars().all() |
| 329 | assert {r.commit_id for r in cg_rows} == all_cids, ( |
| 330 | f"commit graph missing rows: {all_cids - {r.commit_id for r in cg_rows}}" |
| 331 | ) |