gabriel / musehub public
test_multi_tip_manifest_union.py python
450 lines 17.9 KB
Raw
sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226 Merge branch 'fix/two-column-scroll-layout' into dev Human 9 days ago
1 """musehub#113 Phase 0 (RED) — all_oids must union every want-tip's manifest.
2
3 Sub-ticket: musehub#113 — https://staging.musehub.ai/gabriel/musehub/issues/113
4 Master: muse#63 — https://staging.musehub.ai/gabriel/muse/issues/63 (MWP-9)
5
6 Root cause: ``wire_fetch_mpack`` (``musehub_wire_fetch.py:555-559``) picks a
7 single "the tip" snapshot via::
8
9 select(MusehubCommitGraph.snapshot_id)
10 .where(MusehubCommitGraph.commit_id.in_(_needed_cids))
11 .order_by(MusehubCommitGraph.generation.desc())
12 .limit(1)
13
14 and builds ``all_oids`` (the full set of blobs to include in the response,
15 lines 612-648) from **only that one snapshot's manifest**. When a ``want``
16 set spans more than one branch tip — which every clone/fetch request does,
17 confirmed by live testing that even a plain ``muse clone`` with no
18 ``--branch`` flag sends every known tip — every tip except the single
19 globally-deepest one has its manifest silently ignored. The resulting mpack
20 has correct commits/snapshots metadata but is missing blobs unique to every
21 non-winning tip. The client reports ``exit_code: 0`` / ``"cloned"`` with an
22 incomplete working tree; ``muse verify`` reports ``all_ok: true``.
23
24 Confirmed live (against a freshly-restarted local musehub, not a stale
25 worker — see musehub#113's retraction note) via a direct
26 ``musehub_commit_graph`` query and three real clone scenarios:
27
28 MTU_01 — two sibling branches at the SAME generation (a tie): the
29 tie-losing sibling's unique blob is dropped.
30 MTU_02 — two sibling branches at DIFFERENT generations (no tie, the
31 shallower one loses deterministically): the shallower sibling's
32 unique blob is dropped.
33 MTU_03 — the DEFAULT branch itself loses its own newest content merely
34 because some other branch in the repo is deeper. This is the
35 realistic "just run `muse clone <url>`" case, not an edge case.
36
37 Expected Phase 0 status: MTU_01, MTU_02, MTU_03 all RED — each currently
38 fails because the losing tip's blob is absent from the fetch result.
39
40 The fix (Phase 1) makes ``all_oids`` the union of every want-tip's own
41 manifest, resolved from ``MusehubCommit.snapshot_id`` (already loaded into
42 ``commit_rows``, already the authoritative source per the existing MWP1_13
43 guard's own comment: "MusehubCommit always has the correct snapshot_id") —
44 not a single CommitGraph-generation-ranked pick.
45 """
46 from __future__ import annotations
47
48 import datetime
49 from unittest.mock import patch
50
51 import pytest
52 from sqlalchemy.ext.asyncio import AsyncSession
53
54 from muse.core.ids import hash_snapshot
55 from muse.core.mpack import build_wire_mpack, parse_wire_mpack
56 from muse.core.types import blob_id
57 from musehub.core.genesis import compute_identity_id
58 from musehub.services.musehub_repository import create_repo
59 from musehub.services.musehub_wire_fetch import wire_fetch_mpack
60 from musehub.services.musehub_wire_push import wire_push_unpack_mpack
61
62 _OWNER = "gabriel"
63 _IDENTITY_ID = compute_identity_id(b"gabriel")
64
65
66 # ---------------------------------------------------------------------------
67 # Helpers — mirrors test_mwp1_generation_authority.py / test_mwp2_walk_fallback.py
68 # ---------------------------------------------------------------------------
69
70
71 def _cid(seed: str) -> str:
72 """Deterministic sha256: commit id from a seed string."""
73 return blob_id(f"mtu-commit-{seed}".encode())
74
75
76 def _now() -> datetime.datetime:
77 return datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
78
79
80 def _raw_commit(cid: str, parent: str | None, snap_id: str, *, branch: str = "main") -> dict:
81 """A raw commit dict in the shape wire_push_unpack_mpack consumes."""
82 return {
83 "commit_id": cid,
84 "branch": branch,
85 "message": f"commit {cid[:12]}",
86 "author": _OWNER,
87 "committed_at": _now().isoformat(),
88 "parent_commit_id": parent,
89 "parent2_commit_id": None,
90 "snapshot_id": snap_id,
91 "agent_id": "",
92 "model_id": "",
93 "toolchain_id": "",
94 "sem_ver_bump": "none",
95 "breaking_changes": [],
96 "signature": "",
97 "signer_key_id": "",
98 "signer_public_key": "",
99 "prompt_hash": "",
100 }
101
102
103 class _InMemBackend:
104 """In-memory storage backend for E2E push->fetch tests."""
105
106 def __init__(self) -> None:
107 self._mpacks: dict[str, bytes] = {}
108
109 async def put_mpack(self, mpack_id: str, data: bytes) -> None:
110 self._mpacks[mpack_id] = data
111
112 async def get_mpack(self, mpack_id: str) -> bytes | None:
113 return self._mpacks.get(mpack_id)
114
115 async def presign_mpack_get(self, mpack_id: str, ttl: int = 3600) -> str:
116 return f"inmem://{mpack_id}"
117
118 async def put(self, oid: str, data: bytes) -> str:
119 return oid
120
121 async def get(self, oid: str) -> bytes | None:
122 return None
123
124 async def presign_get(self, oid: str, ttl: int = 3600) -> str:
125 return f"inmem://{oid}"
126
127 async def quarantine_mpack(self, *args: object, **kwargs: object) -> None:
128 pass
129
130
131 async def _push_full_manifest(
132 session: AsyncSession,
133 backend: _InMemBackend,
134 repo_id: str,
135 branch: str,
136 cid: str,
137 parent: str | None,
138 manifest: dict[str, str],
139 blobs: dict[str, bytes],
140 ) -> None:
141 """Push one commit whose snapshot's manifest is the FULL (non-delta) file set.
142
143 ``parent_snapshot_id=None`` + a complete ``delta_upsert`` is a valid,
144 self-contained root snapshot representation — avoids relying on
145 delta-chain reconstruction for this test's purposes.
146 """
147 snap_id = hash_snapshot(manifest)
148 mpack_bytes = build_wire_mpack({
149 "commits": [_raw_commit(cid, parent, snap_id, branch=branch)],
150 "snapshots": [{
151 "snapshot_id": snap_id,
152 "parent_snapshot_id": None,
153 "delta_upsert": manifest,
154 "delta_remove": [],
155 "directories": [],
156 }],
157 "blobs": [{"object_id": oid, "content": data} for oid, data in blobs.items()],
158 "tags": [],
159 })
160 mpack_key = blob_id(mpack_bytes)
161 backend._mpacks[mpack_key] = mpack_bytes
162 with patch("musehub.services.musehub_wire.get_backend", return_value=backend), \
163 patch("musehub.services.musehub_wire_push.get_backend", return_value=backend), \
164 patch("musehub.storage.backends.get_backend", return_value=backend):
165 await wire_push_unpack_mpack(
166 session,
167 repo_id,
168 mpack_key,
169 pusher_id=_OWNER,
170 branch=branch,
171 head_commit_id=cid,
172 commits_count=1,
173 blobs_count=len(blobs),
174 force=True,
175 )
176 await session.flush()
177
178
179 async def _fetch(
180 session: AsyncSession,
181 backend: _InMemBackend,
182 repo_id: str,
183 want: list[str],
184 have: list[str] | None = None,
185 ) -> dict:
186 with patch("musehub.services.musehub_wire.get_backend", return_value=backend), \
187 patch("musehub.services.musehub_wire_fetch.get_backend", return_value=backend), \
188 patch("musehub.storage.backends.get_backend", return_value=backend):
189 result = await wire_fetch_mpack(session, repo_id, want=want, have=have or [], force_build=True)
190 if not result.get("mpack_id"):
191 return {"commits": [], "snapshots": [], "blobs": []}
192 mpack_bytes = backend._mpacks[result["mpack_id"]]
193 return parse_wire_mpack(mpack_bytes)
194
195
196 # ---------------------------------------------------------------------------
197 # MTU_01 — sibling branches at the SAME generation (a tie)
198 # ---------------------------------------------------------------------------
199
200
201 @pytest.mark.asyncio
202 async def test_mtu_01_sibling_tie_both_manifests_included(db_session: AsyncSession) -> None:
203 """RED: feature-a and feature-b are both gen-1 children of main (a tie).
204
205 want=[feature-a tip, feature-b tip] must include BOTH a.txt and b.txt.
206 Today only whichever tip wins the arbitrary tie-break is included.
207 """
208 repo = await create_repo(
209 db_session, name="mtu-01", owner=_OWNER, owner_user_id=_IDENTITY_ID,
210 visibility="public", initialize=False,
211 )
212 await db_session.commit()
213 backend = _InMemBackend()
214
215 oid_shared = blob_id(b"mtu01-shared")
216 oid_a = blob_id(b"mtu01-a")
217 oid_b = blob_id(b"mtu01-b")
218 m1 = _cid("01-main")
219 fa1 = _cid("01-feature-a")
220 fb1 = _cid("01-feature-b")
221
222 await _push_full_manifest(
223 db_session, backend, repo.repo_id, "main", m1, None,
224 {"shared.txt": oid_shared}, {oid_shared: b"shared"},
225 )
226 await _push_full_manifest(
227 db_session, backend, repo.repo_id, "feature-a", fa1, m1,
228 {"shared.txt": oid_shared, "a.txt": oid_a}, {oid_a: b"aa"},
229 )
230 await _push_full_manifest(
231 db_session, backend, repo.repo_id, "feature-b", fb1, m1,
232 {"shared.txt": oid_shared, "b.txt": oid_b}, {oid_b: b"bb"},
233 )
234
235 parsed = await _fetch(db_session, backend, repo.repo_id, want=[fa1, fb1])
236 blob_oids = {b["object_id"] for b in parsed.get("blobs", [])}
237
238 assert oid_a in blob_oids, (
239 f"feature-a's unique blob must be present; blobs found: {[o[:16] for o in blob_oids]}"
240 )
241 assert oid_b in blob_oids, (
242 f"feature-b's unique blob must be present; blobs found: {[o[:16] for o in blob_oids]}"
243 )
244
245
246 # ---------------------------------------------------------------------------
247 # MTU_02 — sibling branches at DIFFERENT generations (no tie)
248 # ---------------------------------------------------------------------------
249
250
251 @pytest.mark.asyncio
252 async def test_mtu_02_shallower_sibling_manifest_included(db_session: AsyncSession) -> None:
253 """RED: feature-deep (gen 3) and feature-shallow (gen 1) share base main.
254
255 want=[feature-deep tip, feature-shallow tip] must include shallow.txt.
256 Today feature-deep unambiguously wins ORDER BY generation DESC LIMIT 1,
257 so feature-shallow's manifest — and shallow.txt — is never consulted.
258 """
259 repo = await create_repo(
260 db_session, name="mtu-02", owner=_OWNER, owner_user_id=_IDENTITY_ID,
261 visibility="public", initialize=False,
262 )
263 await db_session.commit()
264 backend = _InMemBackend()
265
266 oid_shared = blob_id(b"mtu02-shared")
267 oid_d1, oid_d2, oid_d3 = (blob_id(f"mtu02-d{i}".encode()) for i in (1, 2, 3))
268 oid_shallow = blob_id(b"mtu02-shallow")
269
270 m1 = _cid("02-main")
271 fd1, fd2, fd3 = _cid("02-fd1"), _cid("02-fd2"), _cid("02-fd3")
272 fs1 = _cid("02-fs1")
273
274 await _push_full_manifest(
275 db_session, backend, repo.repo_id, "main", m1, None,
276 {"shared.txt": oid_shared}, {oid_shared: b"shared"},
277 )
278 await _push_full_manifest(
279 db_session, backend, repo.repo_id, "feature-deep", fd1, m1,
280 {"shared.txt": oid_shared, "d1.txt": oid_d1}, {oid_d1: b"d1"},
281 )
282 await _push_full_manifest(
283 db_session, backend, repo.repo_id, "feature-deep", fd2, fd1,
284 {"shared.txt": oid_shared, "d1.txt": oid_d1, "d2.txt": oid_d2}, {oid_d2: b"d2"},
285 )
286 await _push_full_manifest(
287 db_session, backend, repo.repo_id, "feature-deep", fd3, fd2,
288 {"shared.txt": oid_shared, "d1.txt": oid_d1, "d2.txt": oid_d2, "d3.txt": oid_d3},
289 {oid_d3: b"d3"},
290 )
291 await _push_full_manifest(
292 db_session, backend, repo.repo_id, "feature-shallow", fs1, m1,
293 {"shared.txt": oid_shared, "shallow.txt": oid_shallow}, {oid_shallow: b"shallow"},
294 )
295
296 parsed = await _fetch(db_session, backend, repo.repo_id, want=[fd3, fs1])
297 blob_oids = {b["object_id"] for b in parsed.get("blobs", [])}
298
299 assert oid_shallow in blob_oids, (
300 f"feature-shallow's unique blob must be present even though feature-deep "
301 f"is unambiguously deeper (no tie); blobs found: {[o[:16] for o in blob_oids]}"
302 )
303
304
305 # ---------------------------------------------------------------------------
306 # MTU_03 — the DEFAULT branch loses its own content to a deeper sibling
307 # ---------------------------------------------------------------------------
308
309
310 @pytest.mark.asyncio
311 async def test_mtu_03_default_branch_keeps_its_own_newest_content(db_session: AsyncSession) -> None:
312 """RED: main gets its own new commit; feature-deep is deeper elsewhere.
313
314 want=[main tip, feature-deep tip] (mirrors a real clone, which always
315 requests every known tip) must include main's own newest blob. This is
316 the realistic "just run `muse clone <url>`" case, not a contrived edge
317 case: any repo with an active feature branch deeper than main will
318 silently corrupt clones of main itself today.
319 """
320 repo = await create_repo(
321 db_session, name="mtu-03", owner=_OWNER, owner_user_id=_IDENTITY_ID,
322 visibility="public", initialize=False,
323 )
324 await db_session.commit()
325 backend = _InMemBackend()
326
327 oid_shared = blob_id(b"mtu03-shared")
328 oid_main_unique = blob_id(b"mtu03-main-unique")
329 oid_d1, oid_d2, oid_d3 = (blob_id(f"mtu03-d{i}".encode()) for i in (1, 2, 3))
330
331 m1 = _cid("03-main1")
332 m2 = _cid("03-main2")
333 fd1, fd2, fd3 = _cid("03-fd1"), _cid("03-fd2"), _cid("03-fd3")
334
335 await _push_full_manifest(
336 db_session, backend, repo.repo_id, "main", m1, None,
337 {"shared.txt": oid_shared}, {oid_shared: b"shared"},
338 )
339 await _push_full_manifest(
340 db_session, backend, repo.repo_id, "feature-deep", fd1, m1,
341 {"shared.txt": oid_shared, "d1.txt": oid_d1}, {oid_d1: b"d1"},
342 )
343 await _push_full_manifest(
344 db_session, backend, repo.repo_id, "feature-deep", fd2, fd1,
345 {"shared.txt": oid_shared, "d1.txt": oid_d1, "d2.txt": oid_d2}, {oid_d2: b"d2"},
346 )
347 await _push_full_manifest(
348 db_session, backend, repo.repo_id, "feature-deep", fd3, fd2,
349 {"shared.txt": oid_shared, "d1.txt": oid_d1, "d2.txt": oid_d2, "d3.txt": oid_d3},
350 {oid_d3: b"d3"},
351 )
352 # main gets its own new commit — pushed directly to the default branch,
353 # never inherited by feature-deep.
354 await _push_full_manifest(
355 db_session, backend, repo.repo_id, "main", m2, m1,
356 {"shared.txt": oid_shared, "main_unique.txt": oid_main_unique},
357 {oid_main_unique: b"main-only-content"},
358 )
359
360 parsed = await _fetch(db_session, backend, repo.repo_id, want=[m2, fd3])
361 blob_oids = {b["object_id"] for b in parsed.get("blobs", [])}
362
363 assert oid_main_unique in blob_oids, (
364 f"main's own newest blob must be present in its own clone regardless of "
365 f"any other branch's depth; blobs found: {[o[:16] for o in blob_oids]}"
366 )
367
368
369 # ---------------------------------------------------------------------------
370 # MTU_04 — the mirror bug: `have_oids` also only consults ONE have-tip
371 # ---------------------------------------------------------------------------
372
373
374 @pytest.mark.asyncio
375 async def test_mtu_04_have_oids_unions_every_have_tip(db_session: AsyncSession) -> None:
376 """RED: have_oids must union every tip in `have`, not pick one.
377
378 Client already fully has BOTH have-shallow (adds file_a.txt, generation 1)
379 and have-deep (two commits deeper, generation 3 — deterministically wins
380 any generation-ranked pick, no tie involved). A new push, `target`, is a
381 sibling of have-shallow that reuses have-shallow's exact file_a.txt
382 content (same oid) plus adds its own new file_target.txt.
383
384 want=[target], have=[have-shallow tip, have-deep tip]. Correct
385 behaviour: the response must exclude file_a.txt's oid (the client
386 already has it via have-shallow) and include only file_target.txt's
387 oid as genuinely new.
388
389 Today `have_oids` is built from a single CommitGraph generation-ranked
390 pick across the whole `have` set — have-deep unambiguously wins, so
391 have-shallow's manifest (and file_a.txt's oid) is never consulted, and
392 file_a.txt is wrongly treated as new. Not a data-loss bug like
393 MTU_01-03 — a wasted-bandwidth bug: the server redundantly re-sends
394 content the client already has via a sibling it correctly reported
395 having.
396 """
397 repo = await create_repo(
398 db_session, name="mtu-04", owner=_OWNER, owner_user_id=_IDENTITY_ID,
399 visibility="public", initialize=False,
400 )
401 await db_session.commit()
402 backend = _InMemBackend()
403
404 oid_shared = blob_id(b"mtu04-shared")
405 oid_a = blob_id(b"mtu04-file-a")
406 oid_d1, oid_d2 = blob_id(b"mtu04-deep1"), blob_id(b"mtu04-deep2")
407 oid_target = blob_id(b"mtu04-file-target")
408
409 m1 = _cid("04-main")
410 hs1 = _cid("04-have-shallow")
411 hd1, hd2 = _cid("04-have-deep-1"), _cid("04-have-deep-2")
412 t1 = _cid("04-target")
413
414 await _push_full_manifest(
415 db_session, backend, repo.repo_id, "main", m1, None,
416 {"shared.txt": oid_shared}, {oid_shared: b"shared"},
417 )
418 await _push_full_manifest(
419 db_session, backend, repo.repo_id, "have-shallow", hs1, m1,
420 {"shared.txt": oid_shared, "file_a.txt": oid_a}, {oid_a: b"file-a-content"},
421 )
422 await _push_full_manifest(
423 db_session, backend, repo.repo_id, "have-deep", hd1, m1,
424 {"shared.txt": oid_shared, "deep1.txt": oid_d1}, {oid_d1: b"deep1"},
425 )
426 await _push_full_manifest(
427 db_session, backend, repo.repo_id, "have-deep", hd2, hd1,
428 {"shared.txt": oid_shared, "deep1.txt": oid_d1, "deep2.txt": oid_d2}, {oid_d2: b"deep2"},
429 )
430 # target reuses file_a's exact oid (client already has it via
431 # have-shallow) and adds one genuinely new file.
432 await _push_full_manifest(
433 db_session, backend, repo.repo_id, "target", t1, m1,
434 {"shared.txt": oid_shared, "file_a.txt": oid_a, "file_target.txt": oid_target},
435 {oid_a: b"file-a-content", oid_target: b"file-target-content"},
436 )
437
438 parsed = await _fetch(db_session, backend, repo.repo_id, want=[t1], have=[hs1, hd2])
439 blob_oids = {b["object_id"] for b in parsed.get("blobs", [])}
440
441 assert oid_target in blob_oids, (
442 f"file_target.txt is genuinely new and must be sent; blobs found: {[o[:16] for o in blob_oids]}"
443 )
444 assert oid_a not in blob_oids, (
445 f"file_a.txt must NOT be re-sent — the client already has it via "
446 f"have-shallow, one of the two tips in `have`. Its presence here means "
447 f"have_oids only consulted have-deep's manifest (the deterministic "
448 f"generation winner), not the union of both have-tips. "
449 f"blobs found: {[o[:16] for o in blob_oids]}"
450 )
File History 3 commits
sha256:fc04e4cae9e1774d6a21b65c45daeed0e6787eb581d13aa1b03bfe9384a34226 Merge branch 'fix/two-column-scroll-layout' into dev Human 9 days ago
sha256:408916fc5973ba59c6e4eebaa80ebdcc801c0a63205651e25009d11548f79454 chore: bump version to 0.2.0.dev2 — nightly.2, matching muse Sonnet 4.6 patch 12 days ago
sha256:d035733f21ccff27735fddebfbbe0ed24565a32a22db8de5885402262671ecd2 chore: bump version to 0.2.0rc15 for musehub#113 fix release Sonnet 4.6 patch 15 days ago