test_store_fsync_enospc.py
python
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
| 1 | """ |
| 2 | Tests for the bug: write_text_atomic and _write_msgpack_atomic both silently |
| 3 | swallow ALL OSError from os.fsync() — including ENOSPC and EIO — instead of |
| 4 | only suppressing EINVAL (the errno virtual filesystems return to indicate |
| 5 | fsync is unsupported). |
| 6 | |
| 7 | Root cause (muse/core/store.py): |
| 8 | |
| 9 | write_text_atomic lines 324–327: |
| 10 | try: |
| 11 | os.fsync(fh.fileno()) |
| 12 | except OSError: |
| 13 | pass # best-effort ← BUG: swallows ENOSPC, EIO, etc. |
| 14 | |
| 15 | _write_msgpack_atomic lines 416–419: |
| 16 | except OSError: |
| 17 | pass # best-effort — some virtual filesystems do not support fsync |
| 18 | |
| 19 | When a disk is full (ENOSPC) or has a hardware error (EIO), fsync raises an |
| 20 | OSError with errno.ENOSPC or errno.EIO. The current code silently swallows |
| 21 | these errors. tmp.replace(path) then succeeds — the target file now points at |
| 22 | a temp file whose data is only in the page cache. The caller sees a normal |
| 23 | return (no exception) and believes the write succeeded. The OS may silently |
| 24 | discard the page-cache data if it cannot flush it to disk. |
| 25 | |
| 26 | The fix: only suppress errno.EINVAL. Re-raise everything else (ENOSPC, EIO, |
| 27 | EROFS, EBADF, …). |
| 28 | |
| 29 | Coverage: |
| 30 | Unit — write_text_atomic raises on ENOSPC, EIO; suppresses EINVAL |
| 31 | Unit — _write_msgpack_atomic raises on ENOSPC, EIO; suppresses EINVAL |
| 32 | Unit darwin — F_BARRIERFSYNC path: outer except only suppresses EINVAL |
| 33 | Data integrity — after ENOSPC, no misleading success state in caller |
| 34 | Security — ENOSPC during HEAD/branch ref writes propagates (not silenced) |
| 35 | Integration — coord record write propagates ENOSPC to _write_remote_records |
| 36 | E2E — CLI coord sync gets clean error, not silent corruption |
| 37 | Stress — rapid repeated ENOSPC raises, never succeeds silently |
| 38 | Performance — suppressed EINVAL path (normal) is not dramatically slower |
| 39 | Regression — EINVAL is still suppressed (virtual filesystem compatibility) |
| 40 | """ |
| 41 | from __future__ import annotations |
| 42 | |
| 43 | import errno |
| 44 | import os |
| 45 | import pathlib |
| 46 | import sys |
| 47 | import tempfile |
| 48 | import threading |
| 49 | import time |
| 50 | from collections.abc import Generator |
| 51 | from contextlib import AbstractContextManager |
| 52 | from unittest.mock import MagicMock, patch |
| 53 | |
| 54 | import pytest |
| 55 | |
| 56 | from muse.core._types import MsgpackDict |
| 57 | |
| 58 | # --------------------------------------------------------------------------- |
| 59 | # Helpers |
| 60 | # --------------------------------------------------------------------------- |
| 61 | |
| 62 | |
| 63 | def _make_repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 64 | (tmp_path / ".muse").mkdir(parents=True, exist_ok=True) |
| 65 | return tmp_path |
| 66 | |
| 67 | |
| 68 | def _oserror(err: int) -> OSError: |
| 69 | e = OSError(err, os.strerror(err)) |
| 70 | e.errno = err |
| 71 | return e |
| 72 | |
| 73 | |
| 74 | def _patch_msgpack_sync(err: int) -> AbstractContextManager[None]: |
| 75 | """Context manager: make all sync paths in _write_msgpack_atomic raise `err`. |
| 76 | |
| 77 | On darwin, _write_msgpack_atomic calls fcntl.fcntl(fd, 85) first; if that |
| 78 | raises ANY OSError it falls through to os.fsync(). To exercise the outer |
| 79 | `except OSError` block we must patch BOTH paths so neither succeeds. |
| 80 | |
| 81 | On other platforms only os.fsync is called. |
| 82 | """ |
| 83 | import contextlib |
| 84 | |
| 85 | @contextlib.contextmanager |
| 86 | def _ctx() -> Generator[None, None, None]: |
| 87 | exc = _oserror(err) |
| 88 | if sys.platform == "darwin": |
| 89 | import fcntl as _fcntl |
| 90 | with ( |
| 91 | patch.object(_fcntl, "fcntl", side_effect=exc), |
| 92 | patch("os.fsync", side_effect=exc), |
| 93 | ): |
| 94 | yield |
| 95 | else: |
| 96 | with patch("os.fsync", side_effect=exc): |
| 97 | yield |
| 98 | |
| 99 | return _ctx() |
| 100 | |
| 101 | |
| 102 | # ============================================================================= |
| 103 | # 1. UNIT — write_text_atomic fsync error handling |
| 104 | # ============================================================================= |
| 105 | |
| 106 | |
| 107 | class TestWriteTextAtomicFsync: |
| 108 | """write_text_atomic must re-raise fatal OSErrors and suppress only EINVAL.""" |
| 109 | |
| 110 | def test_enospc_raises(self, tmp_path: pathlib.Path) -> None: |
| 111 | """ENOSPC from fsync must propagate — disk full is a fatal error.""" |
| 112 | from muse.core.store import write_text_atomic |
| 113 | |
| 114 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 115 | with pytest.raises(OSError) as exc_info: |
| 116 | write_text_atomic(tmp_path / "test.txt", "hello") |
| 117 | assert exc_info.value.errno == errno.ENOSPC |
| 118 | |
| 119 | def test_eio_raises(self, tmp_path: pathlib.Path) -> None: |
| 120 | """EIO from fsync must propagate — hardware error is fatal.""" |
| 121 | from muse.core.store import write_text_atomic |
| 122 | |
| 123 | with patch("os.fsync", side_effect=_oserror(errno.EIO)): |
| 124 | with pytest.raises(OSError) as exc_info: |
| 125 | write_text_atomic(tmp_path / "test.txt", "hello") |
| 126 | assert exc_info.value.errno == errno.EIO |
| 127 | |
| 128 | def test_erofs_raises(self, tmp_path: pathlib.Path) -> None: |
| 129 | """EROFS (read-only filesystem) from fsync must propagate.""" |
| 130 | from muse.core.store import write_text_atomic |
| 131 | |
| 132 | with patch("os.fsync", side_effect=_oserror(errno.EROFS)): |
| 133 | with pytest.raises(OSError) as exc_info: |
| 134 | write_text_atomic(tmp_path / "test.txt", "hello") |
| 135 | assert exc_info.value.errno == errno.EROFS |
| 136 | |
| 137 | def test_einval_suppressed(self, tmp_path: pathlib.Path) -> None: |
| 138 | """EINVAL from fsync must be silently suppressed (virtual filesystem compat).""" |
| 139 | from muse.core.store import write_text_atomic |
| 140 | |
| 141 | with patch("os.fsync", side_effect=_oserror(errno.EINVAL)): |
| 142 | write_text_atomic(tmp_path / "test.txt", "hello") # must not raise |
| 143 | assert (tmp_path / "test.txt").read_text() == "hello" |
| 144 | |
| 145 | def test_enospc_leaves_no_temp_files(self, tmp_path: pathlib.Path) -> None: |
| 146 | """On ENOSPC, the temp file must be cleaned up — no orphaned .muse-tmp-* files.""" |
| 147 | from muse.core.store import write_text_atomic |
| 148 | |
| 149 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 150 | with pytest.raises(OSError): |
| 151 | write_text_atomic(tmp_path / "output.txt", "data") |
| 152 | |
| 153 | tmp_files = list(tmp_path.glob(".muse-tmp-*")) |
| 154 | assert tmp_files == [], f"orphaned temp files after ENOSPC: {tmp_files}" |
| 155 | |
| 156 | def test_enospc_does_not_create_target(self, tmp_path: pathlib.Path) -> None: |
| 157 | """On ENOSPC, the target file must not be created (rename never called).""" |
| 158 | from muse.core.store import write_text_atomic |
| 159 | |
| 160 | target = tmp_path / "should-not-exist.txt" |
| 161 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 162 | with pytest.raises(OSError): |
| 163 | write_text_atomic(target, "data") |
| 164 | |
| 165 | assert not target.exists(), "target file created despite ENOSPC" |
| 166 | |
| 167 | def test_enospc_does_not_overwrite_existing(self, tmp_path: pathlib.Path) -> None: |
| 168 | """On ENOSPC, an existing target file must be preserved (not replaced).""" |
| 169 | from muse.core.store import write_text_atomic |
| 170 | |
| 171 | target = tmp_path / "existing.txt" |
| 172 | target.write_text("original content") |
| 173 | |
| 174 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 175 | with pytest.raises(OSError): |
| 176 | write_text_atomic(target, "new content") |
| 177 | |
| 178 | assert target.read_text() == "original content", ( |
| 179 | "existing file was overwritten despite ENOSPC" |
| 180 | ) |
| 181 | |
| 182 | def test_successful_write_still_works(self, tmp_path: pathlib.Path) -> None: |
| 183 | """After the fix, normal writes (no fsync error) must still succeed.""" |
| 184 | from muse.core.store import write_text_atomic |
| 185 | |
| 186 | write_text_atomic(tmp_path / "ok.txt", "success") |
| 187 | assert (tmp_path / "ok.txt").read_text() == "success" |
| 188 | |
| 189 | def test_multiple_enospc_all_raise(self, tmp_path: pathlib.Path) -> None: |
| 190 | """Every ENOSPC call raises — no silent tolerance after repeated failures.""" |
| 191 | from muse.core.store import write_text_atomic |
| 192 | |
| 193 | for i in range(10): |
| 194 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 195 | with pytest.raises(OSError) as exc_info: |
| 196 | write_text_atomic(tmp_path / f"file-{i}.txt", f"content-{i}") |
| 197 | assert exc_info.value.errno == errno.ENOSPC |
| 198 | |
| 199 | |
| 200 | # ============================================================================= |
| 201 | # 2. UNIT — _write_msgpack_atomic fsync error handling |
| 202 | # ============================================================================= |
| 203 | |
| 204 | |
| 205 | class TestWriteMsgpackAtomicFsync: |
| 206 | """_write_msgpack_atomic must re-raise fatal OSErrors and suppress only EINVAL.""" |
| 207 | |
| 208 | def _make_record(self) -> MsgpackDict: |
| 209 | return { |
| 210 | "kind": "commit", |
| 211 | "message": "test", |
| 212 | "run_id": "run-test", |
| 213 | "payload": {}, |
| 214 | } |
| 215 | |
| 216 | def _call(self, tmp_path: pathlib.Path, data: MsgpackDict | None = None) -> None: |
| 217 | from muse.core.store import _write_msgpack_atomic |
| 218 | |
| 219 | if data is None: |
| 220 | data = self._make_record() |
| 221 | _write_msgpack_atomic(tmp_path / "record.msgpack", data) |
| 222 | |
| 223 | def _patch_fsync_enospc(self) -> AbstractContextManager[None]: |
| 224 | """Context manager that makes all sync paths raise ENOSPC. |
| 225 | |
| 226 | On darwin, _write_msgpack_atomic uses fcntl.fcntl(fd, 85) first. |
| 227 | We must patch both paths so the ENOSPC reaches the outer except. |
| 228 | """ |
| 229 | import contextlib |
| 230 | |
| 231 | @contextlib.contextmanager |
| 232 | def _ctx() -> Generator[None, None, None]: |
| 233 | enospc = _oserror(errno.ENOSPC) |
| 234 | if sys.platform == "darwin": |
| 235 | import fcntl as _fcntl |
| 236 | with ( |
| 237 | patch.object(_fcntl, "fcntl", side_effect=enospc), |
| 238 | patch("os.fsync", side_effect=enospc), |
| 239 | ): |
| 240 | yield |
| 241 | else: |
| 242 | with patch("os.fsync", side_effect=enospc): |
| 243 | yield |
| 244 | |
| 245 | return _ctx() |
| 246 | |
| 247 | def _patch_fsync_eio(self) -> AbstractContextManager[None]: |
| 248 | import contextlib |
| 249 | |
| 250 | @contextlib.contextmanager |
| 251 | def _ctx() -> Generator[None, None, None]: |
| 252 | eio = _oserror(errno.EIO) |
| 253 | if sys.platform == "darwin": |
| 254 | import fcntl as _fcntl |
| 255 | with ( |
| 256 | patch.object(_fcntl, "fcntl", side_effect=eio), |
| 257 | patch("os.fsync", side_effect=eio), |
| 258 | ): |
| 259 | yield |
| 260 | else: |
| 261 | with patch("os.fsync", side_effect=eio): |
| 262 | yield |
| 263 | |
| 264 | return _ctx() |
| 265 | |
| 266 | def _patch_fsync_einval(self) -> AbstractContextManager[None]: |
| 267 | import contextlib |
| 268 | |
| 269 | @contextlib.contextmanager |
| 270 | def _ctx() -> Generator[None, None, None]: |
| 271 | einval = _oserror(errno.EINVAL) |
| 272 | if sys.platform == "darwin": |
| 273 | import fcntl as _fcntl |
| 274 | with ( |
| 275 | patch.object(_fcntl, "fcntl", side_effect=einval), |
| 276 | patch("os.fsync", side_effect=einval), |
| 277 | ): |
| 278 | yield |
| 279 | else: |
| 280 | with patch("os.fsync", side_effect=einval): |
| 281 | yield |
| 282 | |
| 283 | return _ctx() |
| 284 | |
| 285 | def test_enospc_raises(self, tmp_path: pathlib.Path) -> None: |
| 286 | """ENOSPC from fsync must propagate.""" |
| 287 | with self._patch_fsync_enospc(): |
| 288 | with pytest.raises(OSError) as exc_info: |
| 289 | self._call(tmp_path) |
| 290 | assert exc_info.value.errno == errno.ENOSPC |
| 291 | |
| 292 | def test_eio_raises(self, tmp_path: pathlib.Path) -> None: |
| 293 | """EIO from fsync must propagate.""" |
| 294 | with self._patch_fsync_eio(): |
| 295 | with pytest.raises(OSError) as exc_info: |
| 296 | self._call(tmp_path) |
| 297 | assert exc_info.value.errno == errno.EIO |
| 298 | |
| 299 | def test_einval_suppressed(self, tmp_path: pathlib.Path) -> None: |
| 300 | """EINVAL from fsync must be suppressed.""" |
| 301 | with self._patch_fsync_einval(): |
| 302 | self._call(tmp_path) # must not raise |
| 303 | assert (tmp_path / "record.msgpack").exists() |
| 304 | |
| 305 | def test_enospc_leaves_no_temp_files(self, tmp_path: pathlib.Path) -> None: |
| 306 | """ENOSPC must not leave orphaned temp files.""" |
| 307 | with self._patch_fsync_enospc(): |
| 308 | with pytest.raises(OSError): |
| 309 | self._call(tmp_path) |
| 310 | tmp_files = list(tmp_path.glob(".muse-tmp-*")) |
| 311 | assert tmp_files == [], f"orphaned temp files: {tmp_files}" |
| 312 | |
| 313 | def test_enospc_does_not_create_target(self, tmp_path: pathlib.Path) -> None: |
| 314 | """ENOSPC must not create the target file.""" |
| 315 | target = tmp_path / "record.msgpack" |
| 316 | with self._patch_fsync_enospc(): |
| 317 | with pytest.raises(OSError): |
| 318 | from muse.core.store import _write_msgpack_atomic |
| 319 | _write_msgpack_atomic(target, self._make_record()) |
| 320 | assert not target.exists() |
| 321 | |
| 322 | def test_successful_write_still_works(self, tmp_path: pathlib.Path) -> None: |
| 323 | """Normal write must still succeed after the fix.""" |
| 324 | self._call(tmp_path) |
| 325 | assert (tmp_path / "record.msgpack").exists() |
| 326 | |
| 327 | |
| 328 | # ============================================================================= |
| 329 | # 2b. UNIT — darwin F_BARRIERFSYNC path (only runs on macOS) |
| 330 | # ============================================================================= |
| 331 | |
| 332 | |
| 333 | @pytest.mark.skipif(sys.platform != "darwin", reason="darwin-specific code path") |
| 334 | class TestWriteMsgpackAtomicDarwin: |
| 335 | """On darwin, F_BARRIERFSYNC (85) is tried first; outer except must not swallow ENOSPC.""" |
| 336 | |
| 337 | def _call(self, tmp_path: pathlib.Path) -> None: |
| 338 | import fcntl |
| 339 | |
| 340 | from muse.core.store import _write_msgpack_atomic |
| 341 | |
| 342 | _write_msgpack_atomic(tmp_path / "rec.msgpack", {"x": 1}) |
| 343 | |
| 344 | def test_f_barrierfsync_enospc_fallthrough_fsync_succeeds(self, tmp_path: pathlib.Path) -> None: |
| 345 | """F_BARRIERFSYNC raises ENOSPC → falls through to os.fsync → fsync OK → write succeeds. |
| 346 | |
| 347 | F_BARRIERFSYNC failure triggers the fallback path (os.fsync), not an |
| 348 | immediate error. The write only fails if os.fsync also fails. |
| 349 | """ |
| 350 | import fcntl |
| 351 | |
| 352 | with patch("fcntl.fcntl", side_effect=_oserror(errno.ENOSPC)): |
| 353 | # os.fsync is NOT patched — it succeeds → write must succeed |
| 354 | self._call(tmp_path) # must not raise |
| 355 | assert (tmp_path / "rec.msgpack").exists() |
| 356 | |
| 357 | def test_f_barrierfsync_enospc_and_fsync_enospc_raises(self, tmp_path: pathlib.Path) -> None: |
| 358 | """F_BARRIERFSYNC raises ENOSPC → fallthrough → os.fsync raises ENOSPC → propagated.""" |
| 359 | import fcntl |
| 360 | |
| 361 | with _patch_msgpack_sync(errno.ENOSPC): |
| 362 | with pytest.raises(OSError) as exc_info: |
| 363 | self._call(tmp_path) |
| 364 | assert exc_info.value.errno == errno.ENOSPC |
| 365 | |
| 366 | def test_f_barrierfsync_einval_falls_through_to_fsync_einval_suppressed(self, tmp_path: pathlib.Path) -> None: |
| 367 | """F_BARRIERFSYNC raises EINVAL → falls through to fsync → fsync raises EINVAL → suppressed.""" |
| 368 | import fcntl |
| 369 | |
| 370 | with ( |
| 371 | patch("fcntl.fcntl", side_effect=_oserror(errno.EINVAL)), |
| 372 | patch("os.fsync", side_effect=_oserror(errno.EINVAL)), |
| 373 | ): |
| 374 | self._call(tmp_path) # must not raise |
| 375 | |
| 376 | def test_f_barrierfsync_einval_falls_through_fsync_enospc_raises(self, tmp_path: pathlib.Path) -> None: |
| 377 | """F_BARRIERFSYNC raises EINVAL → fsync raises ENOSPC → propagated.""" |
| 378 | import fcntl |
| 379 | |
| 380 | with ( |
| 381 | patch("fcntl.fcntl", side_effect=_oserror(errno.EINVAL)), |
| 382 | patch("os.fsync", side_effect=_oserror(errno.ENOSPC)), |
| 383 | ): |
| 384 | with pytest.raises(OSError) as exc_info: |
| 385 | self._call(tmp_path) |
| 386 | assert exc_info.value.errno == errno.ENOSPC |
| 387 | |
| 388 | |
| 389 | # ============================================================================= |
| 390 | # 3. DATA INTEGRITY — caller sees exception, not silent success |
| 391 | # ============================================================================= |
| 392 | |
| 393 | |
| 394 | class TestDataIntegrityOnEnospc: |
| 395 | """After ENOSPC, callers must see an exception — never a silent success.""" |
| 396 | |
| 397 | def test_write_text_atomic_enospc_exception_propagates_to_caller(self, tmp_path: pathlib.Path) -> None: |
| 398 | """Callers of write_text_atomic must see OSError on ENOSPC.""" |
| 399 | from muse.core.store import write_text_atomic |
| 400 | |
| 401 | result = None |
| 402 | exception = None |
| 403 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 404 | try: |
| 405 | write_text_atomic(tmp_path / "out.txt", "data") |
| 406 | result = "success" |
| 407 | except OSError as e: |
| 408 | exception = e |
| 409 | |
| 410 | assert result is None, "write_text_atomic returned normally despite ENOSPC" |
| 411 | assert exception is not None |
| 412 | assert exception.errno == errno.ENOSPC |
| 413 | |
| 414 | def test_write_msgpack_atomic_enospc_exception_propagates_to_caller(self, tmp_path: pathlib.Path) -> None: |
| 415 | """Callers of _write_msgpack_atomic must see OSError on ENOSPC.""" |
| 416 | from muse.core.store import _write_msgpack_atomic |
| 417 | |
| 418 | exception = None |
| 419 | with _patch_msgpack_sync(errno.ENOSPC): |
| 420 | try: |
| 421 | _write_msgpack_atomic(tmp_path / "rec.msgpack", {"k": "v"}) |
| 422 | except OSError as e: |
| 423 | exception = e |
| 424 | |
| 425 | assert exception is not None, "_write_msgpack_atomic swallowed ENOSPC" |
| 426 | assert exception.errno == errno.ENOSPC |
| 427 | |
| 428 | def test_no_stale_state_after_enospc(self, tmp_path: pathlib.Path) -> None: |
| 429 | """After ENOSPC, no partial state should exist in the target path.""" |
| 430 | from muse.core.store import write_text_atomic |
| 431 | |
| 432 | target = tmp_path / "state.txt" |
| 433 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 434 | with pytest.raises(OSError): |
| 435 | write_text_atomic(target, "new state") |
| 436 | |
| 437 | # Target must not exist (was not pre-existing) |
| 438 | assert not target.exists() |
| 439 | |
| 440 | def test_old_file_preserved_after_enospc(self, tmp_path: pathlib.Path) -> None: |
| 441 | """When overwriting, ENOSPC must leave the old file intact.""" |
| 442 | from muse.core.store import write_text_atomic |
| 443 | |
| 444 | target = tmp_path / "config.txt" |
| 445 | target.write_text("version: 1") |
| 446 | |
| 447 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 448 | with pytest.raises(OSError): |
| 449 | write_text_atomic(target, "version: 2") |
| 450 | |
| 451 | assert target.read_text() == "version: 1", "old config was destroyed on ENOSPC" |
| 452 | |
| 453 | |
| 454 | # ============================================================================= |
| 455 | # 4. SECURITY — critical VCS state writes must propagate ENOSPC |
| 456 | # ============================================================================= |
| 457 | |
| 458 | |
| 459 | class TestSecurityCriticalWritesEnospc: |
| 460 | """HEAD, branch refs, and coordination records must not silently corrupt on ENOSPC.""" |
| 461 | |
| 462 | def test_write_head_enospc_raises(self, tmp_path: pathlib.Path) -> None: |
| 463 | """Writing HEAD ref must propagate ENOSPC.""" |
| 464 | from muse.core.store import write_text_atomic |
| 465 | |
| 466 | head_path = tmp_path / ".muse" / "HEAD" |
| 467 | head_path.parent.mkdir(parents=True, exist_ok=True) |
| 468 | |
| 469 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 470 | with pytest.raises(OSError) as exc_info: |
| 471 | write_text_atomic(head_path, "ref: refs/heads/main\n") |
| 472 | assert exc_info.value.errno == errno.ENOSPC |
| 473 | assert not head_path.exists() |
| 474 | |
| 475 | def test_write_branch_ref_enospc_raises(self, tmp_path: pathlib.Path) -> None: |
| 476 | """Writing branch ref must propagate ENOSPC.""" |
| 477 | from muse.core.store import write_text_atomic |
| 478 | |
| 479 | ref_path = tmp_path / ".muse" / "refs" / "heads" / "main" |
| 480 | ref_path.parent.mkdir(parents=True, exist_ok=True) |
| 481 | |
| 482 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 483 | with pytest.raises(OSError) as exc_info: |
| 484 | write_text_atomic(ref_path, "abc123def456\n") |
| 485 | assert exc_info.value.errno == errno.ENOSPC |
| 486 | |
| 487 | def test_write_coord_record_enospc_propagates(self, tmp_path: pathlib.Path) -> None: |
| 488 | """Writing a coordination record must propagate ENOSPC.""" |
| 489 | root = _make_repo(tmp_path) |
| 490 | |
| 491 | import json |
| 492 | |
| 493 | from muse.cli.commands.coord_sync import _write_remote_records |
| 494 | |
| 495 | rec = { |
| 496 | "kind": "reservation", |
| 497 | "record_uuid": "res-enospc-test", |
| 498 | "run_id": "run-test", |
| 499 | "payload": {"data": "important"}, |
| 500 | "expires_at": "2099-12-31T23:59:59+00:00", |
| 501 | } |
| 502 | |
| 503 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 504 | with pytest.raises(OSError) as exc_info: |
| 505 | _write_remote_records(root, [rec]) |
| 506 | |
| 507 | assert exc_info.value.errno == errno.ENOSPC |
| 508 | |
| 509 | def test_enospc_does_not_silently_produce_empty_head(self, tmp_path: pathlib.Path) -> None: |
| 510 | """A zero-byte HEAD would cause every muse command to fail — must not happen.""" |
| 511 | from muse.core.store import write_text_atomic |
| 512 | |
| 513 | head_path = tmp_path / ".muse" / "HEAD" |
| 514 | head_path.parent.mkdir(parents=True, exist_ok=True) |
| 515 | |
| 516 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 517 | with pytest.raises(OSError): |
| 518 | write_text_atomic(head_path, "ref: refs/heads/main\n") |
| 519 | |
| 520 | # HEAD must not exist at all (not as a zero-byte file) |
| 521 | if head_path.exists(): |
| 522 | assert head_path.stat().st_size > 0, "HEAD was created as zero-byte file" |
| 523 | |
| 524 | |
| 525 | # ============================================================================= |
| 526 | # 5. INTEGRATION — coord _write_remote_records propagates ENOSPC |
| 527 | # ============================================================================= |
| 528 | |
| 529 | |
| 530 | class TestIntegrationCoordEnospc: |
| 531 | """_write_remote_records uses write_text_atomic — ENOSPC must bubble up.""" |
| 532 | |
| 533 | def _make_rec(self, kind: str = "reservation", uuid: str = "res-001") -> MsgpackDict: |
| 534 | return { |
| 535 | "kind": kind, |
| 536 | "record_uuid": uuid, |
| 537 | "run_id": "run-torvalds", |
| 538 | "payload": {"data": "x" * 1024}, |
| 539 | "expires_at": "2099-12-31T23:59:59+00:00", |
| 540 | } |
| 541 | |
| 542 | def test_enospc_raises_from_write_remote_records(self, tmp_path: pathlib.Path) -> None: |
| 543 | root = _make_repo(tmp_path) |
| 544 | from muse.cli.commands.coord_sync import _write_remote_records |
| 545 | |
| 546 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 547 | with pytest.raises(OSError) as exc_info: |
| 548 | _write_remote_records(root, [self._make_rec()]) |
| 549 | assert exc_info.value.errno == errno.ENOSPC |
| 550 | |
| 551 | def test_eio_raises_from_write_remote_records(self, tmp_path: pathlib.Path) -> None: |
| 552 | root = _make_repo(tmp_path) |
| 553 | from muse.cli.commands.coord_sync import _write_remote_records |
| 554 | |
| 555 | with patch("os.fsync", side_effect=_oserror(errno.EIO)): |
| 556 | with pytest.raises(OSError) as exc_info: |
| 557 | _write_remote_records(root, [self._make_rec()]) |
| 558 | assert exc_info.value.errno == errno.EIO |
| 559 | |
| 560 | def test_einval_suppressed_in_write_remote_records(self, tmp_path: pathlib.Path) -> None: |
| 561 | """Virtual filesystem compat: EINVAL from fsync must be suppressed.""" |
| 562 | root = _make_repo(tmp_path) |
| 563 | from muse.cli.commands.coord_sync import _write_remote_records |
| 564 | |
| 565 | with patch("os.fsync", side_effect=_oserror(errno.EINVAL)): |
| 566 | _write_remote_records(root, [self._make_rec()]) # must not raise |
| 567 | |
| 568 | # File must be present and valid |
| 569 | path = ( |
| 570 | tmp_path / ".muse" / "coordination" / "remote" / "reservation" / "res-001.json" |
| 571 | ) |
| 572 | assert path.exists() |
| 573 | |
| 574 | def test_enospc_on_second_record_first_record_still_written(self, tmp_path: pathlib.Path) -> None: |
| 575 | """ENOSPC on the second record must not prevent the first from being written.""" |
| 576 | root = _make_repo(tmp_path) |
| 577 | from muse.cli.commands.coord_sync import _write_remote_records |
| 578 | |
| 579 | recs = [ |
| 580 | self._make_rec("reservation", "res-first"), |
| 581 | self._make_rec("intent", "intent-second"), |
| 582 | ] |
| 583 | |
| 584 | call_count = [0] |
| 585 | original_fsync = os.fsync |
| 586 | |
| 587 | def fsync_side_effect(fd: int) -> None: |
| 588 | call_count[0] += 1 |
| 589 | if call_count[0] >= 2: |
| 590 | raise _oserror(errno.ENOSPC) |
| 591 | return original_fsync(fd) |
| 592 | |
| 593 | with patch("os.fsync", side_effect=fsync_side_effect): |
| 594 | with pytest.raises(OSError): |
| 595 | _write_remote_records(root, recs) |
| 596 | |
| 597 | first_path = ( |
| 598 | tmp_path / ".muse" / "coordination" / "remote" / "reservation" / "res-first.json" |
| 599 | ) |
| 600 | assert first_path.exists(), "first record was not written before ENOSPC" |
| 601 | |
| 602 | |
| 603 | # ============================================================================= |
| 604 | # 6. STRESS — repeated ENOSPC never silently succeeds |
| 605 | # ============================================================================= |
| 606 | |
| 607 | |
| 608 | class TestStressEnospc: |
| 609 | """Repeated ENOSPC must always raise — the bug must never be intermittent.""" |
| 610 | |
| 611 | def test_100_consecutive_enospc_all_raise(self, tmp_path: pathlib.Path) -> None: |
| 612 | from muse.core.store import write_text_atomic |
| 613 | |
| 614 | silent_successes = 0 |
| 615 | for i in range(100): |
| 616 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 617 | try: |
| 618 | write_text_atomic(tmp_path / f"file-{i}.txt", f"data-{i}") |
| 619 | silent_successes += 1 |
| 620 | except OSError: |
| 621 | pass |
| 622 | |
| 623 | assert silent_successes == 0, ( |
| 624 | f"{silent_successes} writes silently succeeded despite ENOSPC" |
| 625 | ) |
| 626 | |
| 627 | def test_concurrent_threads_all_see_enospc(self, tmp_path: pathlib.Path) -> None: |
| 628 | """Under concurrent load, every thread sees ENOSPC — not just some. |
| 629 | |
| 630 | Patch os.fsync globally before spawning threads so the mock is in |
| 631 | place for all of them. Patching inside each thread is unsafe because |
| 632 | `patch` modifies a module-level attribute (global state) and concurrent |
| 633 | `with patch(...)` blocks race with each other. |
| 634 | """ |
| 635 | from muse.core.store import write_text_atomic |
| 636 | |
| 637 | silent_successes = [] |
| 638 | exceptions = [] |
| 639 | lock = threading.Lock() |
| 640 | |
| 641 | def worker(idx: int) -> None: |
| 642 | try: |
| 643 | write_text_atomic(tmp_path / f"t-{idx}.txt", f"data-{idx}") |
| 644 | with lock: |
| 645 | silent_successes.append(idx) |
| 646 | except OSError: |
| 647 | with lock: |
| 648 | exceptions.append(idx) |
| 649 | |
| 650 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 651 | threads = [threading.Thread(target=worker, args=(i,)) for i in range(20)] |
| 652 | for t in threads: |
| 653 | t.start() |
| 654 | for t in threads: |
| 655 | t.join() |
| 656 | |
| 657 | assert silent_successes == [], ( |
| 658 | f"Threads {silent_successes} silently succeeded despite ENOSPC" |
| 659 | ) |
| 660 | assert len(exceptions) == 20 |
| 661 | |
| 662 | def test_no_orphaned_temp_files_after_100_enospc(self, tmp_path: pathlib.Path) -> None: |
| 663 | """100 ENOSPC writes must not leave orphaned temp files.""" |
| 664 | from muse.core.store import write_text_atomic |
| 665 | |
| 666 | for i in range(100): |
| 667 | with patch("os.fsync", side_effect=_oserror(errno.ENOSPC)): |
| 668 | with pytest.raises(OSError): |
| 669 | write_text_atomic(tmp_path / "file.txt", f"data-{i}") |
| 670 | |
| 671 | tmp_files = list(tmp_path.glob(".muse-tmp-*")) |
| 672 | assert tmp_files == [], f"{len(tmp_files)} orphaned temp files" |
| 673 | |
| 674 | |
| 675 | # ============================================================================= |
| 676 | # 7. REGRESSION — EINVAL is still suppressed (virtual filesystem compat) |
| 677 | # ============================================================================= |
| 678 | |
| 679 | |
| 680 | class TestRegressionEinvalSuppressed: |
| 681 | """The fix must not break virtual filesystem compatibility.""" |
| 682 | |
| 683 | def test_write_text_atomic_einval_suppressed(self, tmp_path: pathlib.Path) -> None: |
| 684 | from muse.core.store import write_text_atomic |
| 685 | |
| 686 | with patch("os.fsync", side_effect=_oserror(errno.EINVAL)): |
| 687 | write_text_atomic(tmp_path / "v.txt", "virtual-fs-content") |
| 688 | |
| 689 | assert (tmp_path / "v.txt").read_text() == "virtual-fs-content" |
| 690 | |
| 691 | def test_write_msgpack_atomic_einval_suppressed(self, tmp_path: pathlib.Path) -> None: |
| 692 | from muse.core.store import _write_msgpack_atomic |
| 693 | |
| 694 | with patch("os.fsync", side_effect=_oserror(errno.EINVAL)): |
| 695 | _write_msgpack_atomic(tmp_path / "rec.msgpack", {"k": "v"}) |
| 696 | |
| 697 | assert (tmp_path / "rec.msgpack").exists() |
| 698 | |
| 699 | def test_no_fsync_error_still_works(self, tmp_path: pathlib.Path) -> None: |
| 700 | """When fsync succeeds normally, write_text_atomic still works.""" |
| 701 | from muse.core.store import write_text_atomic |
| 702 | |
| 703 | write_text_atomic(tmp_path / "normal.txt", "hello world") |
| 704 | assert (tmp_path / "normal.txt").read_text() == "hello world" |
| 705 | |
| 706 | def test_docker_tmpfs_compat_einval_suppressed_batch(self, tmp_path: pathlib.Path) -> None: |
| 707 | """20 writes with EINVAL suppressed — simulates Docker tmpfs environment.""" |
| 708 | from muse.core.store import write_text_atomic |
| 709 | |
| 710 | with patch("os.fsync", side_effect=_oserror(errno.EINVAL)): |
| 711 | for i in range(20): |
| 712 | write_text_atomic(tmp_path / f"file-{i}.txt", f"content-{i}") |
| 713 | |
| 714 | for i in range(20): |
| 715 | assert (tmp_path / f"file-{i}.txt").read_text() == f"content-{i}" |
| 716 | |
| 717 | |
| 718 | # ============================================================================= |
| 719 | # 8. PERFORMANCE — suppressed EINVAL (normal path) is not dramatically slower |
| 720 | # ============================================================================= |
| 721 | |
| 722 | |
| 723 | class TestPerformanceNormalPath: |
| 724 | """The fix must not introduce significant overhead to the common (no-error) path.""" |
| 725 | |
| 726 | def test_1000_writes_complete_under_5s(self, tmp_path: pathlib.Path) -> None: |
| 727 | from muse.core.store import write_text_atomic |
| 728 | |
| 729 | t0 = time.monotonic() |
| 730 | for i in range(1000): |
| 731 | write_text_atomic(tmp_path / f"perf-{i:04d}.txt", f"data-{i}" * 32) |
| 732 | elapsed = time.monotonic() - t0 |
| 733 | |
| 734 | assert elapsed < 5.0, f"1000 atomic text writes took {elapsed:.3f}s (> 5s)" |
| 735 | |
| 736 | def test_500_msgpack_writes_complete_under_5s(self, tmp_path: pathlib.Path) -> None: |
| 737 | from muse.core.store import _write_msgpack_atomic |
| 738 | |
| 739 | t0 = time.monotonic() |
| 740 | for i in range(500): |
| 741 | _write_msgpack_atomic( |
| 742 | tmp_path / f"rec-{i:04d}.msgpack", |
| 743 | {"kind": "commit", "seq": i, "payload": "x" * 256}, |
| 744 | ) |
| 745 | elapsed = time.monotonic() - t0 |
| 746 | |
| 747 | assert elapsed < 5.0, f"500 atomic msgpack writes took {elapsed:.3f}s (> 5s)" |
File History
5 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6
chore(tests): add docstring to tests/__init__.py so rc14 tr…
Human
26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62
baseline: rc14 re-baseline after rc3 store corruption recovery
Human
patch
26 days ago
sha256:50d413a635f57761c3fce1f70251b957b6423821525bf330747ef4007339222e
Merge branch 'dev' into main
Human
29 days ago
sha256:fb67fed5a4d3e40de84bdd163de94ef1386570bef1dd1a020a732c8a038962ce
Merge branch 'dev' into main
Human
48 days ago
sha256:1c4b3e3a9a1f300774c3ee662b572a698d5fd405bf765a71e6011a2e9c3eaaaa
feat: Muse — version control for the agent era
Human
100 days ago